<commit_before>'use strict';

var tv4 = require('tv4');
var _ = require('lodash');
var path = require('path');
var recursiveReadSync = require('recursive-readdir-sync');
var Promise = require('es6-promise').Promise;

var formats = require('./formats');
var errors = require('../errors');

function Validate (schemaDir) {
  this.schemaDir = schemaDir;
  this.schemas = {};
  this.tv4 = tv4.freshApi();
  this.tv4.addFormat(formats);
  this.readSchemas();
}

/**
 * Add all schemas found in directory
 */
Validate.prototype.readSchemas = function () {
  var self = this;

  var files = [];
  try {
    files = recursiveReadSync(self.schemaDir);
  } catch(e) {
  }

  // Only use .json files
  files = _.filter(files, function(file) {
    return path.extname(file) === '.json';
  });

  // Add all schema files and cache them
  files.forEach(function(schema) {
    var raw = self._requireSchema(schema);
    self.tv4.addSchema(raw);
  });
};

/**
 * Validate props based on schema path
 *
 * @param {string} schemaPath - Path to schema file
 * @param {object} props - Data to validate
 */
Validate.prototype.schema = function (schemaPath, props) {
  var self = this;
  return new Promise(function (resolve, reject) {
    schemaPath = path.join(self.schemaDir, schemaPath+'.json');
    schemaPath = path.resolve(schemaPath);

    var schema = self._requireSchema(schemaPath);
    if (!schema) {
      return reject(new Error('invalid schema'));
    }

    var result = self.tv4.validateMultiple(props, schema);
    if (result.missing && result.missing.length > 0) {
      var nonEmpty = _.filter(result.missing, function (missing) {
        return !!missing.length;
      });
      if (nonEmpty.length === 0) {
        return reject(new Error('invalid $refs, missing schemas'));
      }
      var missing = result.missing.join(',');
      return reject(new Error('missing schemas: ' + missing));
    }
    if (result.valid) {
      return resolve(props);
    }

    var message = Validate.extractErrors(result.errors);
    reject(new errors.BadRequest(message));
  });
};


/**
 * Read properties of a schema
 *
 * @param {string} schemaPath - Path to schema file
 * @param {string} name - Schema to grab properties from
 */
Validate.prototype.readProperties = function (schemaPath) {
  schemaPath = path.join(this.schemaDir, 'schemaPath.json');
  var schema = this._requireSchema(schemaPath);
  return Object.keys(schema.properties);
};

/**
 * Read schema file
 *
 * @param {string} schemaPath
 */
Validate.prototype._requireSchema = function (schemaPath) {
  if (this.schemas[schemaPath]) {
    return this.schemas[schemaPath];
  }

  var self = this;
  try {
    self.schemas[schemaPath] = require(schemaPath);
  } catch(ignoreErr) {
    console.warn('could not read schema', ignoreErr);
  }
  return this.schemas[schemaPath];
};

/**
 * Identify validation errors
 *
 * @param {object} errs - TV4 errors object
 * @param {boolean} stopRecurse
 */
Validate.extractErrors = function (errs, stopRecurse) {
  var messages = [];
  var hasErrors = false;

  var extra = Validate.findPropsWithError(errs, 303);
  if (extra.length) {
    hasErrors = true;
    messages.push('extraneous properties: ' + extra.join(', '));
  }

  var missing = Validate.findPropsWithError(errs, 302);
  if (missing.length) {<commit_msg>Do not ignore missing schema<commit_after>'use strict';

var tv4 = require('tv4');
var _ = require('lodash');
var path = require('path');
var recursiveReadSync = require('recursive-readdir-sync');
var Promise = require('es6-promise').Promise;

var formats = require('./formats');
var errors = require('../errors');

function Validate (schemaDir) {
  this.schemaDir = schemaDir;
  this.schemas = {};
  this.tv4 = tv4.freshApi();
  this.tv4.addFormat(formats);
  this.readSchemas();
}

/**
 * Add all schemas found in directory
 */
Validate.prototype.readSchemas = function () {
  var self = this;

  var files = [];
  try {
    files = recursiveReadSync(self.schemaDir);
  } catch(e) {
  }

  // Only use .json files
  files = _.filter(files, function(file) {
    return path.extname(file) === '.json';
  });

  // Add all schema files and cache them
  files.forEach(function(schema) {
    var raw = self._requireSchema(schema);
    self.tv4.addSchema(raw);
  });
};

/**
 * Validate props based on schema path
 *
 * @param {string} schemaPath - Path to schema file
 * @param {object} props - Data to validate
 */
Validate.prototype.schema = function (schemaPath, props) {
  var self = this;
  return new Promise(function (resolve, reject) {
    schemaPath = path.join(self.schemaDir, schemaPath+'.json');
    schemaPath = path.resolve(schemaPath);

    var schema = self._requireSchema(schemaPath);
    if (!schema) {
      return reject(new Error('invalid schema'));
    }

    var result = self.tv4.validateMultiple(props, schema);
    if (result.missing && result.missing.length > 0) {
      var nonEmpty = _.filter(result.missing, function (missing) {
        return !!missing.length;
      });
      if (nonEmpty.length === 0) {
        return reject(new Error('invalid $refs, missing schemas'));
      }
      var missing = result.missing.join(',');
      return reject(new Error('missing schemas: ' + missing));
    }
    if (result.valid) {
      return resolve(props);
    }

    var message = Validate.extractErrors(result.errors);
    reject(new errors.BadRequest(message));
  });
};


/**
 * Read properties of a schema
 *
 * @param {string} schemaPath - Path to schema file
 * @param {string} name - Schema to grab properties from
 */
Validate.prototype.readProperties = function (schemaPath) {
  schemaPath = path.join(this.schemaDir, 'schemaPath.json');
  var schema = this._requireSchema(schemaPath);
  return Object.keys(schema.properties);
};

/**
 * Read schema file
 *
 * @param {string} schemaPath
 */
Validate.prototype._requireSchema = function (schemaPath) {
  if (this.schemas[schemaPath]) {
    return this.schemas[schemaPath];
  }

  var self = this;
  self.schemas[schemaPath] = require(schemaPath);
  return this.schemas[schemaPath];
};

/**
 * Identify validation errors
 *
 * @param {object} errs - TV4 errors object
 * @param {boolean} stopRecurse
 */
Validate.extractErrors = function (errs, stopRecurse) {
  var messages = [];
  var hasErrors = false;

  var extra = Validate.findPropsWithError(errs, 303);
  if (extra.length) {
    hasErrors = true;
    messages.push('extraneous properties: ' + extra.join(', '));
  }

  var missing = Validate.findPropsWithError(errs, 302);
  if (missing.length) {
{'fraction_non_alphanumeric': 0.14471271393643031, 'fraction_numerical': 0.00519559902200489, 'mean_word_length': 3.573724668064291, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2835098', 'n_tokens_mistral': 2033, 'n_tokens_neox': 1992, 'n_words': 503}