<commit_before>// @flow
/* eslint-disable promise/no-native */
const path = require('path');
const assert = require('assert');
const { NotPermittedError } = require('common-errors');
const callsite = require('callsite');
const _require = require('../utils/require');
const { PluginsTypes } = require('../');

/**
 * Validator configuration, more details in
 * https://github.com/makeomatic/ms-validation
 */
export type ValidatorConfig = Array<string> | void | {
  filter: (filename: string) => boolean,
  schemas: Array<string>,
  ajv: Object,
};

/**
 * Validation function signature - promise based
 * @type {Function}
 */
export type Validate = <T>(schema: string, document: T) => Promise<Error | T>;

/**
 * Validation function signature - sync
 * @type {Function}
 */
export type ValidateSync = <T>(schema: string, document: T) => { error?: Error, doc: T };

/**
 * Plugin name
 * @type {String}
 */
exports.name = 'validator';

/**
 * Plugin Type
 * @type {String}
 */
exports.type = PluginsTypes.essential;

/**
 * Attaches initialized validator based on conf.
 * Provides `validate` and `validateSync` methods.
 * @param {Object} conf - Validator Configuration Object.
 * @param {string} parentFile - From which file this plugin was invoked.
 */
exports.attach = function attachValidator(conf: ValidatorConfig, parentFile: string) {
  const service = this;
  const Validator = _require('ms-validation');
  const schemasPath = '../../schemas';
  let validator;
  let schemas;

  if (Array.isArray(conf) || conf === undefined) {
    validator = new Validator(schemasPath);
    schemas = conf;
  } else {
    validator = new Validator(schemasPath, conf.filter, conf.ajv);
    schemas = conf.schemas;
  }

  if (schemas) {
    if (!Array.isArray(schemas)) {
      throw new NotPermittedError('validator schemas must be an array of directories, where json schemas are located');
    }

    // for relative paths
    const stack = callsite();

    // Note that schemas with same file name will be overwritten
    schemas.forEach((_location) => {
      let dir;
      if (!path.isAbsolute(_location)) {
        const length = stack.length;

        // filter out the file itself
        let iterator = 0;
        let source = '';
        while (iterator < length && !source) {
          const call = stack[iterator];
          const filename = call.getFileName();
          if ([parentFile, __filename, 'native array.js'].indexOf(filename) === -1) {
            source = path.dirname(filename);
          }

          iterator += 1;
        }

        dir = path.resolve(source, _location);
      } else {
        dir = _location;
      }

      validator.init(dir);
    });
  }

  // extend service
  service[`_${exports.name}`] = validator;
  service.validate = (validator.validate: Validate);
  service.validateSync = (validator.validateSync: ValidateSync);

  // if we have schema called `config` - we will use it to validate
  if (validator.$ajv.getSchema('config')) {
    assert.ifError(service.validateSync('config', service.config).error);
  }
};<commit_msg>fix(validator): do not crash on null filename<commit_after>// @flow
/* eslint-disable promise/no-native */
const path = require('path');
const assert = require('assert');
const { NotPermittedError } = require('common-errors');
const callsite = require('callsite');
const _require = require('../utils/require');
const { PluginsTypes } = require('../');

/**
 * Validator configuration, more details in
 * https://github.com/makeomatic/ms-validation
 */
export type ValidatorConfig = Array<string> | void | {
  filter: (filename: string) => boolean,
  schemas: Array<string>,
  ajv: Object,
};

/**
 * Validation function signature - promise based
 * @type {Function}
 */
export type Validate = <T>(schema: string, document: T) => Promise<Error | T>;

/**
 * Validation function signature - sync
 * @type {Function}
 */
export type ValidateSync = <T>(schema: string, document: T) => { error?: Error, doc: T };

/**
 * Plugin name
 * @type {String}
 */
exports.name = 'validator';

/**
 * Plugin Type
 * @type {String}
 */
exports.type = PluginsTypes.essential;

/**
 * Attaches initialized validator based on conf.
 * Provides `validate` and `validateSync` methods.
 * @param {Object} conf - Validator Configuration Object.
 * @param {string} parentFile - From which file this plugin was invoked.
 */
exports.attach = function attachValidator(conf: ValidatorConfig, parentFile: string) {
  const service = this;
  const Validator = _require('ms-validation');
  const schemasPath = '../../schemas';
  let validator;
  let schemas;

  if (Array.isArray(conf) || conf === undefined) {
    validator = new Validator(schemasPath);
    schemas = conf;
  } else {
    validator = new Validator(schemasPath, conf.filter, conf.ajv);
    schemas = conf.schemas;
  }

  if (schemas) {
    if (!Array.isArray(schemas)) {
      throw new NotPermittedError('validator schemas must be an array of directories, where json schemas are located');
    }

    // for relative paths
    const stack = callsite();

    // Note that schemas with same file name will be overwritten
    schemas.forEach((_location) => {
      let dir;
      if (!path.isAbsolute(_location)) {
        const length = stack.length;

        // filter out the file itself
        let iterator = 0;
        let source = '';
        while (iterator < length && !source) {
          const call = stack[iterator];
          const filename = call.getFileName();
          if ([parentFile, __filename, 'native array.js', null].indexOf(filename) === -1) {
            source = path.dirname(filename);
          }

          iterator += 1;
        }

        dir = path.resolve(source, _location);
      } else {
        dir = _location;
      }

      validator.init(dir);
    });
  }

  // extend service
  service[`_${exports.name}`] = validator;
  service.validate = (validator.validate: Validate);
  service.validateSync = (validator.validateSync: ValidateSync);

  // if we have schema called `config` - we will use it to validate
  if (validator.$ajv.getSchema('config')) {
    assert.ifError(service.validateSync('config', service.config).error);
  }
};
{'fraction_non_alphanumeric': 0.14091350826044705, 'fraction_numerical': 0.0009718172983479105, 'mean_word_length': 3.138739946380697, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 21, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1275060', 'n_tokens_mistral': 1790, 'n_tokens_neox': 1756, 'n_words': 517}