<commit_before>const fs = require('fs');

let folders = fs.readdirSync('data');

// load data
let data = folders.reduce((obj, folder) => {
  let directory = `data/${folder}`;

  if (fs.lstatSync(directory).isDirectory()) {
    obj[folder] = fs.readdirSync(directory)
      .reduce((obj, file) => {
        let name = file.replace('.json', '');
        obj[name] = require(`./${directory}/${file}`);
        return obj;
      }, {});
  }
  return obj;
}, {});

// validate powers exist in their sources
Object.keys(data.powers)
  .map(d => Object.assign({ id: d }, data.powers[d]))
  .forEach(power => {
    power.sources
      .forEach(source => {
        let type = source.sub_type || source.type;
        let { powers } = data[type][source.id];

        if (!powers || powers.indexOf(power.id) === -1) {
          console.log(`${power.id} is missing from its source ${source.id}`);
          return true;
        }
      });
  });

// validate powers lists have valid powers
['class', 'major', 'minor', 'weapon']
  .map(type => data[type])
  .forEach(type => {
    Object.keys(type)
      .map(d => Object.assign({ id: d }, type[d]))
      .forEach(d => {<commit_msg>adding more validations<commit_after>const fs = require('fs');

const powerTypeMappings = [
  'Buff',
  'Crowd Control',
  'Crowd Control Self',
  'Debuff',
  'Melee Attack',
  'Passive',
  'Ranged Attack',
  'Non-Aggressive',
  'Retaliate',
  'Transformation',
  'Ultimate'
];

const powerCastTypeMappings = [
  'Instant',
  'Channeled',
  'Charged',
  'Ground Aimed',
  'Hold Charged',
  'Hold Channeled',
  'Passive',
  'Toggle'
];

const targetingMappings = [
  'Area',
  'Cone',
  'Group',
  'Point Blank Area',
  'Ray',
  'Rectangle',
  'Reticle',
  'Self',
  'Sphere'
];

// load data
let data = fs.readdirSync('data')
  .reduce((obj, folder) => {
    let directory = `data/${folder}`;

    if (fs.lstatSync(directory).isDirectory()) {
      obj[folder] = fs.readdirSync(directory)
        .reduce((obj, file) => {
          let name = file.replace('.json', '');
          obj[name] = require(`./${directory}/${file}`);
          return obj;
        }, {});
    }
    return obj;
  }, {});

// type checking
Object.keys(data)
  .map(type => data[type])
  .forEach(type => {
    Object.keys(type)
      .map(d => Object.assign({ id: d }, type[d]))
      .forEach(d => {
        // strings
        ['description', 'name']
          .forEach(t => {
            if (d.hasOwnProperty(t) && typeof d[t] !== 'string') {
              console.log(`${d.id}: description is not a valid string`);
            }
          });

        // array
        ['classes', 'races', 'powers', 'sources', 'previous_chain', 'next_chain']
          .forEach(t => {
            if (d.hasOwnProperty(t) && !Array.isArray(d[t])) {
              console.log(`${d.id}: ${t} is not a valid array`);
            }
          });

        ['cooldown', 'duration', 'max_targets', 'range']
          .forEach(t => {
            // duration
            if (d.hasOwnProperty(t) && typeof d[t] !== 'number') {
              console.log(`${d.id}: ${t} is not a valid number`);
            }
          });
      });
  });

// power validations
Object.keys(data.powers)
  .map(d => Object.assign({ id: d }, data.powers[d]))
  .forEach(p => {
    let { cast_type: castType, targeting, type } = p;

    // cast type
    if (powerCastTypeMappings.indexOf(castType) === -1) {
      console.log(`${p.id}: ${castType} is not a valid power cast type`);
    }

    // power type
    if (powerTypeMappings.indexOf(type) === -1) {
      console.log(`${p.id}: ${type} is not a valid power type`);
    }

    // targeting
    if (targetingMappings.indexOf(targeting) === -1) {
      console.log(`${p.id}: ${targeting} is not a valid targeting value`);
    }
  });

// validate powers exist in their sources
Object.keys(data.powers)
  .map(d => Object.assign({ id: d }, data.powers[d]))
  .forEach(power => {
    power.sources
      .forEach(source => {
        let type = source.sub_type || source.type;
        let { powers } = data[type][source.id];

        if (!powers || powers.indexOf(power.id) === -1) {
          console.log(`${power.id} is missing from its source ${source.id}`);
          return true;
        }
      });
  });

// validate powers lists have valid powers
['class', 'major', 'minor', 'weapon']
  .map(type => data[type])
  .forEach(type => {
    Object.keys(type)
      .map(d => Object.assign({ id: d }, type[d]))
      .forEach(d => {
{'fraction_non_alphanumeric': 0.19513826940231935, 'fraction_numerical': 0.0011150758251561106, 'mean_word_length': 2.3874622356495467, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 30, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1716315', 'n_tokens_mistral': 1511, 'n_tokens_neox': 1490, 'n_words': 333}