<commit_before>const fs = require('fs');
const semver = require('semver');
const mime = require('mime');

module.exports = () => {
  const libraries = {};
  const caches = {};
  const updaters = {};

  for (const folder of fs.readdirSync('libraries/libs')) {
    const versions = fs.readdirSync(`libraries/libs/${folder}`).filter(v => !v.endsWith('.json'));

    versions.sort(semver.compare);
    versions.reverse();

    const final = {};
    for (const version of versions) {
      final[version] = {};
      final[version].files = fs.readdirSync(`libraries/libs/${folder}/${version}`);
      if (final[version].files.indexOf('library.json') < 0) {
        delete final[version];
        continue;
      }
      final[version].info = JSON.parse(fs.readFileSync(`libraries/libs/${folder}/${version}/library.json`));
    }

    const cache = {};
    for (const version of versions) {
      cache[version] = {};
      const path = `libraries/libs/${folder}/${version}/${final[version].info.index}`;
      cache[version].file = fs.readFileSync(path, 'utf8');
      cache[version].mime = mime.lookup(path);
    }

    let updater = {};

    try {
      updater = JSON.parse(fs.readFileSync(`libraries/libs/${folder}/updater.json`));
    } catch (err) {
      updater = {};
    }

    libraries[folder] = {
      versions: final,
      name: folder,
      latestVersion: versions[0],
      cache,
      updater,
    };
    caches[folder] = cache;
    updaters[folder] = updater;
  }
<commit_msg>whoops<commit_after>const fs = require('fs');
const semver = require('semver');
const mime = require('mime');

module.exports = () => {
  const libraries = {};
  const caches = {};
  const updaters = {};

  for (const folder of fs.readdirSync('libraries/libs')) {
    const versions = fs.readdirSync(`libraries/libs/${folder}`).filter(v => !v.endsWith('.json'));

    versions.sort(semver.compare);
    versions.reverse();

    const final = {};
    for (const version of versions) {
      final[version] = {};
      final[version].files = fs.readdirSync(`libraries/libs/${folder}/${version}`);
      if (final[version].files.indexOf('library.json') < 0) {
        delete final[version];
        continue;
      }
      final[version].info = JSON.parse(fs.readFileSync(`libraries/libs/${folder}/${version}/library.json`));
    }

    const cache = {};
    for (const version of versions) {
      cache[version] = {};
      const path = `libraries/libs/${folder}/${version}/${final[version].info.index}`;
      cache[version].file = fs.readFileSync(path, 'utf8');
      cache[version].mime = mime.lookup(path);
    }

    let updater = {};

    try {
      updater = JSON.parse(fs.readFileSync(`libraries/libs/${folder}/updater.json`));
    } catch (err) {
      updater = {};
    }

    libraries[folder] = {
      versions: final,
      name: folder,
      latestVersion: versions[0],
    };
    caches[folder] = cache;
    updaters[folder] = updater;
  }

{'fraction_non_alphanumeric': 0.18040013563919974, 'fraction_numerical': 0.002034587995930824, 'mean_word_length': 3.0027137042062417, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6307554', 'n_tokens_mistral': 947, 'n_tokens_neox': 937, 'n_words': 162}