<commit_before>require('colors');
const glob = require('glob');
const fs = require('fs');
const path = require('path');
const defaultFeatures = JSON.parse(fs.readFileSync(path.join(__dirname, '..', 'package.json'))).defaultFeatures;

const LEAGACY_SETTINGS_PROJECT_DIR = 'source/common/res/features';
const NEW_SETTINGS_PROJECT_DIR = 'sauce/features';
const ALL_SETTINGS_OUTPUT = 'source/common/res/features/allSettings.js';
const REQUIRED_SETTINGS = ['name', 'type', 'default', 'section', 'title'];

let previousSettings;

function run(callback) {
  previousSettings = new Set();
  Promise.all([
    gatherLegacySettings(),
    gatherNewSettings()
  ]).then(values => {
    let validatedSettings = [];
    let settingsConcatenated = values[0].concat(values[1]);
    settingsConcatenated.forEach(setting => {
      if (Array.isArray(setting.setting)) {
        setting.setting.forEach(subSetting => validatedSettings.push(validateSetting({
          setting: subSetting,
          file: setting.file
        })));
      } else {
        validatedSettings.push(validateSetting(setting));
      }
    });

    let allSetingsFile = generateAllSettingsFile(validatedSettings);
    fs.writeFile(ALL_SETTINGS_OUTPUT, allSetingsFile, callback);
  }, reason => {
    callback(reason);
  });
}

function gatherLegacySettings() {
  return new Promise((resolve, reject) => {
    glob(`${LEAGACY_SETTINGS_PROJECT_DIR}/**/settings.js*`, (error, files) => {
      if (error) return reject(error);

      let legacySettingsPromises = [];

      files.forEach(file => {
        legacySettingsPromises.push(new Promise((resolve, reject) => {
          let setting;
          const filePath = `${__dirname}/../${file}`;<commit_msg>generate settings outside of webpack<commit_after>require('colors');
const glob = require('glob');
const fs = require('fs');
const path = require('path');
const defaultFeatures = JSON.parse(fs.readFileSync(path.join(__dirname, '..', 'package.json'))).defaultFeatures;

const LEAGACY_SETTINGS_PROJECT_DIR = 'source/common/res/features';
const NEW_SETTINGS_PROJECT_DIR = 'sauce/features';
const ALL_SETTINGS_OUTPUT = 'source/common/res/features/allSettings.js';
const REQUIRED_SETTINGS = ['name', 'type', 'default', 'section', 'title'];

let previousSettings;

function run(callback) {
  previousSettings = new Set();
  Promise.all([
    gatherLegacySettings(),
    gatherNewSettings()
  ]).then(values => {
    let validatedSettings = [];
    let settingsConcatenated = values[0].concat(values[1]);
    settingsConcatenated.forEach(setting => {
      if (Array.isArray(setting.setting)) {
        setting.setting.forEach(subSetting => {
          let validatedSetting = validateSetting({
            setting: subSetting,
            file: setting.file
          });

          if (validatedSetting.hidden !== true) {
            validatedSettings.push(validatedSetting);
          }
        });
      } else {
        let validatedSetting = validateSetting(setting);

        if (validatedSetting.hidden !== true) {
          validatedSettings.push(validatedSetting);
        }
      }
    });

    let allSetingsFile = generateAllSettingsFile(validatedSettings);
    fs.writeFile(ALL_SETTINGS_OUTPUT, allSetingsFile, callback);
  }, reason => {
    callback(reason);
  }).catch(exception => {
    callback(exception.stack);
  });
}

function gatherLegacySettings() {
  return new Promise((resolve, reject) => {
    glob(`${LEAGACY_SETTINGS_PROJECT_DIR}/**/settings.js*`, (error, files) => {
      if (error) return reject(error);

      let legacySettingsPromises = [];

      files.forEach(file => {
        legacySettingsPromises.push(new Promise((resolve, reject) => {
          let setting;
          const filePath = `${__dirname}/../${file}`;
{'fraction_non_alphanumeric': 0.14357883541611274, 'fraction_numerical': 0.0010635469290082426, 'mean_word_length': 3.732075471698113, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 20, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4979646', 'n_tokens_mistral': 1091, 'n_tokens_neox': 1096, 'n_words': 195}