<commit_before>'use strict';

var _ = require('lodash'),
    apiUmbrellaConfig = require('api-umbrella-config'),
    async = require('async'),
    clc = require('cli-color'),
    config = require('api-umbrella-config').global(),
    events = require('events'),
    execFile = require('child_process').execFile,
    forever = require('forever-monitor'),
    fs = require('fs'),
    fsExtra = require('fs-extra'),
    glob = require('glob'),
    handlebars = require('handlebars'),
    http = require('http'),
    mkdirp = require('mkdirp'),
    mongoConnect = require('./mongo_connect'),
    mongoose = require('mongoose'),
    net = require('net'),
    path = require('path'),
    posix = require('posix'),
    processEnv = require('./process_env'),
    supervisord = require('supervisord'),
    url = require('url'),
    util = require('util'),
    uuid = require('node-uuid'),
    yaml = require('js-yaml');

var models = require('api-umbrella-gatekeeper').models(mongoose);

var ApiUser = models.ApiUser;

var Router = function() {
  this.initialize.apply(this, arguments);
};

util.inherits(Router, events.EventEmitter);
_.extend(Router.prototype, {
  initialize: function(options) {
    process.stdout.write('Starting api-umbrella...');
    this.startWaitLog = setInterval(function() {
      process.stdout.write('.');
    }, 2000);

    process.on('SIGHUP', this.reload.bind(this));
    process.on('SIGINT', this.stop.bind(this, function() {
      process.exit(0);
    }));

    apiUmbrellaConfig.loader({
      defaults: {},
      paths: [
        path.resolve(__dirname, '../config/default.yml'),
      ].concat(options.config),
    }, this.handleConfigReady.bind(this));
  },

  handleConfigReady: function(error, configLoader) {
    this.configLoader = configLoader;

    async.series([
      this.setupGlobalConfig.bind(this),
      this.permissionCheck.bind(this),
      this.internalConfigDefaults.bind(this),
      this.computedConfigValues.bind(this),
      this.reloadGlobalConfig.bind(this),
      this.prepare.bind(this),
      this.writeTemplates.bind(this),
      this.startSupervisor.bind(this),
      this.waitForProcesses.bind(this),
      this.waitForConnections.bind(this),
      mongoConnect,
      this.migrateDatabase.bind(this),
      this.migrateDatabaseWeb.bind(this),
      this.seedDatabase.bind(this),
      this.writeStaticSiteEnv.bind(this),
    ], this.finishStart.bind(this));
  },

  reload: function() {
    var configPath = processEnv.supervisordConfigPath();
    var execOpts = {
      env: processEnv.env(),
    };

    execFile('supervisorctl', ['-c', configPath, 'update'], execOpts, function() {
      async.parallel([
        function(callback) {
          execFile('supervisorctl', ['-c', configPath, 'kill', 'HUP', 'router-nginx'], execOpts, callback);
        },
        function(callback) {
          execFile('supervisorctl', ['-c', configPath, 'kill', 'HUP', 'web-nginx'], execOpts, callback);
        },
        function(callback) {
          execFile('supervisorctl', ['-c', configPath, 'kill', 'USR2', 'web-puma'], execOpts, callback);
        },
        function(callback) {
          execFile('supervisorctl', ['-c', configPath, 'serialrestart', 'gatekeeper:*'], execOpts, callback);
        },
        function(callback) {
          execFile('supervisorctl', ['-c', configPath, 'restart', 'config-reloader', 'distributed-rate-limits-sync', 'log-processor', 'router-log-listener'], execOpts, callback);
        },
      ], function(error) {
        if(error) {
          console.error('Error reloading: ', error);
        }
      });
    }.bind(this));
  },

  stop: function(callback) {
    this.stopCallback = callback;

    process.stdout.write('\nStopping api-umbrella...');

    if(this.configLoader) {
      this.configLoader.close();
    }

    if(this.supervisordProcess && this.supervisordProcess.running) {
      this.supervisordProcess.stop();
    } else {
      this.finishStop();
    }
  },

  prepare: function(callback) {
    mkdirp.sync(path.join(config.get('db_dir'), 'elasticsearch'));
    mkdirp.sync(path.join(config.get('db_dir'), 'mongodb'));
    mkdirp.sync(path.join(config.get('db_dir'), 'redis'));
    mkdirp.sync(path.join(config.get('etc_dir'), 'elasticsearch'));
    mkdirp.sync(path.join(config.get('etc_dir'), 'nginx'));
    mkdirp.sync(path.join(config.get('log_dir'), 'supervisor'));
    mkdirp.sync(path.join(config.get('run_dir'), 'varnish/api-umbrella'));
    mkdirp.sync(config.get('tmp_dir'), { mode: parseInt('0777', 8) });

    if(config.get('user')) {
      var uid = posix.getpwnam(config.get('user')).uid;
      var gid = posix.getgrnam(config.get('group')).gid;

      fs.chownSync(path.join(config.get('db_dir'), 'elasticsearch'), uid, gid);
      fs.chownSync(path.join(config.get('db_dir'), 'mongodb'), uid, gid);
      fs.chownSync(path.join(config.get('db_dir'), 'redis'), uid, gid);
      fs.chownSync(path.join(config.get('log_dir'), 'supervisor'), uid, gid);
      fs.chownSync(path.join(config.get('run_dir'), 'varnish'), uid, gid);
      fs.chownSync(path.join(config.get('run_dir'), 'varnish/api-umbrella'), uid, gid);
      fs.chownSync(config.get('tmp_dir'), uid, gid);
    }

    var primaryHostsNoSsl = _.filter(config.get('hosts'), function(host) {
      return !host.secondary && !host.ssl_cert;
    });

    var sslDir = path.join(config.get('etc_dir'), 'ssl');
    var sslKeyPath = path.join(sslDir, 'self_signed.key');
    var sslCrtPath = path.join(sslDir, 'self_signed.crt');

    var generateSelfSignedCert = false;
    if(primaryHostsNoSsl.length > 0) {
      if(!fs.existsSync(sslKeyPath) || !fs.existsSync(sslCrtPath)) {
        generateSelfSignedCert = true;
      }
    }

    if(generateSelfSignedCert) {
      mkdirp.sync(sslDir);
      execFile('openssl', [
        'req',
        '-new',
        '-newkey', 'rsa:2048',
        '-days', 365 * 5,
        '-nodes',
        '-x509',
        '-subj', '/C=/ST=/L=/O=API Umbrella/CN=apiumbrella.example.com',
        '-keyout', sslKeyPath,
        '-out', sslCrtPath,
      ], {
        env: processEnv.env(),
      }, callback);
    } else {
      callback();
    }
  },

  internalConfigDefaults: function(callback) {
    var filePath = path.join(config.get('run_dir'), 'internal_config_defaults.yml');
    var defaults = yaml.safeDump({
      internal_apis: [
        {
          _id: 'api-umbrella-web-backend',
          name: 'API Umbrella - Default',
          frontend_host: '*',
          backend_host: 'localhost',
          backend_protocol: 'http',
          balance_algorithm: 'least_conn',
          sort_order: 1,
          servers: [
            {
              _id: uuid.v4(),
              host: 'localhost',
              port: config.get('web.port'),
            }
          ],
          url_matches: [
            {
              _id: uuid.v4(),
              frontend_prefix: '/api-umbrella/',
              backend_prefix: '/api-umbrella/',
            }
          ],
          sub_settings: [
            {
              _id: uuid.v4(),
              http_method: 'post',
              regex: '^/api-umbrella/v1/users',
              settings: {
                _id: uuid.v4(),
                required_roles: ['api-umbrella-key-creator'],
              },
            },
            {
              _id: uuid.v4(),
              http_method: 'post',
              regex: '^/api-umbrella/v1/contact',
              settings: {
                _id: uuid.v4(),
                required_roles: ['api-umbrella-contact-form'],
              },
            },
          ],
        },
      ],
    });

    fs.writeFile(filePath, defaults, function(error) {
      if(error) { return callback(error); }

      this.configLoader.options.paths.push(filePath);
      this.configLoader.reload(callback);
    }.bind(this));
  },

  computedConfigValues: function(callback) {
    var filePath = path.join(config.get('run_dir'), 'computed_config_values.yml');
    var defaults = yaml.safeDump({
      service_general_db_enabled: _.contains(config.get('services'), 'general_db'),
      service_log_db_enabled: _.contains(config.get('services'), 'log_db'),
      service_router_enabled: _.contains(config.get('services'), 'router'),
      service_web_enabled: _.contains(config.get('services'), 'web'),
    });

    fs.writeFile(filePath, defaults, function(error) {
      if(error) { return callback(error); }

      this.configLoader.options.paths.push(filePath);
      this.configLoader.reload(callback);
    }.bind(this));
  },

  setupGlobalConfig: function(callback) {
    apiUmbrellaConfig.setGlobal(this.configLoader.runtimeFile);
    config = require('api-umbrella-config').global();

    // A dance to move the runtime config file into a less temporary location.
    //
    // This could maybe be addressed better in the api-umbrella-config module,
    // but we want to install the runtime config file into a location based on
    // paths defined in the config file, so it's a little tricky.
    //
    // Copying the file also prevents a build-up of lots of temporary yaml
    // files, which is how api-umbrella-config will behave by default (since we
    // don't necessarily want api-umbrella-config to cleanup the files when it
    // closes, since other processes might be reading the files).
    var tempConfigPath = this.configLoader.runtimeFile;
    try {
      var permanentConfigPath = path.join(config.get('run_dir'), 'runtime_config.yml');
      fsExtra.copySync(tempConfigPath, permanentConfigPath);
      this.configLoader.runtimeFile = permanentConfigPath;
      apiUmbrellaConfig.setGlobal(this.configLoader.runtimeFile);
      config.reload();
    } finally {
      // Always cleanup the temp file, even if the app dies because
      // permanentConfigPath is not writable.
      fs.unlinkSync(tempConfigPath);
    }

    processEnv.overrideEnv({
      'API_UMBRELLA_CONFIG': this.configLoader.runtimeFile,
    });

    callback();
  },

  reloadGlobalConfig: function(callback) {
    config.reload();
    callback();
  },

  permissionCheck: function(callback) {
    if(config.get('user')) {
      if(posix.geteuid() !== 0) {
        return callback('Must be started with super-user privileges to change user to \'' + config.get('user') + '\'');
      }

      try {
        posix.getpwnam(config.get('user'));
      } catch(e) {
        return callback('User \'' + config.get('user') + '\' does not exist');
      }

    }

    if(config.get('group')) {
      if(posix.geteuid() !== 0) {
        return callback('Must be started with super-user privileges to change group to \'' + config.get('group') + '\'');
      }

      try {
        posix.getgrnam(config.get('group'));
      } catch(e) {
        return callback('Group \'' + config.get('group') + '\' does not exist');
      }

      // Change the process group id so the api-umbrella-config file that gets
      // written is readable by the api-umbrella group.
      process.setgid(config.get('group'));
    }

    if(config.get('http_port') < 1024 || config.get('https_port') < 1024) {
      if(posix.geteuid() !== 0) {
        return callback('Must be started with super-user privileges to use http ports below 1024');
      }
    }

    if(posix.geteuid() === 0) {
      if(!config.get('user') || !config.get('group')) {
        return callback('Must define a user and group to run worker processes as when starting with with super-user privileges');
      }
    }

    callback();
  },

  migrateDatabase: function(callback) {
    execFile('east', ['migrate'], {
      cwd: path.resolve(__dirname, '../'),
      env: processEnv.env(),
    }, callback);<commit_msg>Fix migrations don't get run on first-time installs.<commit_after>'use strict';

var _ = require('lodash'),
    apiUmbrellaConfig = require('api-umbrella-config'),
    async = require('async'),
    clc = require('cli-color'),
    config = require('api-umbrella-config').global(),
    events = require('events'),
    execFile = require('child_process').execFile,
    forever = require('forever-monitor'),
    fs = require('fs'),
    fsExtra = require('fs-extra'),
    glob = require('glob'),
    handlebars = require('handlebars'),
    http = require('http'),
    mkdirp = require('mkdirp'),
    mongoConnect = require('./mongo_connect'),
    mongoose = require('mongoose'),
    net = require('net'),
    path = require('path'),
    posix = require('posix'),
    processEnv = require('./process_env'),
    supervisord = require('supervisord'),
    url = require('url'),
    util = require('util'),
    uuid = require('node-uuid'),
    yaml = require('js-yaml');

var models = require('api-umbrella-gatekeeper').models(mongoose);

var ApiUser = models.ApiUser;

var Router = function() {
  this.initialize.apply(this, arguments);
};

util.inherits(Router, events.EventEmitter);
_.extend(Router.prototype, {
  initialize: function(options) {
    process.stdout.write('Starting api-umbrella...');
    this.startWaitLog = setInterval(function() {
      process.stdout.write('.');
    }, 2000);

    process.on('SIGHUP', this.reload.bind(this));
    process.on('SIGINT', this.stop.bind(this, function() {
      process.exit(0);
    }));

    apiUmbrellaConfig.loader({
      defaults: {},
      paths: [
        path.resolve(__dirname, '../config/default.yml'),
      ].concat(options.config),
    }, this.handleConfigReady.bind(this));
  },

  handleConfigReady: function(error, configLoader) {
    this.configLoader = configLoader;

    async.series([
      this.setupGlobalConfig.bind(this),
      this.permissionCheck.bind(this),
      this.internalConfigDefaults.bind(this),
      this.computedConfigValues.bind(this),
      this.reloadGlobalConfig.bind(this),
      this.prepare.bind(this),
      this.writeTemplates.bind(this),
      this.startSupervisor.bind(this),
      this.waitForProcesses.bind(this),
      this.waitForConnections.bind(this),
      mongoConnect,
      this.preMigrateNewDatabaseRouter.bind(this),
      this.preMigrateNewDatabaseWeb.bind(this),
      this.migrateDatabaseRouter.bind(this),
      this.migrateDatabaseWeb.bind(this),
      this.seedDatabase.bind(this),
      this.writeStaticSiteEnv.bind(this),
    ], this.finishStart.bind(this));
  },

  reload: function() {
    var configPath = processEnv.supervisordConfigPath();
    var execOpts = {
      env: processEnv.env(),
    };

    execFile('supervisorctl', ['-c', configPath, 'update'], execOpts, function() {
      async.parallel([
        function(callback) {
          execFile('supervisorctl', ['-c', configPath, 'kill', 'HUP', 'router-nginx'], execOpts, callback);
        },
        function(callback) {
          execFile('supervisorctl', ['-c', configPath, 'kill', 'HUP', 'web-nginx'], execOpts, callback);
        },
        function(callback) {
          execFile('supervisorctl', ['-c', configPath, 'kill', 'USR2', 'web-puma'], execOpts, callback);
        },
        function(callback) {
          execFile('supervisorctl', ['-c', configPath, 'serialrestart', 'gatekeeper:*'], execOpts, callback);
        },
        function(callback) {
          execFile('supervisorctl', ['-c', configPath, 'restart', 'config-reloader', 'distributed-rate-limits-sync', 'log-processor', 'router-log-listener'], execOpts, callback);
        },
      ], function(error) {
        if(error) {
          console.error('Error reloading: ', error);
        }
      });
    }.bind(this));
  },

  stop: function(callback) {
    this.stopCallback = callback;

    process.stdout.write('\nStopping api-umbrella...');

    if(this.configLoader) {
      this.configLoader.close();
    }

    if(this.supervisordProcess && this.supervisordProcess.running) {
      this.supervisordProcess.stop();
    } else {
      this.finishStop();
    }
  },

  prepare: function(callback) {
    mkdirp.sync(path.join(config.get('db_dir'), 'elasticsearch'));
    mkdirp.sync(path.join(config.get('db_dir'), 'mongodb'));
    mkdirp.sync(path.join(config.get('db_dir'), 'redis'));
    mkdirp.sync(path.join(config.get('etc_dir'), 'elasticsearch'));
    mkdirp.sync(path.join(config.get('etc_dir'), 'nginx'));
    mkdirp.sync(path.join(config.get('log_dir'), 'supervisor'));
    mkdirp.sync(path.join(config.get('run_dir'), 'varnish/api-umbrella'));
    mkdirp.sync(config.get('tmp_dir'), { mode: parseInt('0777', 8) });

    if(config.get('user')) {
      var uid = posix.getpwnam(config.get('user')).uid;
      var gid = posix.getgrnam(config.get('group')).gid;

      fs.chownSync(path.join(config.get('db_dir'), 'elasticsearch'), uid, gid);
      fs.chownSync(path.join(config.get('db_dir'), 'mongodb'), uid, gid);
      fs.chownSync(path.join(config.get('db_dir'), 'redis'), uid, gid);
      fs.chownSync(path.join(config.get('log_dir'), 'supervisor'), uid, gid);
      fs.chownSync(path.join(config.get('run_dir'), 'varnish'), uid, gid);
      fs.chownSync(path.join(config.get('run_dir'), 'varnish/api-umbrella'), uid, gid);
      fs.chownSync(config.get('tmp_dir'), uid, gid);
    }

    var primaryHostsNoSsl = _.filter(config.get('hosts'), function(host) {
      return !host.secondary && !host.ssl_cert;
    });

    var sslDir = path.join(config.get('etc_dir'), 'ssl');
    var sslKeyPath = path.join(sslDir, 'self_signed.key');
    var sslCrtPath = path.join(sslDir, 'self_signed.crt');

    var generateSelfSignedCert = false;
    if(primaryHostsNoSsl.length > 0) {
      if(!fs.existsSync(sslKeyPath) || !fs.existsSync(sslCrtPath)) {
        generateSelfSignedCert = true;
      }
    }

    if(generateSelfSignedCert) {
      mkdirp.sync(sslDir);
      execFile('openssl', [
        'req',
        '-new',
        '-newkey', 'rsa:2048',
        '-days', 365 * 5,
        '-nodes',
        '-x509',
        '-subj', '/C=/ST=/L=/O=API Umbrella/CN=apiumbrella.example.com',
        '-keyout', sslKeyPath,
        '-out', sslCrtPath,
      ], {
        env: processEnv.env(),
      }, callback);
    } else {
      callback();
    }
  },

  internalConfigDefaults: function(callback) {
    var filePath = path.join(config.get('run_dir'), 'internal_config_defaults.yml');
    var defaults = yaml.safeDump({
      internal_apis: [
        {
          _id: 'api-umbrella-web-backend',
          name: 'API Umbrella - Default',
          frontend_host: '*',
          backend_host: 'localhost',
          backend_protocol: 'http',
          balance_algorithm: 'least_conn',
          sort_order: 1,
          servers: [
            {
              _id: uuid.v4(),
              host: 'localhost',
              port: config.get('web.port'),
            }
          ],
          url_matches: [
            {
              _id: uuid.v4(),
              frontend_prefix: '/api-umbrella/',
              backend_prefix: '/api-umbrella/',
            }
          ],
          sub_settings: [
            {
              _id: uuid.v4(),
              http_method: 'post',
              regex: '^/api-umbrella/v1/users',
              settings: {
                _id: uuid.v4(),
                required_roles: ['api-umbrella-key-creator'],
              },
            },
            {
              _id: uuid.v4(),
              http_method: 'post',
              regex: '^/api-umbrella/v1/contact',
              settings: {
                _id: uuid.v4(),
                required_roles: ['api-umbrella-contact-form'],
              },
            },
          ],
        },
      ],
    });

    fs.writeFile(filePath, defaults, function(error) {
      if(error) { return callback(error); }

      this.configLoader.options.paths.push(filePath);
      this.configLoader.reload(callback);
    }.bind(this));
  },

  computedConfigValues: function(callback) {
    var filePath = path.join(config.get('run_dir'), 'computed_config_values.yml');
    var defaults = yaml.safeDump({
      service_general_db_enabled: _.contains(config.get('services'), 'general_db'),
      service_log_db_enabled: _.contains(config.get('services'), 'log_db'),
      service_router_enabled: _.contains(config.get('services'), 'router'),
      service_web_enabled: _.contains(config.get('services'), 'web'),
    });

    fs.writeFile(filePath, defaults, function(error) {
      if(error) { return callback(error); }

      this.configLoader.options.paths.push(filePath);
      this.configLoader.reload(callback);
    }.bind(this));
  },

  setupGlobalConfig: function(callback) {
    apiUmbrellaConfig.setGlobal(this.configLoader.runtimeFile);
    config = require('api-umbrella-config').global();

    // A dance to move the runtime config file into a less temporary location.
    //
    // This could maybe be addressed better in the api-umbrella-config module,
    // but we want to install the runtime config file into a location based on
    // paths defined in the config file, so it's a little tricky.
    //
    // Copying the file also prevents a build-up of lots of temporary yaml
    // files, which is how api-umbrella-config will behave by default (since we
    // don't necessarily want api-umbrella-config to cleanup the files when it
    // closes, since other processes might be reading the files).
    var tempConfigPath = this.configLoader.runtimeFile;
    try {
      var permanentConfigPath = path.join(config.get('run_dir'), 'runtime_config.yml');
      fsExtra.copySync(tempConfigPath, permanentConfigPath);
      this.configLoader.runtimeFile = permanentConfigPath;
      apiUmbrellaConfig.setGlobal(this.configLoader.runtimeFile);
      config.reload();
    } finally {
      // Always cleanup the temp file, even if the app dies because
      // permanentConfigPath is not writable.
      fs.unlinkSync(tempConfigPath);
    }

    processEnv.overrideEnv({
      'API_UMBRELLA_CONFIG': this.configLoader.runtimeFile,
    });

    callback();
  },

  reloadGlobalConfig: function(callback) {
    config.reload();
    callback();
  },

  permissionCheck: function(callback) {
    if(config.get('user')) {
      if(posix.geteuid() !== 0) {
        return callback('Must be started with super-user privileges to change user to \'' + config.get('user') + '\'');
      }

      try {
        posix.getpwnam(config.get('user'));
      } catch(e) {
        return callback('User \'' + config.get('user') + '\' does not exist');
      }

    }

    if(config.get('group')) {
      if(posix.geteuid() !== 0) {
        return callback('Must be started with super-user privileges to change group to \'' + config.get('group') + '\'');
      }

      try {
        posix.getgrnam(config.get('group'));
      } catch(e) {
        return callback('Group \'' + config.get('group') + '\' does not exist');
      }

      // Change the process group id so the api-umbrella-config file that gets
      // written is readable by the api-umbrella group.
      process.setgid(config.get('group'));
    }

    if(config.get('http_port') < 1024 || config.get('https_port') < 1024) {
      if(posix.geteuid() !== 0) {
        return callback('Must be started with super-user privileges to use http ports below 1024');
      }
    }

    if(posix.geteuid() === 0) {
      if(!config.get('user') || !config.get('group')) {
        return callback('Must define a user and group to run worker processes as when starting with with super-user privileges');
      }
    }

    callback();
  },

  preMigrateNewDatabaseRouter: function(callback) {
    var db = mongoose.connection.db;
    var collectionName = '_migrations';
    db.collection(collectionName, { strict: true }, function(error) {
      // If the collection doesn't yet exist, assume this is a new
      // installation, so we can mark all of our migrations as already done.
      if(error) {
        db.createCollection(collectionName, function(error, collection) {
          glob(path.resolve(__dirname, '../migrations/*.js'), function(error, migrationPaths) {
            if(error) { return callback(error); }

            var migrations = _.map(migrationPaths, function(migrationPath) {
              return {
                _id: path.basename(migrationPath, '.js'),
              };
            });

            collection.insert(migrations, callback);
          });
        });
      } else {
        callback();
      }
    });
  },

  preMigrateNewDatabaseWeb: function(callback) {
    var db = mongoose.connection.db;
    var collectionName = 'data_migrations';
    db.collection(collectionName, { strict: true }, function(error) {
      // If the collection doesn't yet exist, assume this is a new
      // installation, so we can mark all of our migrations as already done.
      if(error) {
        db.createCollection(collectionName, function(error, collection) {
          glob(path.join(config.get('web.dir'), 'db/migrate/*.rb'), function(error, migrationPaths) {
            if(error) { return callback(error); }

            var migrations = _.map(migrationPaths, function(migrationPath) {
              return {
                version: path.basename(migrationPath, '.rb').split('_')[0],
              };
            });

            collection.insert(migrations, callback);
          });
        });
      } else {
        callback();
      }
    });
  },

  migrateDatabaseRouter: function(callback) {
    execFile('east', ['migrate'], {
      cwd: path.resolve(__dirname, '../'),
      env: processEnv.env(),
    }, callback);
{'fraction_non_alphanumeric': 0.1571807143139659, 'fraction_numerical': 0.0038365700272910653, 'mean_word_length': 2.676603169986913, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1916588', 'n_tokens_mistral': 7891, 'n_tokens_neox': 7700, 'n_words': 1426}