<commit_before>'use strict';

var _ = require('lodash'),
    apiUmbrellaConfig = require('api-umbrella-config'),
    async = require('async'),
    clc = require('cli-color'),
    config = require('api-umbrella-config').global(),
    crypto = require('crypto'),
    events = require('events'),
    execFile = require('child_process').execFile,
    forever = require('forever-monitor'),
    fs = require('fs'),
    fsExtra = require('fs-extra'),
    glob = require('glob'),
    handlebars = require('handlebars'),
    mkdirp = require('mkdirp'),
    mongoose = require('mongoose'),
    net = require('net'),
    path = require('path'),
    posix = require('posix'),
    processEnv = require('./process_env'),
    request = require('request'),
    supervisord = require('supervisord'),
    url = require('url'),
    util = require('util'),
    uuid = require('node-uuid'),
    yaml = require('js-yaml');

// Defer loading some internal dependencies until the config file is loaded (so
// we can ensure the global config is available for any of our internal
// libraries).
var mongoConnect,
    Admin,
    ApiUser;

var Router = function() {
  this.initialize.apply(this, arguments);
};

util.inherits(Router, events.EventEmitter);
_.extend(Router.prototype, {
  initialize: function(options) {
    this.startingInProgress = true;

    process.stdout.write('Starting api-umbrella...');
    this.startWaitLog = setInterval(function() {
      process.stdout.write('.');
    }, 2000);

    process.on('SIGHUP', this.reload.bind(this));
    process.on('SIGUSR1', this.reopenLogs.bind(this));
    process.on('SIGINT', this.stop.bind(this, function() {
      process.exit(0);
    }));

    apiUmbrellaConfig.loader({
      defaults: {},
      paths: [
        path.resolve(__dirname, '../config/default.yml'),
        path.resolve(__dirname, '../config/default.local.yml'),
      ].concat(options.config),
    }, this.handleConfigReady.bind(this));
  },

  handleConfigReady: function(error, configLoader) {
    this.configLoader = configLoader;

    async.series([
      this.setupGlobalConfig.bind(this),
      this.permissionCheck.bind(this),
      this.internalConfigDefaults.bind(this),
      this.computedConfigValues.bind(this),
      this.cachedRandomConfigValues.bind(this),
      this.reloadGlobalConfig.bind(this),
      this.requireDeferredDependencies.bind(this),
      this.prepare.bind(this),
      this.writeTemplates.bind(this),
      this.startSupervisor.bind(this),
      this.waitForProcesses.bind(this),
      this.waitForConnections.bind(this),
      this.mongoConnect.bind(this),
      this.preMigrateNewDatabaseRouter.bind(this),
      this.preMigrateNewDatabaseWeb.bind(this),
      this.migrateDatabaseRouter.bind(this),
      this.migrateDatabaseWeb.bind(this),
      this.seedDatabase.bind(this),
      this.writeStaticSiteEnv.bind(this),
    ], this.finishStart.bind(this));
  },

  reload: function() {
    var logger = require('./logger');
    var configPath = processEnv.supervisordConfigPath();
    var execOpts = {
      env: processEnv.env(),
    };

    logger.info('Begin reloading api-umbrella...');
    execFile('supervisorctl', ['-c', configPath, 'update'], execOpts, function(error, stdout, stderr) {
      if(error) {
        logger.error('supervisorctl update error: ' + error.message + '\n\nSTDOUT: ' + stdout + '\n\nSTDERR:' + stderr);
        return false;
      }

      var tasks = [];

      if(config.get('service_router_enabled')) {
        tasks = tasks.concat([
          function(callback) {
            logger.info('Reloading router-nginx...');
            execFile('supervisorctl', ['-c', configPath, 'kill', 'HUP', 'router-nginx'], execOpts, callback);
          },
          function(callback) {
            logger.info('Reloading gatekeeper...');
            execFile('supervisorctl', ['-c', configPath, 'serialrestart', 'gatekeeper:*'], execOpts, callback);
          },
          function(callback) {
            logger.info('Reloading config-reloader...');
            execFile('supervisorctl', ['-c', configPath, 'restart', 'config-reloader'], execOpts, callback);
          },
          function(callback) {
            logger.info('Reloading distributed-rate-limits-sync...');
            execFile('supervisorctl', ['-c', configPath, 'restart', 'distributed-rate-limits-sync'], execOpts, callback);
          },
          function(callback) {
            logger.info('Reloading log-processor...');
            execFile('supervisorctl', ['-c', configPath, 'restart', 'log-processor'], execOpts, callback);
          },
          function(callback) {
            logger.info('Reloading router-log-listener...');
            execFile('supervisorctl', ['-c', configPath, 'restart', 'router-log-listener'], execOpts, callback);
          },
        ]);
      }

      if(config.get('service_web_enabled')) {
        tasks = tasks.concat([
          function(callback) {
            logger.info('Reloading web-nginx...');
            execFile('supervisorctl', ['-c', configPath, 'kill', 'HUP', 'web-nginx'], execOpts, callback);
          },
          function(callback) {
            logger.info('Reloading web-puma...');
            execFile('supervisorctl', ['-c', configPath, 'kill', 'USR2', 'web-puma'], execOpts, callback);
          },
        ]);
      }

      if(tasks.length > 0) {
        async.parallel(tasks, function(error) {
          if(error) {
            logger.error('Error reloading api-umbrella: ', error);
            return false;
          } else {
            logger.info('Finished reloading api-umbrella');
          }
        });
      }
    }.bind(this));
  },

  reopenLogs: function() {
    var logger = require('./logger');
    var configPath = processEnv.supervisordConfigPath();
    var execOpts = {
      env: processEnv.env(),
    };

    logger.info('Begin reopening api-umbrella logs...');
    var tasks = [
      function(callback) {
        logger.info('Reopening logs for supervisord...');
        var pidPath = path.join(config.get('run_dir'), 'supervisord.pid');
        fs.readFile(pidPath, function(error, data) {
          if(error) {
            return callback(error);
          }

          if(!data || !data.toString().trim()) {
            return callback('Could not determine pid of supervisord');
          }

          execFile('kill', ['-USR2', data.toString().trim()], execOpts, callback);
        });
      },
    ];

    if(config.get('service_router_enabled')) {
      tasks = tasks.concat([
        function(callback) {
          logger.info('Reopening logs for router-nginx...');
          execFile('supervisorctl', ['-c', configPath, 'kill', 'USR1', 'router-nginx'], execOpts, callback);
        },
        function(callback) {
          logger.info('Reopening logs for varnishncsa...');
          execFile('supervisorctl', ['-c', configPath, 'kill', 'HUP', 'varnishncsa'], execOpts, callback);
        },
      ]);
    }

    if(config.get('service_web_enabled')) {
      tasks = tasks.concat([
        function(callback) {
          logger.info('Reopening logs for web-nginx...');
          execFile('supervisorctl', ['-c', configPath, 'kill', 'USR1', 'web-nginx'], execOpts, callback);
        },
      ]);
    }

    if(config.get('service_general_db_enabled')) {
      tasks = tasks.concat([
        function(callback) {
          logger.info('Reopening logs for mongod...');
          execFile('supervisorctl', ['-c', configPath, 'kill', 'USR1', 'mongod'], execOpts, callback);
        },
      ]);
    }

    if(tasks.length > 0) {
      async.parallel(tasks, function(error) {
        if(error) {
          logger.error('Error reopening log files: ', error);
          return false;
        }
        logger.info('Finished reopening api-umbrella logs');
      });
    }
  },

  stop: function(callback) {
    this.stopCallback = callback;

    process.stdout.write('\nStopping api-umbrella...');

    if(this.configLoader) {
      this.configLoader.close();
    }

    if(mongoose && mongoose.connection) {
      mongoose.connection.close();
    }

    if(this.supervisordProcess && this.supervisordProcess.running) {
      this.supervisordProcess.stop();
    } else {
      this.finishStop();
    }
  },

  prepare: function(callback) {
    mkdirp.sync(path.join(config.get('db_dir'), 'elasticsearch'));
    mkdirp.sync(path.join(config.get('db_dir'), 'mongodb'));
    mkdirp.sync(path.join(config.get('db_dir'), 'redis'));
    mkdirp.sync(path.join(config.get('etc_dir'), 'elasticsearch'));
    mkdirp.sync(path.join(config.get('etc_dir'), 'nginx'));
    mkdirp.sync(path.join(config.get('log_dir'), 'supervisor'));
    mkdirp.sync(path.join(config.get('run_dir'), 'varnish/api-umbrella'));
    mkdirp.sync(config.get('tmp_dir'), { mode: parseInt('0777', 8) });

    if(config.get('user')) {
      var uid = posix.getpwnam(config.get('user')).uid;
      var gid = posix.getgrnam(config.get('group')).gid;

      fs.chownSync(path.join(config.get('db_dir'), 'elasticsearch'), uid, gid);
      fs.chownSync(path.join(config.get('db_dir'), 'mongodb'), uid, gid);
      fs.chownSync(path.join(config.get('db_dir'), 'redis'), uid, gid);
      fs.chownSync(path.join(config.get('log_dir'), 'supervisor'), uid, gid);
      fs.chownSync(path.join(config.get('run_dir'), 'varnish'), uid, gid);
      fs.chownSync(path.join(config.get('run_dir'), 'varnish/api-umbrella'), uid, gid);
      fs.chownSync(config.get('tmp_dir'), uid, gid);
    }

    var primaryHostsNoSsl = _.filter(config.get('hosts'), function(host) {
      return !host.secondary && !host.ssl_cert;
    });

    var sslDir = path.join(config.get('etc_dir'), 'ssl');
    var sslKeyPath = path.join(sslDir, 'self_signed.key');
    var sslCrtPath = path.join(sslDir, 'self_signed.crt');

    var generateSelfSignedCert = false;
    if(primaryHostsNoSsl.length > 0) {
      if(!fs.existsSync(sslKeyPath) || !fs.existsSync(sslCrtPath)) {
        generateSelfSignedCert = true;
      }
    }

    if(generateSelfSignedCert) {
      mkdirp.sync(sslDir);
      execFile('openssl', [
        'req',
        '-new',
        '-newkey', 'rsa:2048',
        '-days', 365 * 5,
        '-nodes',
        '-x509',
        '-subj', '/C=/ST=/L=/O=API Umbrella/CN=apiumbrella.example.com',
        '-keyout', sslKeyPath,
        '-out', sslCrtPath,
      ], {
        env: processEnv.env(),
      }, callback);
    } else {
      callback();
    }
  },

  internalConfigDefaults: function(callback) {
    var filePath = path.join(config.get('run_dir'), 'internal_config_defaults.yml');
    var defaults = yaml.safeDump({
      internal_apis: [
        {
          _id: 'api-umbrella-web-backend',
          name: 'API Umbrella - Default',
          frontend_host: '*',
          backend_host: 'localhost',
          backend_protocol: 'http',
          balance_algorithm: 'least_conn',
          sort_order: 1,
          servers: [
            {
              _id: uuid.v4(),
              host: 'localhost',
              port: config.get('web.port'),
            }
          ],
          url_matches: [
            {
              _id: uuid.v4(),
              frontend_prefix: '/api-umbrella/',
              backend_prefix: '/api-umbrella/',
            }
          ],
          sub_settings: [
            {
              _id: uuid.v4(),
              http_method: 'post',
              regex: '^/api-umbrella/v1/users',
              settings: {
                _id: uuid.v4(),
                required_roles: ['api-umbrella-key-creator'],
              },
            },
            {
              _id: uuid.v4(),
              http_method: 'post',
              regex: '^/api-umbrella/v1/contact',
              settings: {
                _id: uuid.v4(),
                required_roles: ['api-umbrella-contact-form'],
              },
            },
          ],
        },
      ],
    });

    fs.writeFile(filePath, defaults, function(error) {
      if(error) { return callback(error); }

      this.configLoader.options.paths.push(filePath);
      this.configLoader.reload(callback);
    }.bind(this));
  },

  computedConfigValues: function(callback) {
    var filePath = path.join(config.get('run_dir'), 'computed_config_values.yml');
    var defaults = {
      service_general_db_enabled: _.contains(config.get('services'), 'general_db'),
      service_log_db_enabled: _.contains(config.get('services'), 'log_db'),
      service_router_enabled: _.contains(config.get('services'), 'router'),
      service_web_enabled: _.contains(config.get('services'), 'web'),
      router: {
        trusted_proxies: ['127.0.0.1'].concat(config.get('router.trusted_proxies') || []),
      },
    };

    if(config.get('static_site.dir') && !config.get('static_site.build_dir')) {
      _.merge(defaults, {
        static_site: {
          build_dir: path.join(config.get('static_site.dir'), 'build'),
        },
      });
    }

    fs.writeFile(filePath, yaml.safeDump(defaults), function(error) {
      if(error) { return callback(error); }

      this.configLoader.options.paths.push(filePath);
      this.configLoader.reload(callback);
    }.bind(this));
  },

  cachedRandomConfigValues: function(callback) {
    var filePath = path.join(config.get('run_dir'), 'cached_random_config_values.yml');
    var data = '';
    if(fs.existsSync(filePath)) {
      data = fs.readFileSync(filePath).toString();
    }

    var cached = yaml.safeLoad(data) || {};

    if(!config.get('web.rails_secret_token')) {
      cached = _.merge({
        web: {
          rails_secret_token: crypto.randomBytes(64).toString('hex'),
        },
      }, cached);
    }

    if(!config.get('web.devise_secret_key')) {
      cached = _.merge({
        web: {
          devise_secret_key: crypto.randomBytes(64).toString('hex'),
        },
      }, cached);
    }

    fs.writeFile(filePath, yaml.safeDump(cached), function(error) {
      if(error) { return callback(error); }

      this.configLoader.options.paths.push(filePath);
      this.configLoader.reload(callback);
    }.bind(this));
  },

  setupGlobalConfig: function(callback) {
    apiUmbrellaConfig.setGlobal(this.configLoader.runtimeFile);
    config = require('api-umbrella-config').global();

    // Don't poll for mongodb config changes unless we need to (the router and
    // child gatekeeper processes are the only things that need to be listening
    // for those runtime changes).
    if(!_.contains(config.get('services'), 'router')) {
      this.configLoader.close();
    }

    // A dance to move the runtime config file into a less temporary location.
    //
    // This could maybe be addressed better in the api-umbrella-config module,
    // but we want to install the runtime config file into a location based on
    // paths defined in the config file, so it's a little tricky.
    //
    // Copying the file also prevents a build-up of lots of temporary yaml
    // files, which is how api-umbrella-config will behave by default (since we
    // don't necessarily want api-umbrella-config to cleanup the files when it
    // closes, since other processes might be reading the files).
    var tempConfigPath = this.configLoader.runtimeFile;
    try {
      var permanentConfigPath = path.join(config.get('run_dir'), 'runtime_config.yml');
      fsExtra.copySync(tempConfigPath, permanentConfigPath);
      this.configLoader.runtimeFile = permanentConfigPath;
      apiUmbrellaConfig.setGlobal(this.configLoader.runtimeFile);
      config.reload();
    } finally {
      // Always cleanup the temp file, even if the app dies because
      // permanentConfigPath is not writable.
      fs.unlinkSync(tempConfigPath);
    }

    processEnv.overrideEnv({
      'API_UMBRELLA_CONFIG': this.configLoader.runtimeFile,
    });

    callback();
  },

  reloadGlobalConfig: function(callback) {
    config.reload();
    callback();
  },

  requireDeferredDependencies: function(callback) {
    Admin = require('./models/admin');
    mongoConnect = require('./mongo_connect');

    var models = require('api-umbrella-gatekeeper').models(mongoose);
    ApiUser = models.ApiUser;

    callback();
  },

  permissionCheck: function(callback) {
    if(config.get('user')) {
      if(posix.geteuid() !== 0) {
        return callback('Must be started with super-user privileges to change user to \'' + config.get('user') + '\'');
      }

      try {
        posix.getpwnam(config.get('user'));
      } catch(e) {
        return callback('User \'' + config.get('user') + '\' does not exist');
      }

      if(config.get('rlimits.nofile')) {
        try {
          posix.setrlimit('nofile', { soft: config.get('rlimits.nofile'), hard: config.get('rlimits.nofile') });
        } catch(e) {
          return callback('Could not set \'nofile\' resource limit');
        }
      }
    }

    if(config.get('group')) {
      if(posix.geteuid() !== 0) {
        return callback('Must be started with super-user privileges to change group to \'' + config.get('group') + '\'');
      }

      try {
        posix.getgrnam(config.get('group'));
      } catch(e) {
        return callback('Group \'' + config.get('group') + '\' does not exist');
      }

      // Change the process group id so the api-umbrella-config file that gets
      // written is readable by the api-umbrella group.
      process.setgid(config.get('group'));
    }

    if(config.get('http_port') < 1024 || config.get('https_port') < 1024) {
      if(posix.geteuid() !== 0) {
        return callback('Must be started with super-user privileges to use http ports below 1024');
      }
    }

    if(posix.geteuid() === 0) {
      if(!config.get('user') || !config.get('group')) {
        return callback('Must define a user and group to run worker processes as when starting with with super-user privileges');
      }
    }

    callback();
  },

  preMigrateNewDatabaseRouter: function(callback) {
    if(!config.get('service_router_enabled')) { return callback(); }

    var db = mongoose.connection.db;
    var collectionName = '_migrations';
    db.collection(collectionName, { strict: true }, function(error) {
      // If the collection doesn't yet exist, assume this is a new
      // installation, so we can mark all of our migrations as already done.
      if(error) {
        db.createCollection(collectionName, function(error, collection) {
          glob(path.resolve(__dirname, '../migrations/*.js'), function(error, migrationPaths) {
            if(error) { return callback(error); }

            var migrations = _.map(migrationPaths, function(migrationPath) {
              return {
                _id: path.basename(migrationPath, '.js'),
              };
            });

            collection.insert(migrations, callback);
          });
        });
      } else {
        callback();
      }
    });
  },

  preMigrateNewDatabaseWeb: function(callback) {
    if(!config.get('service_web_enabled')) { return callback(); }

    var db = mongoose.connection.db;
    var collectionName = 'data_migrations';
    db.collection(collectionName, { strict: true }, function(error) {
      // If the collection doesn't yet exist, assume this is a new
      // installation, so we can mark all of our migrations as already done.
      if(error) {
        db.createCollection(collectionName, function(error, collection) {
          glob(path.join(config.get('web.dir'), 'db/migrate/*.rb'), function(error, migrationPaths) {
            if(error) { return callback(error); }

            var migrations = _.map(migrationPaths, function(migrationPath) {
              return {
                version: path.basename(migrationPath, '.rb').split('_')[0],
              };
            });

            collection.insert(migrations, callback);
          });
        });
      } else {
        callback();
      }
    });
  },

  migrateDatabaseRouter: function(callback) {
    if(!config.get('service_router_enabled')) { return callback(); }

    execFile('east', ['migrate'], {
      cwd: path.resolve(__dirname, '../'),
      env: processEnv.env(),
    }, function(error, stdout, stderr) {
      if(error) {
        error.message = 'router migration error: ' + error.message + '\n\nSTDOUT: ' + stdout + '\n\nSTDERR:' + stderr;
      }

      callback(error);
    });
  },

  migrateDatabaseWeb: function(callback) {
    if(!config.get('service_web_enabled')) { return callback(); }

    execFile('bundle', [
      'exec',
      'rake',
      'db:mongoid:create_indexes',
      'db:migrate',
      'db:seed_fu',
    ], {
      cwd: config.get('web.dir'),
      env: processEnv.env(),
    }, function(error, stdout, stderr) {
      if(error) {
        error.message = 'web migration error: ' + error.message + '\n\nSTDOUT: ' + stdout + '\n\nSTDERR:' + stderr;
      }

      callback(error);
    });
  },

  seedDatabase: function(callback) {
    if(!config.get('service_router_enabled')) { return callback(); }

    async.series([
      function(seriesCallback) {
        var email = '<EMAIL>';
        ApiUser.findOne({ email: email }, function(error, user) {
          if(error) { return seriesCallback(error); }

          if(!user) {
            user = new ApiUser({ _id: uuid.v4(), email: email });
          }

          user.set({
            first_name: 'API Umbrella Static Site',
            last_name: 'Key',
            website: 'http://' + config.get('default_host') + '/',
            use_description: 'An API key for the API Umbrella static website to use for ajax requests.',
            terms_and_conditions: '1',
            registration_source: 'seed',
            roles: ['api-umbrella-key-creator', 'api-umbrella-contact-form'],
            settings: {
              _id: uuid.v4(),
              rate_limit_mode: 'custom',
              rate_limits: [
                {
                  _id: uuid.v4(),
                  duration: 1 * 60 * 1000, // 1 minute
                  accuracy: 5 * 1000, // 5 seconds
                  limit_by: 'ip',
                  limit: 5,
                  response_headers: false,
                },
                {
                  _id: uuid.v4(),
                  duration: 60 * 60 * 1000, // 1 hour
                  accuracy: 1 * 60 * 1000, // 1 minute
                  limit_by: 'ip',
                  limit: 20,
                  response_headers: true,
                },
              ],
            },
          });

          this.staticSiteApiUser = user;
          user.save(seriesCallback);
        }.bind(this));
      }.bind(this),
      function(seriesCallback) {
        async.each(config.get('web.admin.initial_superusers') || [], function(username, eachCallback) {
          Admin.findOne({ username: username }, function(error, admin) {
            if(error) { return eachCallback(error); }

            if(!admin) {
              admin = new Admin({ _id: uuid.v4(), username: username });
            }

            admin.set({
              superuser: true,
            });

            admin.save(eachCallback);
          });
        }, seriesCallback);
      }.bind(this),
    ], callback);
  },

  writeTemplates: function(callback) {
    var gatekeeperHosts = _.times(config.get('gatekeeper.workers'), function(n) {
      var port = parseInt(config.get('gatekeeper.starting_port'), 10) + n;
      return {
        port: port,
        host: config.get('gatekeeper.host') + ':' + port,
        process_name: 'gatekeeper' + (n + 1),
      };
    });

    var templateConfig = _.extend({}, config.getAll(), {
      api_umbrella_config_runtime_file: this.configLoader.runtimeFile,
      api_umbrella_config_args: '--config ' + this.configLoader.runtimeFile,
      gatekeeper_hosts: gatekeeperHosts,
      gatekeeper_supervisor_process_names: _.pluck(gatekeeperHosts, 'process_name'),
      test_env: (config.get('app_env') === 'test'),
      development_env: (config.get('app_env') === 'development'),
      primary_hosts: _.filter(config.get('hosts'), function(host) { return !host.secondary; }),
      secondary_hosts: _.filter(config.get('hosts'), function(host) { return host.secondary; }),
      has_default_host: (_.where(config.get('hosts'), { default: true }).length > 0),
      supervisor_conditional_user: (config.get('user')) ? 'user=' + config.get('user') : '',
      mongodb_yaml: yaml.safeDump(_.merge({
        systemLog: {
          path: path.join(config.get('log_dir'), 'mongod.log'),
        },
        storage: {
          dbPath: path.join(config.get('db_dir'), 'mongodb'),
        },
      }, config.get('mongodb.embedded_server_config'))),
      elasticsearch_yaml: yaml.safeDump(_.merge({
        path: {
          conf: path.join(config.get('etc_dir'), 'elasticsearch'),
          data: path.join(config.get('db_dir'), 'elasticsearch'),
          logs: path.join(config.get('log_dir'), 'elasticsearch'),
        },
      }, config.get('elasticsearch.embedded_server_config'))),
    });

    var templateRoot = path.resolve(__dirname, '../templates/etc');
    glob(path.join(templateRoot, '**/*'), function(error, templatePaths) {
      async.each(templatePaths, function(templatePath, eachCallback) {
        if(fs.statSync(templatePath).isDirectory()) { return eachCallback(); }

        var installPath = templatePath.replace(/\.hbs$/, '');
        installPath = installPath.replace(templateRoot, '');
        installPath = path.join(config.get('etc_dir'), installPath);

        var content = '';

        // For the api_backends template, write an empty file, since we don't
        // have the necessary API backend information yet. This template gets
        // managed by the config_reloader worker process after things are
        // started.
        if(!_.contains(installPath, 'nginx/api_backends.conf')) {
          content = fs.readFileSync(templatePath).toString();
          if(/\.hbs$/.test(templatePath)) {
            var template = handlebars.compile(content);
            content = template(templateConfig);
          }
        }

        mkdirp.sync(path.dirname(installPath));
        fs.writeFile(installPath, content, eachCallback);

        // Since the api_backends file gets written by the separate
        // config-reloader process, make sure it's writable if that process is
        // running as the less-privileged user.
        if(_.contains(installPath, 'nginx/api_backends.conf')) {
          if(config.get('user')) {
            var uid = posix.getpwnam(config.get('user')).uid;
            var gid = posix.getgrnam(config.get('group')).gid;

            fs.chownSync(installPath, uid, gid);
          }
        }
      }.bind(this), callback);
    }.bind(this));<commit_msg>Fix chown happening before file is written on first start.<commit_after>'use strict';

var _ = require('lodash'),
    apiUmbrellaConfig = require('api-umbrella-config'),
    async = require('async'),
    clc = require('cli-color'),
    config = require('api-umbrella-config').global(),
    crypto = require('crypto'),
    events = require('events'),
    execFile = require('child_process').execFile,
    forever = require('forever-monitor'),
    fs = require('fs'),
    fsExtra = require('fs-extra'),
    glob = require('glob'),
    handlebars = require('handlebars'),
    mkdirp = require('mkdirp'),
    mongoose = require('mongoose'),
    net = require('net'),
    path = require('path'),
    posix = require('posix'),
    processEnv = require('./process_env'),
    request = require('request'),
    supervisord = require('supervisord'),
    url = require('url'),
    util = require('util'),
    uuid = require('node-uuid'),
    yaml = require('js-yaml');

// Defer loading some internal dependencies until the config file is loaded (so
// we can ensure the global config is available for any of our internal
// libraries).
var mongoConnect,
    Admin,
    ApiUser;

var Router = function() {
  this.initialize.apply(this, arguments);
};

util.inherits(Router, events.EventEmitter);
_.extend(Router.prototype, {
  initialize: function(options) {
    this.startingInProgress = true;

    process.stdout.write('Starting api-umbrella...');
    this.startWaitLog = setInterval(function() {
      process.stdout.write('.');
    }, 2000);

    process.on('SIGHUP', this.reload.bind(this));
    process.on('SIGUSR1', this.reopenLogs.bind(this));
    process.on('SIGINT', this.stop.bind(this, function() {
      process.exit(0);
    }));

    apiUmbrellaConfig.loader({
      defaults: {},
      paths: [
        path.resolve(__dirname, '../config/default.yml'),
        path.resolve(__dirname, '../config/default.local.yml'),
      ].concat(options.config),
    }, this.handleConfigReady.bind(this));
  },

  handleConfigReady: function(error, configLoader) {
    this.configLoader = configLoader;

    async.series([
      this.setupGlobalConfig.bind(this),
      this.permissionCheck.bind(this),
      this.internalConfigDefaults.bind(this),
      this.computedConfigValues.bind(this),
      this.cachedRandomConfigValues.bind(this),
      this.reloadGlobalConfig.bind(this),
      this.requireDeferredDependencies.bind(this),
      this.prepare.bind(this),
      this.writeTemplates.bind(this),
      this.startSupervisor.bind(this),
      this.waitForProcesses.bind(this),
      this.waitForConnections.bind(this),
      this.mongoConnect.bind(this),
      this.preMigrateNewDatabaseRouter.bind(this),
      this.preMigrateNewDatabaseWeb.bind(this),
      this.migrateDatabaseRouter.bind(this),
      this.migrateDatabaseWeb.bind(this),
      this.seedDatabase.bind(this),
      this.writeStaticSiteEnv.bind(this),
    ], this.finishStart.bind(this));
  },

  reload: function() {
    var logger = require('./logger');
    var configPath = processEnv.supervisordConfigPath();
    var execOpts = {
      env: processEnv.env(),
    };

    logger.info('Begin reloading api-umbrella...');
    execFile('supervisorctl', ['-c', configPath, 'update'], execOpts, function(error, stdout, stderr) {
      if(error) {
        logger.error('supervisorctl update error: ' + error.message + '\n\nSTDOUT: ' + stdout + '\n\nSTDERR:' + stderr);
        return false;
      }

      var tasks = [];

      if(config.get('service_router_enabled')) {
        tasks = tasks.concat([
          function(callback) {
            logger.info('Reloading router-nginx...');
            execFile('supervisorctl', ['-c', configPath, 'kill', 'HUP', 'router-nginx'], execOpts, callback);
          },
          function(callback) {
            logger.info('Reloading gatekeeper...');
            execFile('supervisorctl', ['-c', configPath, 'serialrestart', 'gatekeeper:*'], execOpts, callback);
          },
          function(callback) {
            logger.info('Reloading config-reloader...');
            execFile('supervisorctl', ['-c', configPath, 'restart', 'config-reloader'], execOpts, callback);
          },
          function(callback) {
            logger.info('Reloading distributed-rate-limits-sync...');
            execFile('supervisorctl', ['-c', configPath, 'restart', 'distributed-rate-limits-sync'], execOpts, callback);
          },
          function(callback) {
            logger.info('Reloading log-processor...');
            execFile('supervisorctl', ['-c', configPath, 'restart', 'log-processor'], execOpts, callback);
          },
          function(callback) {
            logger.info('Reloading router-log-listener...');
            execFile('supervisorctl', ['-c', configPath, 'restart', 'router-log-listener'], execOpts, callback);
          },
        ]);
      }

      if(config.get('service_web_enabled')) {
        tasks = tasks.concat([
          function(callback) {
            logger.info('Reloading web-nginx...');
            execFile('supervisorctl', ['-c', configPath, 'kill', 'HUP', 'web-nginx'], execOpts, callback);
          },
          function(callback) {
            logger.info('Reloading web-puma...');
            execFile('supervisorctl', ['-c', configPath, 'kill', 'USR2', 'web-puma'], execOpts, callback);
          },
        ]);
      }

      if(tasks.length > 0) {
        async.parallel(tasks, function(error) {
          if(error) {
            logger.error('Error reloading api-umbrella: ', error);
            return false;
          } else {
            logger.info('Finished reloading api-umbrella');
          }
        });
      }
    }.bind(this));
  },

  reopenLogs: function() {
    var logger = require('./logger');
    var configPath = processEnv.supervisordConfigPath();
    var execOpts = {
      env: processEnv.env(),
    };

    logger.info('Begin reopening api-umbrella logs...');
    var tasks = [
      function(callback) {
        logger.info('Reopening logs for supervisord...');
        var pidPath = path.join(config.get('run_dir'), 'supervisord.pid');
        fs.readFile(pidPath, function(error, data) {
          if(error) {
            return callback(error);
          }

          if(!data || !data.toString().trim()) {
            return callback('Could not determine pid of supervisord');
          }

          execFile('kill', ['-USR2', data.toString().trim()], execOpts, callback);
        });
      },
    ];

    if(config.get('service_router_enabled')) {
      tasks = tasks.concat([
        function(callback) {
          logger.info('Reopening logs for router-nginx...');
          execFile('supervisorctl', ['-c', configPath, 'kill', 'USR1', 'router-nginx'], execOpts, callback);
        },
        function(callback) {
          logger.info('Reopening logs for varnishncsa...');
          execFile('supervisorctl', ['-c', configPath, 'kill', 'HUP', 'varnishncsa'], execOpts, callback);
        },
      ]);
    }

    if(config.get('service_web_enabled')) {
      tasks = tasks.concat([
        function(callback) {
          logger.info('Reopening logs for web-nginx...');
          execFile('supervisorctl', ['-c', configPath, 'kill', 'USR1', 'web-nginx'], execOpts, callback);
        },
      ]);
    }

    if(config.get('service_general_db_enabled')) {
      tasks = tasks.concat([
        function(callback) {
          logger.info('Reopening logs for mongod...');
          execFile('supervisorctl', ['-c', configPath, 'kill', 'USR1', 'mongod'], execOpts, callback);
        },
      ]);
    }

    if(tasks.length > 0) {
      async.parallel(tasks, function(error) {
        if(error) {
          logger.error('Error reopening log files: ', error);
          return false;
        }
        logger.info('Finished reopening api-umbrella logs');
      });
    }
  },

  stop: function(callback) {
    this.stopCallback = callback;

    process.stdout.write('\nStopping api-umbrella...');

    if(this.configLoader) {
      this.configLoader.close();
    }

    if(mongoose && mongoose.connection) {
      mongoose.connection.close();
    }

    if(this.supervisordProcess && this.supervisordProcess.running) {
      this.supervisordProcess.stop();
    } else {
      this.finishStop();
    }
  },

  prepare: function(callback) {
    mkdirp.sync(path.join(config.get('db_dir'), 'elasticsearch'));
    mkdirp.sync(path.join(config.get('db_dir'), 'mongodb'));
    mkdirp.sync(path.join(config.get('db_dir'), 'redis'));
    mkdirp.sync(path.join(config.get('etc_dir'), 'elasticsearch'));
    mkdirp.sync(path.join(config.get('etc_dir'), 'nginx'));
    mkdirp.sync(path.join(config.get('log_dir'), 'supervisor'));
    mkdirp.sync(path.join(config.get('run_dir'), 'varnish/api-umbrella'));
    mkdirp.sync(config.get('tmp_dir'), { mode: parseInt('0777', 8) });

    if(config.get('user')) {
      var uid = posix.getpwnam(config.get('user')).uid;
      var gid = posix.getgrnam(config.get('group')).gid;

      fs.chownSync(path.join(config.get('db_dir'), 'elasticsearch'), uid, gid);
      fs.chownSync(path.join(config.get('db_dir'), 'mongodb'), uid, gid);
      fs.chownSync(path.join(config.get('db_dir'), 'redis'), uid, gid);
      fs.chownSync(path.join(config.get('log_dir'), 'supervisor'), uid, gid);
      fs.chownSync(path.join(config.get('run_dir'), 'varnish'), uid, gid);
      fs.chownSync(path.join(config.get('run_dir'), 'varnish/api-umbrella'), uid, gid);
      fs.chownSync(config.get('tmp_dir'), uid, gid);
    }

    var primaryHostsNoSsl = _.filter(config.get('hosts'), function(host) {
      return !host.secondary && !host.ssl_cert;
    });

    var sslDir = path.join(config.get('etc_dir'), 'ssl');
    var sslKeyPath = path.join(sslDir, 'self_signed.key');
    var sslCrtPath = path.join(sslDir, 'self_signed.crt');

    var generateSelfSignedCert = false;
    if(primaryHostsNoSsl.length > 0) {
      if(!fs.existsSync(sslKeyPath) || !fs.existsSync(sslCrtPath)) {
        generateSelfSignedCert = true;
      }
    }

    if(generateSelfSignedCert) {
      mkdirp.sync(sslDir);
      execFile('openssl', [
        'req',
        '-new',
        '-newkey', 'rsa:2048',
        '-days', 365 * 5,
        '-nodes',
        '-x509',
        '-subj', '/C=/ST=/L=/O=API Umbrella/CN=apiumbrella.example.com',
        '-keyout', sslKeyPath,
        '-out', sslCrtPath,
      ], {
        env: processEnv.env(),
      }, callback);
    } else {
      callback();
    }
  },

  internalConfigDefaults: function(callback) {
    var filePath = path.join(config.get('run_dir'), 'internal_config_defaults.yml');
    var defaults = yaml.safeDump({
      internal_apis: [
        {
          _id: 'api-umbrella-web-backend',
          name: 'API Umbrella - Default',
          frontend_host: '*',
          backend_host: 'localhost',
          backend_protocol: 'http',
          balance_algorithm: 'least_conn',
          sort_order: 1,
          servers: [
            {
              _id: uuid.v4(),
              host: 'localhost',
              port: config.get('web.port'),
            }
          ],
          url_matches: [
            {
              _id: uuid.v4(),
              frontend_prefix: '/api-umbrella/',
              backend_prefix: '/api-umbrella/',
            }
          ],
          sub_settings: [
            {
              _id: uuid.v4(),
              http_method: 'post',
              regex: '^/api-umbrella/v1/users',
              settings: {
                _id: uuid.v4(),
                required_roles: ['api-umbrella-key-creator'],
              },
            },
            {
              _id: uuid.v4(),
              http_method: 'post',
              regex: '^/api-umbrella/v1/contact',
              settings: {
                _id: uuid.v4(),
                required_roles: ['api-umbrella-contact-form'],
              },
            },
          ],
        },
      ],
    });

    fs.writeFile(filePath, defaults, function(error) {
      if(error) { return callback(error); }

      this.configLoader.options.paths.push(filePath);
      this.configLoader.reload(callback);
    }.bind(this));
  },

  computedConfigValues: function(callback) {
    var filePath = path.join(config.get('run_dir'), 'computed_config_values.yml');
    var defaults = {
      service_general_db_enabled: _.contains(config.get('services'), 'general_db'),
      service_log_db_enabled: _.contains(config.get('services'), 'log_db'),
      service_router_enabled: _.contains(config.get('services'), 'router'),
      service_web_enabled: _.contains(config.get('services'), 'web'),
      router: {
        trusted_proxies: ['127.0.0.1'].concat(config.get('router.trusted_proxies') || []),
      },
    };

    if(config.get('static_site.dir') && !config.get('static_site.build_dir')) {
      _.merge(defaults, {
        static_site: {
          build_dir: path.join(config.get('static_site.dir'), 'build'),
        },
      });
    }

    fs.writeFile(filePath, yaml.safeDump(defaults), function(error) {
      if(error) { return callback(error); }

      this.configLoader.options.paths.push(filePath);
      this.configLoader.reload(callback);
    }.bind(this));
  },

  cachedRandomConfigValues: function(callback) {
    var filePath = path.join(config.get('run_dir'), 'cached_random_config_values.yml');
    var data = '';
    if(fs.existsSync(filePath)) {
      data = fs.readFileSync(filePath).toString();
    }

    var cached = yaml.safeLoad(data) || {};

    if(!config.get('web.rails_secret_token')) {
      cached = _.merge({
        web: {
          rails_secret_token: crypto.randomBytes(64).toString('hex'),
        },
      }, cached);
    }

    if(!config.get('web.devise_secret_key')) {
      cached = _.merge({
        web: {
          devise_secret_key: crypto.randomBytes(64).toString('hex'),
        },
      }, cached);
    }

    fs.writeFile(filePath, yaml.safeDump(cached), function(error) {
      if(error) { return callback(error); }

      this.configLoader.options.paths.push(filePath);
      this.configLoader.reload(callback);
    }.bind(this));
  },

  setupGlobalConfig: function(callback) {
    apiUmbrellaConfig.setGlobal(this.configLoader.runtimeFile);
    config = require('api-umbrella-config').global();

    // Don't poll for mongodb config changes unless we need to (the router and
    // child gatekeeper processes are the only things that need to be listening
    // for those runtime changes).
    if(!_.contains(config.get('services'), 'router')) {
      this.configLoader.close();
    }

    // A dance to move the runtime config file into a less temporary location.
    //
    // This could maybe be addressed better in the api-umbrella-config module,
    // but we want to install the runtime config file into a location based on
    // paths defined in the config file, so it's a little tricky.
    //
    // Copying the file also prevents a build-up of lots of temporary yaml
    // files, which is how api-umbrella-config will behave by default (since we
    // don't necessarily want api-umbrella-config to cleanup the files when it
    // closes, since other processes might be reading the files).
    var tempConfigPath = this.configLoader.runtimeFile;
    try {
      var permanentConfigPath = path.join(config.get('run_dir'), 'runtime_config.yml');
      fsExtra.copySync(tempConfigPath, permanentConfigPath);
      this.configLoader.runtimeFile = permanentConfigPath;
      apiUmbrellaConfig.setGlobal(this.configLoader.runtimeFile);
      config.reload();
    } finally {
      // Always cleanup the temp file, even if the app dies because
      // permanentConfigPath is not writable.
      fs.unlinkSync(tempConfigPath);
    }

    processEnv.overrideEnv({
      'API_UMBRELLA_CONFIG': this.configLoader.runtimeFile,
    });

    callback();
  },

  reloadGlobalConfig: function(callback) {
    config.reload();
    callback();
  },

  requireDeferredDependencies: function(callback) {
    Admin = require('./models/admin');
    mongoConnect = require('./mongo_connect');

    var models = require('api-umbrella-gatekeeper').models(mongoose);
    ApiUser = models.ApiUser;

    callback();
  },

  permissionCheck: function(callback) {
    if(config.get('user')) {
      if(posix.geteuid() !== 0) {
        return callback('Must be started with super-user privileges to change user to \'' + config.get('user') + '\'');
      }

      try {
        posix.getpwnam(config.get('user'));
      } catch(e) {
        return callback('User \'' + config.get('user') + '\' does not exist');
      }

      if(config.get('rlimits.nofile')) {
        try {
          posix.setrlimit('nofile', { soft: config.get('rlimits.nofile'), hard: config.get('rlimits.nofile') });
        } catch(e) {
          return callback('Could not set \'nofile\' resource limit');
        }
      }
    }

    if(config.get('group')) {
      if(posix.geteuid() !== 0) {
        return callback('Must be started with super-user privileges to change group to \'' + config.get('group') + '\'');
      }

      try {
        posix.getgrnam(config.get('group'));
      } catch(e) {
        return callback('Group \'' + config.get('group') + '\' does not exist');
      }

      // Change the process group id so the api-umbrella-config file that gets
      // written is readable by the api-umbrella group.
      process.setgid(config.get('group'));
    }

    if(config.get('http_port') < 1024 || config.get('https_port') < 1024) {
      if(posix.geteuid() !== 0) {
        return callback('Must be started with super-user privileges to use http ports below 1024');
      }
    }

    if(posix.geteuid() === 0) {
      if(!config.get('user') || !config.get('group')) {
        return callback('Must define a user and group to run worker processes as when starting with with super-user privileges');
      }
    }

    callback();
  },

  preMigrateNewDatabaseRouter: function(callback) {
    if(!config.get('service_router_enabled')) { return callback(); }

    var db = mongoose.connection.db;
    var collectionName = '_migrations';
    db.collection(collectionName, { strict: true }, function(error) {
      // If the collection doesn't yet exist, assume this is a new
      // installation, so we can mark all of our migrations as already done.
      if(error) {
        db.createCollection(collectionName, function(error, collection) {
          glob(path.resolve(__dirname, '../migrations/*.js'), function(error, migrationPaths) {
            if(error) { return callback(error); }

            var migrations = _.map(migrationPaths, function(migrationPath) {
              return {
                _id: path.basename(migrationPath, '.js'),
              };
            });

            collection.insert(migrations, callback);
          });
        });
      } else {
        callback();
      }
    });
  },

  preMigrateNewDatabaseWeb: function(callback) {
    if(!config.get('service_web_enabled')) { return callback(); }

    var db = mongoose.connection.db;
    var collectionName = 'data_migrations';
    db.collection(collectionName, { strict: true }, function(error) {
      // If the collection doesn't yet exist, assume this is a new
      // installation, so we can mark all of our migrations as already done.
      if(error) {
        db.createCollection(collectionName, function(error, collection) {
          glob(path.join(config.get('web.dir'), 'db/migrate/*.rb'), function(error, migrationPaths) {
            if(error) { return callback(error); }

            var migrations = _.map(migrationPaths, function(migrationPath) {
              return {
                version: path.basename(migrationPath, '.rb').split('_')[0],
              };
            });

            collection.insert(migrations, callback);
          });
        });
      } else {
        callback();
      }
    });
  },

  migrateDatabaseRouter: function(callback) {
    if(!config.get('service_router_enabled')) { return callback(); }

    execFile('east', ['migrate'], {
      cwd: path.resolve(__dirname, '../'),
      env: processEnv.env(),
    }, function(error, stdout, stderr) {
      if(error) {
        error.message = 'router migration error: ' + error.message + '\n\nSTDOUT: ' + stdout + '\n\nSTDERR:' + stderr;
      }

      callback(error);
    });
  },

  migrateDatabaseWeb: function(callback) {
    if(!config.get('service_web_enabled')) { return callback(); }

    execFile('bundle', [
      'exec',
      'rake',
      'db:mongoid:create_indexes',
      'db:migrate',
      'db:seed_fu',
    ], {
      cwd: config.get('web.dir'),
      env: processEnv.env(),
    }, function(error, stdout, stderr) {
      if(error) {
        error.message = 'web migration error: ' + error.message + '\n\nSTDOUT: ' + stdout + '\n\nSTDERR:' + stderr;
      }

      callback(error);
    });
  },

  seedDatabase: function(callback) {
    if(!config.get('service_router_enabled')) { return callback(); }

    async.series([
      function(seriesCallback) {
        var email = '<EMAIL>';
        ApiUser.findOne({ email: email }, function(error, user) {
          if(error) { return seriesCallback(error); }

          if(!user) {
            user = new ApiUser({ _id: uuid.v4(), email: email });
          }

          user.set({
            first_name: 'API Umbrella Static Site',
            last_name: 'Key',
            website: 'http://' + config.get('default_host') + '/',
            use_description: 'An API key for the API Umbrella static website to use for ajax requests.',
            terms_and_conditions: '1',
            registration_source: 'seed',
            roles: ['api-umbrella-key-creator', 'api-umbrella-contact-form'],
            settings: {
              _id: uuid.v4(),
              rate_limit_mode: 'custom',
              rate_limits: [
                {
                  _id: uuid.v4(),
                  duration: 1 * 60 * 1000, // 1 minute
                  accuracy: 5 * 1000, // 5 seconds
                  limit_by: 'ip',
                  limit: 5,
                  response_headers: false,
                },
                {
                  _id: uuid.v4(),
                  duration: 60 * 60 * 1000, // 1 hour
                  accuracy: 1 * 60 * 1000, // 1 minute
                  limit_by: 'ip',
                  limit: 20,
                  response_headers: true,
                },
              ],
            },
          });

          this.staticSiteApiUser = user;
          user.save(seriesCallback);
        }.bind(this));
      }.bind(this),
      function(seriesCallback) {
        async.each(config.get('web.admin.initial_superusers') || [], function(username, eachCallback) {
          Admin.findOne({ username: username }, function(error, admin) {
            if(error) { return eachCallback(error); }

            if(!admin) {
              admin = new Admin({ _id: uuid.v4(), username: username });
            }

            admin.set({
              superuser: true,
            });

            admin.save(eachCallback);
          });
        }, seriesCallback);
      }.bind(this),
    ], callback);
  },

  writeTemplates: function(callback) {
    var gatekeeperHosts = _.times(config.get('gatekeeper.workers'), function(n) {
      var port = parseInt(config.get('gatekeeper.starting_port'), 10) + n;
      return {
        port: port,
        host: config.get('gatekeeper.host') + ':' + port,
        process_name: 'gatekeeper' + (n + 1),
      };
    });

    var templateConfig = _.extend({}, config.getAll(), {
      api_umbrella_config_runtime_file: this.configLoader.runtimeFile,
      api_umbrella_config_args: '--config ' + this.configLoader.runtimeFile,
      gatekeeper_hosts: gatekeeperHosts,
      gatekeeper_supervisor_process_names: _.pluck(gatekeeperHosts, 'process_name'),
      test_env: (config.get('app_env') === 'test'),
      development_env: (config.get('app_env') === 'development'),
      primary_hosts: _.filter(config.get('hosts'), function(host) { return !host.secondary; }),
      secondary_hosts: _.filter(config.get('hosts'), function(host) { return host.secondary; }),
      has_default_host: (_.where(config.get('hosts'), { default: true }).length > 0),
      supervisor_conditional_user: (config.get('user')) ? 'user=' + config.get('user') : '',
      mongodb_yaml: yaml.safeDump(_.merge({
        systemLog: {
          path: path.join(config.get('log_dir'), 'mongod.log'),
        },
        storage: {
          dbPath: path.join(config.get('db_dir'), 'mongodb'),
        },
      }, config.get('mongodb.embedded_server_config'))),
      elasticsearch_yaml: yaml.safeDump(_.merge({
        path: {
          conf: path.join(config.get('etc_dir'), 'elasticsearch'),
          data: path.join(config.get('db_dir'), 'elasticsearch'),
          logs: path.join(config.get('log_dir'), 'elasticsearch'),
        },
      }, config.get('elasticsearch.embedded_server_config'))),
    });

    var templateRoot = path.resolve(__dirname, '../templates/etc');
    glob(path.join(templateRoot, '**/*'), function(error, templatePaths) {
      async.each(templatePaths, function(templatePath, eachCallback) {
        if(fs.statSync(templatePath).isDirectory()) { return eachCallback(); }

        var installPath = templatePath.replace(/\.hbs$/, '');
        installPath = installPath.replace(templateRoot, '');
        installPath = path.join(config.get('etc_dir'), installPath);

        var content = '';

        // For the api_backends template, write an empty file, since we don't
        // have the necessary API backend information yet. This template gets
        // managed by the config_reloader worker process after things are
        // started.
        if(!_.contains(installPath, 'nginx/api_backends.conf')) {
          content = fs.readFileSync(templatePath).toString();
          if(/\.hbs$/.test(templatePath)) {
            var template = handlebars.compile(content);
            content = template(templateConfig);
          }
        }

        mkdirp.sync(path.dirname(installPath));
        fs.writeFileSync(installPath, content);

        // Since the api_backends file gets written by the separate
        // config-reloader process, make sure it's writable if that process is
        // running as the less-privileged user.
        if(_.contains(installPath, 'nginx/api_backends.conf')) {
          if(config.get('user')) {
            var uid = posix.getpwnam(config.get('user')).uid;
            var gid = posix.getgrnam(config.get('group')).gid;

            fs.chownSync(installPath, uid, gid);
          }
        }

        eachCallback();
      }.bind(this), callback);
    }.bind(this));
{'fraction_non_alphanumeric': 0.154333045340145, 'fraction_numerical': 0.004132076180459036, 'mean_word_length': 2.4385817618186514, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 2, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3043873', 'n_tokens_mistral': 16195, 'n_tokens_neox': 15741, 'n_words': 2978}