<commit_before>var util = require('util');
var debug = require('debug')('ld:init');
var async = require('async');
var mkdirp = require('mkdirp');
var command = require('./command.js');
var fs = require('fs');
var _ = require('lodash');

module.exports = function(argv, systems, cb) {
  debug(system);

  var usage = 'Usage: init <system-name>\n e.g. init phase1';
  var sysName = argv._[1];
  if (!sysName) return cb(usage);
  if (!systems[sysName]) return cb('System not found: ' + sysName);

  var system = systems[sysName];
  var workspace = 'workspace-' + sysName;
  console.log('Initialising system:', sysName, system.stringify(), 'workspace: ' + workspace);

  // do all the setup
  async.series([
    createWorkspace,
    initRepos,
    checkoutBranches,
    npmInstalls,
    createDatabases
  ], cb);

  function createWorkspace(cb){
    mkdirp(workspace, cb);
  }

  function initRepos(cb) {
    async.map(system.services, initRepo, cb);
  }

  function initRepo(service, cb) {
    fs.exists(workspace + '/' + service.name, function(exists) {
      if (exists === true) return cb();
      //var cmd = 'cd ' + workspace + ' && git clone ' + service.repo + ' ' + service.name;
      var cmd = 'git clone ' + service.repo + ' ' + service.name;
      debug('initRepo', workspace, cmd);
      command(cmd, workspace, service.env, cb);
    });
  }

  function checkoutBranches(cb) {
    async.map(system.services, checkoutBranch, cb);
  }

  function checkoutBranch(service, cb) {
    //var cmd = 'cd ' + workspace + '/' + service.name + ' && git checkout ' + service.branch;
    var cmd = 'git checkout ' + service.branch;
    var dir = workspace + '/' + service.name;
    debug('checkoutBranch', dir, cmd);
    command(cmd, dir, service.env, cb);
  }

  function npmInstalls(cb) {
    async.map(system.services, npmInstall, cb);
  }

  function npmInstall(service, cb) {
    //var cmd = 'cd ' + workspace + '/' + service.name + ' && npm install .';
    var dir = workspace + '/' + service.name;
    var cmd = 'npm install .';
    debug('npmInstall', dir, cmd);
    command(cmd, dir, service.env, cb);
  }

  function createDatabases(cb) {
    var pg = require('pg');
    var conString = util.format('postgres://%s:%s@%s/postgres',
                                service.env.POSTGRES_USERNAME,
                                service.env.POSTGRES_PASSWORD,
                                service.env.POSTGRES_HOST);

    var client = new pg.Client(conString);
    client.connect(function(err) {
      if(err) return cb('Postgres connection error: ' + err);

      async.map(system.services, createDatabase, function(err) {
        client.end();
        cb(err);
      });

      function createDatabase(service, cb) {
        if (!service.database) return cb();<commit_msg>dont re-init the workspace dir if it already exists<commit_after>var util = require('util');
var debug = require('debug')('ld:init');
var async = require('async');
var mkdirp = require('mkdirp');
var command = require('./command.js');
var fs = require('fs');
var _ = require('lodash');

module.exports = function(argv, systems, cb) {
  debug(system);

  var usage = 'Usage: init <system-name>\n e.g. init phase1';
  var sysName = argv._[1];
  if (!sysName) return cb(usage);
  if (!systems[sysName]) return cb('System not found: ' + sysName);

  var system = systems[sysName];
  var workspace = 'workspace-' + sysName;
  console.log('Initialising system:', sysName, util.inspect(system.stringify(), true, null), 'workspace: ' + workspace);

  // do all the setup
  async.series([
    createWorkspace,
    initRepos,
    createDatabases
  ], cb);

  function createWorkspace(cb){
    mkdirp(workspace, cb);
  }

  function initRepos(cb) {
    async.map(system.services, initRepo, cb);
  }

  function initRepo(service, cb) {
    fs.exists(workspace + '/' + service.name, function(exists) {
      if (exists === true) return cb();
      async.series([
        function(cb) { cloneRepo(service, cb); },
        function(cb) { checkoutBranch(service, cb); },
        function(cb) { npmInstall(service, cb); }
      ], cb);
    });
  }

  function cloneRepo(service, cb) {
    //var cmd = 'cd ' + workspace + ' && git clone ' + service.repo + ' ' + service.name;
    var cmd = 'git clone ' + service.repo + ' ' + service.name;
    debug('cloneRepo', workspace, cmd);
    command(cmd, workspace, service.env, cb);
  }

  function checkoutBranch(service, cb) {
    //var cmd = 'cd ' + workspace + '/' + service.name + ' && git checkout ' + service.branch;
    var cmd = 'git checkout ' + service.branch;
    var dir = workspace + '/' + service.name;
    debug('checkoutBranch', dir, cmd);
    command(cmd, dir, service.env, cb);
  }

  function npmInstall(service, cb) {
    //var cmd = 'cd ' + workspace + '/' + service.name + ' && npm install .';
    var dir = workspace + '/' + service.name;
    var cmd = 'npm install .';
    debug('npmInstall', dir, cmd);
    command(cmd, dir, service.env, cb);
  }

  function createDatabases(cb) {
    var pg = require('pg');
    var conString = util.format('postgres://%s:%s@%s/postgres',
                                system.env.POSTGRES_USERNAME,
                                system.env.POSTGRES_PASSWORD,
                                system.env.POSTGRES_HOST);

    var client = new pg.Client(conString);
    client.connect(function(err) {
      if(err) return cb('Postgres connection error: ' + err);

      async.map(system.services, createDatabase, function(err) {
        client.end();
        cb(err);
      });

      function createDatabase(service, cb) {
        if (!service.database) return cb();
{'fraction_non_alphanumeric': 0.15840880838217014, 'fraction_numerical': 0.0007103534008169064, 'mean_word_length': 2.868131868131868, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5742269', 'n_tokens_mistral': 1792, 'n_tokens_neox': 1745, 'n_words': 427}