<commit_before>'use strict';

var
  fs,
  tmp,
  path,
  shell,
  logger,
  readline,
  spawnSync,
  initFlagName;

fs = require('fs');
tmp = require('tmp');
path = require('path');
shell = require('shelljs');
readline = require('readline-sync');
spawnSync = require('child_process').spawnSync;

logger = require('./logger.js');

// Clean temp on exception.
tmp.setGracefulCleanup();

exports.checkRequirements = function () {
  var
    required,
    requirement;

  required = {
    'VBoxManage': "VirtualBox doesn't appear to be installed. Please see https://www.virtualbox.org/wiki/Downloads for installation instructions.",
    'brew': "Homebrew doesn't appear to be installed. Please see http://brew.sh/ for installation instructions.",
    'dinghy': "dinghy is not installed. Please see https://github.com/chinthakagodawita/homebrew-dh for more info."
  };

  // Make sure we have each requirement.
  for (requirement in required) {
    if (shell.which(requirement) === null) {
      throw new Error(required[requirement]);
    }
  }
};

exports.getYargsContainerInfo = function (yargs, cmd) {
  var
    name,
    subArgv,
    sources;

  subArgv = yargs.usage('dh soe ' + cmd + ' <container name> [options]')
  .option('s', {
    alias: 'sources',
    description: 'Directory to use for sources, defaults to the current directory'
  })
  .help('help')
  .argv;

  if (!subArgv._[1]) {
    name = null;
  }
  else {
    name = subArgv._[1];
  }

  if (!subArgv.s) {
    sources = shell.pwd();
  }

  return {
    'name': name,
    'sources': sources
  };
};

exports.runSubscript = function (scriptname, yargs) {
  var
    args,
    scriptPath;

  // Pass all other arguments to sub-script.
  args = yargs.argv._.slice(1);

  // Pass the help & debug flags through if it exists too.
  if (yargs.argv.help || yargs.argv.h) {
    args = ['--help'].concat(args);
  }
  if (yargs.argv.debug || yargs.argv.d) {
    args.push('-d');
  }

  scriptPath = path.resolve(__dirname, '..', '..', 'bin', scriptname + '.js');
  spawnSync(scriptPath, args, {
    cwd: process.cwd(),
    stdio: 'inherit'
  })
};

exports.shell = function (cmd, args, opts) {
  if (typeof args === 'undefined') {
    args = [];
  }
  if (typeof opts === 'undefined') {
    opts = {};
  }

  if (typeof args === 'string') {
    args = [args];
  }

  if (args) {
    cmd += ' ' + args.join(' ');
  }

  logger.debug('Executing: ' + cmd);

  // @TODO: check return code and throw Error with message if >0.
  return shell.exec(cmd, opts);
};
<commit_msg>Fixed bug where not all flags were being passed to sub-scripts.<commit_after>'use strict';

var
  fs,
  tmp,
  path,
  shell,
  logger,
  readline,
  spawnSync,
  initFlagName;

fs = require('fs');
tmp = require('tmp');
path = require('path');
shell = require('shelljs');
readline = require('readline-sync');
spawnSync = require('child_process').spawnSync;

logger = require('./logger.js');

// Clean temp on exception.
tmp.setGracefulCleanup();

exports.checkRequirements = function () {
  var
    required,
    requirement;

  required = {
    'VBoxManage': "VirtualBox doesn't appear to be installed. Please see https://www.virtualbox.org/wiki/Downloads for installation instructions.",
    'brew': "Homebrew doesn't appear to be installed. Please see http://brew.sh/ for installation instructions.",
    'dinghy': "dinghy is not installed. Please see https://github.com/chinthakagodawita/homebrew-dh for more info."
  };

  // Make sure we have each requirement.
  for (requirement in required) {
    if (shell.which(requirement) === null) {
      throw new Error(required[requirement]);
    }
  }
};

exports.getYargsContainerInfo = function (yargs, cmd) {
  var
    name,
    subArgv,
    sources;

  subArgv = yargs.usage('dh soe ' + cmd + ' <container name> [options]')
  .option('s', {
    alias: 'sources',
    description: 'Directory to use for sources, defaults to the current directory'
  })
  .help('help')
  .argv;

  if (!subArgv._[1]) {
    name = null;
  }
  else {
    name = subArgv._[1];
  }

  if (!subArgv.s) {
    sources = shell.pwd();
  }

  return {
    'name': name,
    'sources': sources
  };
};

exports.runSubscript = function (scriptname, yargs) {
  var
    args,
    scriptPath;

  // Pass all other arguments to sub-script.
  args = yargs.argv._.slice(1);

  // Pass help in as the first flag if set.
  if (yargs.argv.help || yargs.argv.h) {
    args = ['--help'].concat(args);

    if (yargs.argv.help) {
      delete yargs.argv.help;
    }
    if (yargs.argv.h) {
      delete yargs.argv.h;
    }
  }

  // Pass all other valid flags through to the sub-script too.
  for (var flag in yargs.argv) {
    var flagType = '-';
    if (flag.indexOf('$') > -1 || flag == '_' || flag == 'h' || flag == 'help') {
      continue;
    }

    if (flag.length > 1) {
      flagType = '--';
    }
    args.push(flagType + flag + '=' + yargs.argv[flag]);
  }

  scriptPath = path.resolve(__dirname, '..', '..', 'bin', scriptname + '.js');
  spawnSync(scriptPath, args, {
    cwd: process.cwd(),
    stdio: 'inherit'
  })
};

exports.shell = function (cmd, args, opts) {
  if (typeof args === 'undefined') {
    args = [];
  }
  if (typeof opts === 'undefined') {
    opts = {};
  }

  if (typeof args === 'string') {
    args = [args];
  }

  if (args) {
    cmd += ' ' + args.join(' ');
  }

  logger.debug('Executing: ' + cmd);

  // @TODO: check return code and throw Error with message if >0.
  return shell.exec(cmd, opts);
};

{'fraction_non_alphanumeric': 0.16471877282688094, 'fraction_numerical': 0.0018261504747991235, 'mean_word_length': 2.881644223954642, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 9, 'https://': 4, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1734381', 'n_tokens_mistral': 1885, 'n_tokens_neox': 1854, 'n_words': 490}