<commit_before>var _ = require('lodash'),
  async = require('async'),
  child = require('child_process'),
  colors = require('colors'),
  commander = require('commander'),
  fs = require('fs'),
  iz = require('iz'),
  p = require('path'),
  util = require('util'),
  wrench = require('wrench');

/**
 * Parse command line arguments and execute actions based on the specified commands.
 * Uses commander.js to provide -V / --version to display version number,
 * and -h / --help to display help info.
 *
 * @param {String} base: base directory where the client module is located,
 *   used as a base directory to read command file and package.json file,
 *   ideally the value would be the client's __dirname
 * @param {Object} actions: action function for each command in format: { command: { action: function () {} }},
 *   the command name in actions object is then mapped to the command name specified in commandFile
 * @param {Object} opts: optional
 *   - commandFile: relative path to command file from base directory, defaults to 'conf/commands.json'
 */
function command(base, actions, opts) {

  actions = actions || {};
  opts = opts || {};

  var commands = JSON.parse(fs.readFileSync(p.join(base, opts.commandFile || '../conf/commands.json'))),
    pkg = JSON.parse(fs.readFileSync(p.join(base, '../package.json')));

  if (actions.commands && commands.commands) {
    _.each(actions.commands, function (command, name) {
      if (commands.commands[name]) {
        commands.commands[name].action = command.action; 
      }
    });
  }

  commander.version(pkg.version);

  if (commands.options) {
    _.each(commands.options, function (option) {
      commander.option(option.arg, option.desc, option.action);
    });
  }

  _.each(commands.commands, function (command, name) {
    var program = commander
      .command(name)
      .description(command.desc);

    _.each(command.options, function (option) {
      program.option(option.arg, option.desc, option.action);
    });

    program.action(command.action);
  });

  _preCommand(commands.commands);

  commander.parse(process.argv);

  // NOTE: commander.args is populated by commander#parse,
  // hence _postCommand relies on commander#parse finishing without exiting or throwing error,
  // otherwise _postCommand won't be executed
  _postCommand(commander.args, commands.commands);
}

// Pre-command tasks:
// - if --help flag is specified, append examples after standard help output
function _preCommand(commands) {
  commander.on('--help', function () {

    var hasExample = _.pluck(_.values(commands), 'examples')
      .filter(function (elem) {
        return elem !== undefined;
      });
    if (hasExample.length > 0) {
      hasExample = hasExample.reduce(function (a, b) {
        return a.concat(b);
      });
    }
    hasExample = hasExample.length > 0;

    if (hasExample) {
      
      console.log('  Examples:\n');

      Object.keys(commands).forEach(function (command) {
        if (!_.isEmpty(commands[command].examples)) {
          console.log('    %s:', command);
          commands[command].examples.forEach(function (example) {
            console.log('      %s', example);
          });
        }
      });
    }
  });
}


// Post-command tasks:
// - if there's no command, display help then exit
// - if command is unknown, display error message then exit
// - if there's command, validate arguments then exit
function _postCommand(args, commands) {

  function _validate(arg, commandArg) {
    return function (rule) {
      if (iz.hasOwnProperty(rule)) {
        var isValid = iz[rule](arg);
        if (isValid === null || isValid === false) {
          exit(new Error(util.format('Invalid argument: <%s> must be %s', commandArg.name, rule)));
        }
      } else {
        exit(new Error(util.format('Invalid argument rule: %s', rule)));
      }
    };
  }

  if (!args) {
    return;

  } else if (args.length === 0) {
    // 2 -> 'node' and 'somecommand'
    // which means that the command is executed without args, hence display help menu
    // NOTE: this check is needed because for some reason commander.args
    // also returns empty array when one of the args is an opt (-- prefixed)
    if (process.argv.length === 2) {
      commander.help();
    }

  } else {
    var lastArg = args[args.length - 1],
      commandName = lastArg._name,
      commandArgs = (commands[commandName]) ? commands[commandName].args : undefined;

    // non-object last arg means command is not configured in setup
    if (typeof lastArg !== 'object') {
      exit(new Error(util.format('Unknown command: %s, use --help for more info', lastArg)));

    } else if (commandArgs && commandArgs.length > 0) {
      var programName = lastArg.parent._name;

      // display usage info when expected arguments don't exist
      if (args.length === 1) {
        exit(new Error(util.format('Usage: %s %s %s', programName, commandName, commandArgs.map(function (arg) {
          return util.format((arg.optional) ? '[%s]' : '<%s>', arg.name);
        }).join(' '))));
      } else {
        // validate arguments as configured in commands setup
        for (var i = 0, ln = commandArgs.length; i < ln; i += 1) {
          if (!commandArgs[i].optional) {
            commandArgs[i].rules.forEach(_validate(args[i], commandArgs[i]));
          }
        }
      }
    }
  }
}

/**
 * Execute a one-liner command.
 * Both stderr and stdout will be logged via console.error/log accordingly.
 * Fallthrough is handy in situation where there are multiple execs running in sequence/parallel,
 * and they all have to be executed regardless of success/error on either one of them.
 *
 * @param {String} command: command to execute
 * @param {Boolean} fallthrough: allow error to be camouflaged as a non-error
 * @param {Function} cb: standard cb(err, result) callback
 */
function exec(command, fallthrough, cb) {

  var _exec = child.exec(command, function (err) {
    var result;
    if (err && fallthrough) {
      // camouflage error to allow other execs to keep running
      result = err;
      err = null;
    }
    cb(err, result);
  });

  _exec.stdout.on('data', function (data) {<commit_msg>Modify post-command validation to support non-argument, rename some vars for clarity.<commit_after>var _ = require('lodash'),
  async = require('async'),
  child = require('child_process'),
  colors = require('colors'),
  commander = require('commander'),
  fs = require('fs'),
  iz = require('iz'),
  p = require('path'),
  util = require('util'),
  wrench = require('wrench');

/**
 * Parse command line arguments and execute actions based on the specified commands.
 * Uses commander.js to provide -V / --version to display version number,
 * and -h / --help to display help info.
 *
 * @param {String} base: base directory where the client module is located,
 *   used as a base directory to read command file and package.json file,
 *   ideally the value would be the client's __dirname
 * @param {Object} actions: action function for each command in format: { command: { action: function () {} }},
 *   the command name in actions object is then mapped to the command name specified in commandFile
 * @param {Object} opts: optional
 *   - commandFile: relative path to command file from base directory, defaults to 'conf/commands.json'
 */
function command(base, actions, opts) {

  actions = actions || {};
  opts = opts || {};

  var commands = JSON.parse(fs.readFileSync(p.join(base, opts.commandFile || '../conf/commands.json'))),
    pkg = JSON.parse(fs.readFileSync(p.join(base, '../package.json')));

  if (actions.commands && commands.commands) {
    _.each(actions.commands, function (command, name) {
      if (commands.commands[name]) {
        commands.commands[name].action = command.action; 
      }
    });
  }

  commander.version(pkg.version);

  if (commands.options) {
    _.each(commands.options, function (option) {
      commander.option(option.arg, option.desc, option.action);
    });
  }

  _.each(commands.commands, function (command, name) {
    var program = commander
      .command(name)
      .description(command.desc);

    _.each(command.options, function (option) {
      program.option(option.arg, option.desc, option.action);
    });

    program.action(command.action);
  });

  _preCommand(commands.commands);

  commander.parse(process.argv);

  // NOTE: commander.args is populated by commander#parse,
  // hence _postCommand relies on commander#parse finishing without exiting or throwing error,
  // otherwise _postCommand won't be executed
  _postCommand(commander.args, commands.commands);
}

// Pre-command tasks:
// - if --help flag is specified, append examples after standard help output
function _preCommand(commands) {
  commander.on('--help', function () {

    var hasExample = _.pluck(_.values(commands), 'examples')
      .filter(function (elem) {
        return elem !== undefined;
      });
    if (hasExample.length > 0) {
      hasExample = hasExample.reduce(function (a, b) {
        return a.concat(b);
      });
    }
    hasExample = hasExample.length > 0;

    if (hasExample) {
      
      console.log('  Examples:\n');

      Object.keys(commands).forEach(function (command) {
        if (!_.isEmpty(commands[command].examples)) {
          console.log('    %s:', command);
          commands[command].examples.forEach(function (example) {
            console.log('      %s', example);
          });
        }
      });
    }
  });
}


// Post-command tasks:
// - if there's no command, display help then exit
// - if command is unknown, display error message then exit
// - if there's command, validate arguments then exit
function _postCommand(args, commandsConfig) {

  function _validate(value, name, desc) {
    return function (rule) {
      if (iz.hasOwnProperty(rule)) {
        var isValid = iz[rule](value);
        if (isValid === null || isValid === false) {
          exit(new Error(util.format('Invalid %s: <%s> must be %s', desc, name, rule)));
        }
      } else {
        exit(new Error(util.format('Invalid %s rule: %s', desc, rule)));
      }
    };
  }

  if (!args) {
    return;

  } else if (args.length === 0) {
    // 2 -> 'node' and 'somecommand'
    // which means that the command is executed without args, hence display help menu
    // NOTE: this check is needed because for some reason commander.args
    // also returns empty array when one of the args is an opt (-- prefixed)
    if (process.argv.length === 2) {
      commander.help();
    }

  } else {
    var lastArg = args[args.length - 1],
      commandName = lastArg._name,
      commandArgs = (commandsConfig[commandName]) ? commandsConfig[commandName].args : undefined;

    // non-object last arg means command is not configured in setup
    if (typeof lastArg !== 'object') {
      exit(new Error(util.format('Unknown command: %s, use --help for more info', lastArg)));

    } else if (commandArgs && commandArgs.length > 0) {
      var programName = lastArg.parent._name;

      // display usage info when expected arguments don't exist
      if (args.length === 1) {
        exit(new Error(util.format('Usage: %s %s %s', programName, commandName, commandArgs.map(function (arg) {
          return util.format((arg.optional) ? '[%s]' : '<%s>', arg.name);
        }).join(' '))));
      } else {
        // validate arguments as configured in commands setup
        for (var i = 0, ln = commandArgs.length; i < ln; i += 1) {
          if (!commandArgs[i].optional) {
            commandArgs[i].rules.forEach(_validate(args[i], commandArgs[i].name, 'argument'));
          }
        }
      }
    }
  }
}

/**
 * Execute a one-liner command.
 * Both stderr and stdout will be logged via console.error/log accordingly.
 * Fallthrough is handy in situation where there are multiple execs running in sequence/parallel,
 * and they all have to be executed regardless of success/error on either one of them.
 *
 * @param {String} command: command to execute
 * @param {Boolean} fallthrough: allow error to be camouflaged as a non-error
 * @param {Function} cb: standard cb(err, result) callback
 */
function exec(command, fallthrough, cb) {

  var _exec = child.exec(command, function (err) {
    var result;
    if (err && fallthrough) {
      // camouflage error to allow other execs to keep running
      result = err;
      err = null;
    }
    cb(err, result);
  });

  _exec.stdout.on('data', function (data) {
{'fraction_non_alphanumeric': 0.13462772785622593, 'fraction_numerical': 0.0016046213093709885, 'mean_word_length': 2.9396333754740835, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3167067', 'n_tokens_mistral': 3614, 'n_tokens_neox': 3522, 'n_words': 1130}