<commit_before>var path = require('path')
  , fs = require('fs')
  , util = require('util')
  , async = require('async')
  , circular = require('circular')
  , esprima = require('esprima')
  , escodegen = require('escodegen')
  , EOL = require('os').EOL;

var formats = {
  prg: 'var prg = %s;',
  pkg: 'var pkg = %s;',
  cfg: 'var cfg = %s;',
  opt: 'var opt = %s;',
  cmd: 'var cmd = %s;',
  key: '\'%s\': new %s(%s),',
}

/**
 *  Get the transformer document ast.
 */
function transformer(cb) {
  var mod = path.join(__dirname, 'transformer.js');
  fs.readFile(mod, function(err, contents) {
    cb(err, contents ? esprima.parse('' + contents) : contents);
  })
}

/**
 *  Apply transformations to the core ast and
 *  add them to an array of ast blocks.
 */
function apply(req, ast, cb) {
  var opts = req.argv
    , program = req.program
    , opts = program.options()
    , cmds = program.commands()
    , funcbody = ast.body[0].body.body;

  function extract(ast) {
    return ast.body[0];
  }

  function preamble(cb) {
    var js = 'var define = require("cli-define")' + EOL;
    js += '  , Command = define.Command' + EOL;
    js += '  , Option = define.Option' + EOL;
    js += '  , Flag = define.Flag;';
    cb(null, [esprima.parse(js)]);
  }


  // declare the prg variable
  function prg(cb) {
    var prg = program.toObject({all: true, recurse: false});
    delete prg.names;
    var js = util.format(formats.prg, circular.stringify(prg));
    funcbody.unshift(extract(
      esprima.parse('program.sections(prg.sections);')));
    funcbody.unshift(extract(
      esprima.parse('program.detail(prg.detail);')));
    funcbody.unshift(extract(
      esprima.parse('program.description(prg.description);')));
    funcbody.unshift(extract(
      esprima.parse('program.version(prg.version);')));
    funcbody.unshift(extract(
      esprima.parse('program.name(prg.name);')));
    cb(null, [esprima.parse(js)]);
  }


  // declare the pkg variable
  function pkg(cb) {
    var pkg = program.package();
    if(!pkg) return cb(null, []);
    var js = util.format(formats.pkg, circular.stringify(pkg));
    funcbody.unshift(extract(esprima.parse('program.package(pkg);')));
    cb(null, [esprima.parse(js)]);
  }

  // declare the cfg variable
  function cfg(cb) {
    var c = esprima.parse('' + req.sources.configure.data);
    // get module exports object expression
    var oe = c.body[0].expression.right;
    var js = util.format(formats.cfg, escodegen.generate(oe));
    funcbody.unshift(extract(esprima.parse('program.configure(cfg);')));
    cb(null, [esprima.parse(js)]);
  }

  function getArgMapString(map) {
    var k, v, s = '{';
    for(k in map) {
      //console.dir(k);
      v = map[k];
      //console.dir(v);
      v.toObject({all: true, item: function(o) {
        //console.log('item callback');
        s += util.format(
          formats.key,
          k, o.constructor.name, circular.stringify(o))
        return o;
      }});
    }
    s += '}';
    return s;
  }

  // declare the opt variable
  function opt(cb) {
    var s = getArgMapString(opts);
    var js = util.format(formats.opt, s);
    funcbody.unshift(extract(esprima.parse('program.options(opt);')));
    cb(null, [esprima.parse(js)]);
  }

  // declare the cmd variable
  function cmd(cb) {
    var s = getArgMapString(cmds);
    var js = util.format(formats.cmd, s);
    funcbody.unshift(extract(esprima.parse('program.commands(cmd);')));
    cb(null, [esprima.parse(js)]);
  }

  // add the transformer ast, should be at the end
  function std(cb) {
    cb(null, [ast]);
  }

  var transformations = [];
  transformations.push(preamble);

  transformations.push(prg);

  if(Object.keys(opts).length) {
    transformations.push(opt);
  }
  if(Object.keys(cmds).length) {
    transformations.push(cmd);
  }
  if(req.sources.configure) {
    transformations.push(cfg);
  }
  transformations.push(pkg);
  transformations.push(std);

  async.concatSeries(transformations, function iterator(func, cb) {
    func(cb);
  }, function(err, blocks) {
    cb(err, blocks);
  })
}

/**
 *  Flatten the transformation blocks back to a single ast.<commit_msg>Work on subcommand support.<commit_after>var path = require('path')
  , fs = require('fs')
  , util = require('util')
  , async = require('async')
  , circular = require('circular')
  , esprima = require('esprima')
  , escodegen = require('escodegen')
  , EOL = require('os').EOL;

var formats = {
  prg: 'var prg = %s;',
  pkg: 'var pkg = %s;',
  cfg: 'var cfg = %s;',
  opt: 'var opt = %s;',
  cmd: 'var cmd = %s;',
  key: '\'%s\': new %s(%s),',
}

/**
 *  Get the transformer document ast.
 */
function transformer(cb) {
  var mod = path.join(__dirname, 'transformer.js');
  fs.readFile(mod, function(err, contents) {
    cb(err, contents ? esprima.parse('' + contents) : contents);
  })
}

/**
 *  Apply transformations to the core ast and
 *  add them to an array of ast blocks.
 */
function apply(req, ast, cb) {
  var opts = req.argv
    , program = req.program
    , opts = program.options()
    , cmds = program.commands()
    , funcbody = ast.body[0].body.body;

  function extract(ast) {
    return ast.body[0];
  }

  function preamble(cb) {
    var js = 'var define = require("cli-define")' + EOL;
    js += '  , Command = define.Command' + EOL;
    js += '  , Option = define.Option' + EOL;
    js += '  , Flag = define.Flag;';
    cb(null, [esprima.parse(js)]);
  }


  // declare the prg variable
  function prg(cb) {
    var prg = program.toObject({all: true, recurse: true});
    delete prg.names;
    var js = util.format(formats.prg, circular.stringify(prg));
    funcbody.unshift(extract(
      esprima.parse('program.sections(prg.sections);')));
    funcbody.unshift(extract(
      esprima.parse('program.detail(prg.detail);')));
    funcbody.unshift(extract(
      esprima.parse('program.description(prg.description);')));
    funcbody.unshift(extract(
      esprima.parse('program.version(prg.version);')));
    funcbody.unshift(extract(
      esprima.parse('program.name(prg.name);')));
    cb(null, [esprima.parse(js)]);
  }


  // declare the pkg variable
  function pkg(cb) {
    var pkg = program.package();
    if(!pkg) return cb(null, []);
    var js = util.format(formats.pkg, circular.stringify(pkg));
    funcbody.unshift(extract(esprima.parse('program.package(pkg);')));
    cb(null, [esprima.parse(js)]);
  }

  // declare the cfg variable
  function cfg(cb) {
    var c = esprima.parse('' + req.sources.configure.data);
    // get module exports object expression
    var oe = c.body[0].expression.right;
    var js = util.format(formats.cfg, escodegen.generate(oe));
    funcbody.unshift(extract(esprima.parse('program.configure(cfg);')));
    cb(null, [esprima.parse(js)]);
  }

  function getArgMapString(map) {
    var k, v, o, s = '{', cmd;
    var out = {};
    for(k in map) {
      v = map[k];
      cmd = typeof v.commands === 'function';
      //console.dir(v);
        //console.log('item callback %s (%s)', this.name(), depth);
      function fmt(o, depth){
        //o.source = util.format(
          //formats.key,
          //o.key, o.constructor.name, circular.stringify(o))
        return o;
      }

      o = v.toObject(
        {all: true, recurse: true, item: fmt});

      //if(cmd && Object.keys(v.commands()).length) {
        //console.dir(getArgMapString(v.commands()));
        //s += 'commands: {},';
      //}
        //
        //
      out[k] = o;
      s += util.format(
        formats.key,
        o.key, o.constructor.name, circular.stringify(o))
    }

    s += '}';
    return s;
  }

  // declare the opt variable
  function opt(cb) {
    funcbody.unshift(
      extract(esprima.parse('program.options(prg.options);')));
    cb(null, []);
  }

  // declare the cmd variable
  function cmd(cb) {
    //var s = getArgMapString(cmds);
    //var js = util.format(formats.cmd, s);
    funcbody.unshift(
      extract(esprima.parse('program.commands(prg.commands);')));
    cb(null, []);
  }

  // add the transformer ast, should be at the end
  function std(cb) {
    cb(null, [ast]);
  }

  var transformations = [];
  transformations.push(preamble);

  transformations.push(prg);

  if(Object.keys(opts).length) {
    transformations.push(opt);
  }
  if(Object.keys(cmds).length) {
    transformations.push(cmd);
  }

  if(req.sources.configure) {
    transformations.push(cfg);
  }

  transformations.push(pkg);
  transformations.push(std);

  async.concatSeries(transformations, function iterator(func, cb) {
    func(cb);
  }, function(err, blocks) {
    cb(err, blocks);
  })
}

/**
 *  Flatten the transformation blocks back to a single ast.
{'fraction_non_alphanumeric': 0.17619212562059808, 'fraction_numerical': 0.0006927606511950121, 'mean_word_length': 2.986194201564657, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6670738', 'n_tokens_mistral': 3064, 'n_tokens_neox': 3014, 'n_words': 618}