<commit_before>/**
 * Output as JSON (pretty) to /output/csv/
 */
module.exports = function(options, opfile, columns, log, timer, moduleCallback) {
  var dt = new Date(),
    dir = dt.getFullYear() + '-' + ('0' + (Number(dt.getMonth()) + 1).toString()).slice(-2) + '-' + ('0' + dt.getDate()).slice(-2);

  var table = options.source + '.' + options.table,
    fs = require('graceful-fs'),
    mkdirp = require('mkdirp'),
    async = require('async'),
    Stream = require('stream'),
    keys = [],
    split = require('split');

  for (var i = 0; i < columns.length; i++) {
    keys.push(columns[i].name);
  }

  log.group('JSON Output').log('Copying file (and transformining) to output/json/' + dir + '/' + table + '.json');

  async.waterfall([
    function(cb) {
      mkdirp('./output/json/' + dir, function(err) {
        if (err) return cb(err);
        cb(null);
      })
    },
    function(cb) {
      var first = true;
      var second = true;
      var tab2JSONStream = new Stream.Transform();
      tab2JSONStream._transform = function(chunk, encoding, done) {
        if (first) {
          first = false;
          return done();
        }
        var str = (second) ? '' : ',\n';
        if (second) second = false;
        var values = chunk.toString().split('\t');
        if (values.length !== keys.length) return done();
        var data = {};
        for (var i = 0; i < values.length; i++) {
          data[keys[i]] = values[i];
        }
        this.push(str + JSON.stringify(data, null, 2));
        done();
      };
      tab2JSONStream.on('error', function(err) {
        cb(err);
      })
      var opfileRStream = opfile.createReadStream();
      var outputJSONStream = fs.createWriteStream('./output/json/' + dir + '/' + table + '.json');
      outputJSONStream.on('error', function(err) {
        cb(err);
      })
      outputJSONStream.on('finish', function() {
        fs.appendFile('./output/json/' + dir + '/' + table + '.json', ']', function(err) {
          if (err) return cb(err);
          cb(null);
        });
      })
      outputJSONStream.write('[');
      opfileRStream.pipe(split()).pipe(tab2JSONStream).pipe(outputJSONStream);

    }
  ], function(err) {
    if (err) return moduleCallback(err);
    moduleCallback(null, opfile);
  })

}<commit_msg>updating to opt object style<commit_after>/**
 * Output as JSON (pretty) to /output/csv/
 */
module.exports = function(opt, columns, moduleCallback) {
  var dt = new Date(),
    dir = dt.getFullYear() + '-' + ('0' + (Number(dt.getMonth()) + 1).toString()).slice(-2) + '-' + ('0' + dt.getDate()).slice(-2);

  var table = opt.source + '.' + opt.table,
    fs = require('graceful-fs'),
    mkdirp = require('mkdirp'),
    async = require('async'),
    Stream = require('stream'),
    keys = [],
    split = require('split'),
    opfile = opt.opfile,
    log = opt.log,
    timer = opt.timer;

  for (var i = 0; i < columns.length; i++) {
    keys.push(columns[i].name);
  }

  log.group('JSON Output').log('Copying file (and transformining) to output/json/' + dir + '/' + table + '.json');

  async.waterfall([
    function(cb) {
      log.log('Making directory for json output.');
      mkdirp(opt.cfg.dirs.output + 'json/' + dir, function(err) {
        if (err) return cb(err);
        cb(null);
      })
    },
    function(cb) {
      log.log('Transforming data from opfile to json.');
      var first = true;
      var second = true;
      var tab2JSONStream = new Stream.Transform();
      tab2JSONStream._transform = function(chunk, encoding, done) {
        //skip first row (columns)
        if (first) {
          first = false;
          return done();
        }
        //only put new line and comma before each row (except first data row)
        var str = (second) ? '' : ',\n';
        if (second) second = false;
        //handle windows \r return
        var values = chunk.toString().replace(/\r/g, '').split('\t');
        if (values.length !== keys.length) return done();
        var data = {};
        for (var i = 0; i < values.length; i++) {
          data[keys[i]] = values[i];
        }
        this.push(str + JSON.stringify(data, null, 2));
        done();
      };
      tab2JSONStream.on('error', function(err) {
        cb(err);
      })
      var opfileRStream = opfile.createReadStream();
      var outputJSONStream = fs.createWriteStream(opt.cfg.dirs.output + 'json/' + dir + '/' + table + '.json');
      outputJSONStream.on('error', function(err) {
        cb(err);
      })
      outputJSONStream.on('finish', function() {
        fs.appendFile(opt.cfg.dirs.output + 'json/' + dir + '/' + table + '.json', ']', function(err) {
          if (err) return cb(err);
          cb(null);
        });
      })
      outputJSONStream.write('[');
      opfileRStream.pipe(split()).pipe(tab2JSONStream).pipe(outputJSONStream);

    }
  ], function(err) {
    if (err) return moduleCallback(err);
    moduleCallback(null);
  })
}
{'fraction_non_alphanumeric': 0.18262626262626264, 'fraction_numerical': 0.0048484848484848485, 'mean_word_length': 2.3497970230040597, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5186307', 'n_tokens_mistral': 1580, 'n_tokens_neox': 1554, 'n_words': 356}