<commit_before>#!/usr/bin/env node

var Avails = require('../lib/avails');
var packageJson = require('../package.json');

var program = require('commander');
var split = require('split');
var JSONStream = require('JSONStream')

program
  .version(packageJson.version)
  .usage('[options] <infile')
  .option('-i, --input [type]', 'Add the specified type of input [tsv|json] (required)')
  .option('-o, --output [type]', 'Add the specified type of output [tsv|json] (required)')
  .parse(process.argv);

if (!program.input || !program.output) {
  program.outputHelp();
  process.exit(1);
}

switch (program.input) {
  case 'tsv':
    inputTSV(process.stdin);
    break;
  case 'json':
    inputJSON(process.stdin);
    break;
  default:
    console.error('Unsupported input type: ' + program.input);
    process.exit(1);
}

//
// INPUT
//

/*
 * TSV can be streamed in
 */
function inputTSV (stream) {
  var avails = [];
  stream
    .pipe(split())
    .on('data', function (line) {
      if (!line) return;
      avails.push(Avails.fromTSVLine(line));
    })
    .on('end', function () {
      output(avails);
    })
}

/*
 * TSV can be streamed in
 */
function inputJSON (stream) {
  var avails;
  // FIXME: I don't think I'm using this correctly
  stream
    .pipe(JSONStream.parse())
    .on('data', function (line) {
      if (!line) return;
      avails = line;
    })
    .on('end', function () {
      output(avails);
    })
}

//
// OUTPUT
//

function output(avails) {
  switch (program.output) {
    case 'tsv':
      outputTSV(avails);
      break;
    case 'json':
      outputJSON(avails);
      break;
    case 'js':
      outputJS(avails);
      break;
    default:
      console.error('Unsupported output type: ' + program.output);
      process.exit(1);
  }
}

function outputJSON(avails) {
  var json;<commit_msg>Add XLSX read/write feature<commit_after>#!/usr/bin/env node

var Avails = require('../lib/avails');
var packageJson = require('../package.json');
var writeAvailsToXLSXStream = require('../lib/writeAvailsToXLSXStream');
var readAvailsFromXLSXStream = require('../lib/readAvailsFromXLSXStream');

var program = require('commander');
var split = require('split');
var JSONStream = require('JSONStream')

program
  .version(packageJson.version)
  .usage('[options] <infile')
  .option('-i, --input [type]', 'Add the specified type of input [tsv|json] (required)')
  .option('-o, --output [type]', 'Add the specified type of output [tsv|json|js|xlsx] (required)')
  .parse(process.argv);

if (!program.input || !program.output) {
  program.outputHelp();
  process.exit(1);
}

switch (program.input) {
  case 'tsv':
    inputTSV(process.stdin);
    break;
  case 'json':
    inputJSON(process.stdin);
    break;
  case 'xlsx':
    readAvailsFromXLSXStream(process.stdin)
      .then(function (avails) {
        output(avails);
      })
      .then(null, function (err) {
        console.error(err)
        process.exit(1);
      })
    break;
  default:
    console.error('Unsupported input type: ' + program.input);
    process.exit(1);
}

//
// INPUT
//

/*
 * TSV can be streamed in
 */
function inputTSV (stream) {
  var avails = [];
  stream
    .pipe(split())
    .on('data', function (line) {
      if (!line) return;
      avails.push(Avails.fromTSVLine(line));
    })
    .on('end', function () {
      output(avails);
    })
}

/*
 * TSV can be streamed in
 */
function inputJSON (stream) {
  var avails;
  // FIXME: I don't think I'm using this correctly
  stream
    .pipe(JSONStream.parse())
    .on('data', function (line) {
      if (!line) return;
      avails = line;
    })
    .on('end', function () {
      output(avails);
    })
}

//
// OUTPUT
//

function output(avails) {
  switch (program.output) {
    case 'tsv':
      outputTSV(avails);
      break;
    case 'json':
      outputJSON(avails);
      break;
    case 'js':
      outputJS(avails);
      break;
    case 'xlsx':
      writeAvailsToXLSXStream(avails, process.stdout);
      break;
    default:
      console.error('Unsupported output type: ' + program.output);
      process.exit(1);
  }
}

function outputJSON(avails) {
  var json;
{'fraction_non_alphanumeric': 0.16730954676952747, 'fraction_numerical': 0.0016875602700096432, 'mean_word_length': 2.877570093457944, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6221073', 'n_tokens_mistral': 1438, 'n_tokens_neox': 1436, 'n_words': 317}