<commit_before>#!/usr/bin/env node

var fs = require('fs');
var minimist = require('minimist');
var path = require('path');
var zlib = require('zlib');

var minimistOpts = {
  boolean: ['d', 'h', '1', '9'],
  alias: {
    d: ['decompress', 'uncompress'],
    h: 'help',
    1: 'fast',
    9: 'best'
  }
};

var argv = minimist(process.argv.slice(2), minimistOpts);

var decompress = argv.decompress;
var help = argv.help;
var compressionLevel = (argv.fast) ? zlib.Z_BEST_SPEED :
    (argv.best) ? zlib.Z_BEST_COMPRESSION :
    zlib.Z_DEFAULT_COMPRESSION;

if (help) { // help
  return fs.createReadStream(__dirname + '/usage.txt')
    .pipe(process.stdout)
    .on('close', function () { process.exit(1); });
}

function gzip(inStream, outStream) {
  var gzip = zlib.createGzip({ level: compressionLevel });
  inStream
    .on('error', errorExit)
    .pipe(gzip)
    .pipe(outStream);
}

function gunzip(inStream, outStream) {
  var gunzip = zlib.createGunzip();
  inStream
    .on('error', errorExit)
    .pipe(gunzip)
    .pipe(outStream);
}

function errorExit(err) {
    if (err.stack) {
        console.error(err.stack);
    }
    else {
        console.error(String(err));
    }
    process.exit(1);
}

var inStream = process.stdin;
var outStream = process.stdout;

if (decompress) { // decompression
  return gunzip(inStream, outStream);
}
<commit_msg>Allow input filename to be passed in as an arg<commit_after>#!/usr/bin/env node

var fs = require('fs');
var minimist = require('minimist');
var zlib = require('zlib');

var minimistOpts = {
  boolean: ['d', 'h', '1', '9'],
  alias: {
    d: ['decompress', 'uncompress'],
    h: 'help',
    1: 'fast',
    9: 'best'
  }
};

var argv = minimist(process.argv.slice(2), minimistOpts);

var decompress = argv.decompress;
var help = argv.help;
var compressionLevel = (argv.fast) ? zlib.Z_BEST_SPEED :
    (argv.best) ? zlib.Z_BEST_COMPRESSION :
    zlib.Z_DEFAULT_COMPRESSION;
var inputFilename = argv._[0]

if (help) { // help
  return fs.createReadStream(__dirname + '/usage.txt')
    .pipe(process.stdout)
    .on('close', function () { process.exit(1); });
}

function gzip(inStream, outStream) {
  var gzip = zlib.createGzip({ level: compressionLevel });
  inStream
    .on('error', errorExit)
    .pipe(gzip)
    .pipe(outStream);
}

function gunzip(inStream, outStream) {
  var gunzip = zlib.createGunzip();
  inStream
    .on('error', errorExit)
    .pipe(gunzip)
    .pipe(outStream);
}

function errorExit(err) {
    if (err.stack) {
        console.error(err.stack);
    }
    else {
        console.error(String(err));
    }
    process.exit(1);
}

var inStream = inputFilename ? fs.createReadStream(inputFilename) : process.stdin;
var outStream = process.stdout;

if (decompress) { // decompression
  return gunzip(inStream, outStream);
}

{'fraction_non_alphanumeric': 0.16357804704205275, 'fraction_numerical': 0.005345687811831789, 'mean_word_length': 3.5274193548387096, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6554386', 'n_tokens_mistral': 1015, 'n_tokens_neox': 1014, 'n_words': 215}