<commit_before>"use strict"

let path = require("path")
let opts = require("commander")
let fs = require("fs")
let _ = require('lodash')

let postabular = require("..")

let pkg = fs.readFileSync(`${__dirname}/../package.json`)

let list = function(val) { return val.split(',') }

opts
    .version(JSON.parse(pkg).version)
    .option('-u, --use <plugins>', 'use plugins', list, [])
    .option('-l, --local <plugins>', 'use local plugins', list, [])
    .option('-c, --config <config>', 'use config')
    .option('-o, --output <filename>', 'output file name')
    .parse(process.argv)

if (!process.argv.slice(2).length) {
    opts.help()
}

fs.createReadStream(opts.args[0])
    .pipe(postabular((opts.use.map(plugin => { return require(plugin) }))
                     .concat(opts.local.map(plugin => { return require(path.join(process.cwd(), plugin)) }))))
    .pipe(fs.createWriteStream(opts.output))<commit_msg>output to stdout if no -o given<commit_after>"use strict"

let path = require("path")
let opts = require("commander")
let fs = require("fs")
let _ = require('lodash')

let postabular = require("..")

let pkg = fs.readFileSync(`${__dirname}/../package.json`)

let list = function(val) { return val.split(',') }

opts
    .version(JSON.parse(pkg).version)
    .option('-u, --use <plugins>', 'use plugins', list, [])
    .option('-l, --local <plugins>', 'use local plugins', list, [])
    .option('-c, --config <config>', 'use config')
    .option('-o, --output <filename>', 'output file name')
    .parse(process.argv)

if (!process.argv.slice(2).length) {
    opts.help()
}

let plugins = []
plugins.concat(opts.use.map(plugin => { return require(plugin) }))
plugins.concat(opts.local.map(plugin => { return require(path.join(process.cwd(), plugin)) }))

let os = fs.createReadStream(opts.args[0]).pipe(postabular(plugins))

if (opts.output) {
    os.pipe(fs.createWriteStream(opts.output))
} else {
    os.pipe(process.stdout)
}
{'fraction_non_alphanumeric': 0.20903954802259886, 'fraction_numerical': 0.002054442732408834, 'mean_word_length': 3.9441624365482233, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '41808', 'n_tokens_mistral': 663, 'n_tokens_neox': 663, 'n_words': 139}