<commit_before>#!/usr/bin/env node

const debug = require('debug')('ottomaton');
const chalk = require('chalk');
const getStdin = require('get-stdin');
const Promise = require('native-promise-only');
const fs = require('fs');
const path = require('path');
const Ottomaton = require('..');
const otto = Ottomaton();
const argv = require('minimist')(process.argv.slice(2));
if (argv.version) {
  console.log(require('../package.json').version);
  process.exit(0);
}


let currentLib;
const cwd = process.cwd();
const scripts = [];
try {
  [].concat(argv._).forEach(libPath => {
    currentLib = path.resolve(cwd, libPath);

    if (!currentLib.match(/[.]txt$/)) {
      debug('registering library: %s', currentLib);
      otto.register(require(currentLib));
    } else {
      scripts.push(currentLib);
    }
  });
} catch (err) {
  debug('unable to register library "%s" reason: %j', currentLib, err);
  console.error(chalk.red(`ERROR: unable to register library "${ currentLib }"`));
  process.exit(1);
}

async function runScripts(scripts, state) {
  for (let script of scripts) {
    await otto.run(script, state);
  }

  return state;
}

function loadScriptFiles(scriptPaths, state) {
  return Object.values(scriptPaths.map(srcPath => {
    srcPath = path.resolve(cwd, srcPath);
    if (!fs.existsSync(srcPath))
      throw new Error(`file could not be loaded: ${ srcPath }`);

    return fs.readFileSync(srcPath, 'utf-8');
  }));
}

let displayError = err => {
  console.error(chalk.red(`ERROR: ${ err.message }`));
  process.exit(1);
};

let startKeys = Object.keys(argv);

let finish = state => {
  var outputState = {};

  Object.keys(state).forEach(function(key) {
    // Ignore keys that were passed in
    if (startKeys.indexOf(key) === -1 && key !== 'html') {
      outputState[key] = state[key];
    }
  });

  if (argv.output === 'json') {
    console.log(JSON.stringify(outputState))
  } else {
    Object.keys(outputState).forEach(function(key) {
      console.log(key, '=', outputState[key]);
    });
  }

  process.stdin.unref();
};<commit_msg>Adding --testing=1 flag<commit_after>#!/usr/bin/env node

const debug = require('debug')('ottomaton');
const chalk = require('chalk');
const getStdin = require('get-stdin');
const Promise = require('native-promise-only');
const fs = require('fs');
const path = require('path');
const Ottomaton = require('..');

const argv = require('minimist')(process.argv.slice(2));
if (argv.version) {
  console.log(require('../package.json').version);
  process.exit(0);
}

const otto = Ottomaton(argv);

let currentLib;
const cwd = process.cwd();
const scripts = [];
try {
  [].concat(argv._).forEach(libPath => {
    currentLib = path.resolve(cwd, libPath);

    if (!currentLib.match(/[.]txt$/)) {
      debug('registering library: %s', currentLib);
      otto.register(require(currentLib));
    } else {
      scripts.push(currentLib);
    }
  });
} catch (err) {
  debug('unable to register library "%s" reason: %j', currentLib, err);
  console.error(chalk.red(`ERROR: unable to register library "${ currentLib }"`));
  process.exit(1);
}

async function runScripts(scripts, state) {
  for (let script of scripts) {
    if (argv.testing) {
      script = script.split("\n").join("\nWAIT FOR KEYSTROKE\n");       
    }

    await otto.run(script, state);
  }

  return state;
}

function loadScriptFiles(scriptPaths, state) {
  return Object.values(scriptPaths.map(srcPath => {
    srcPath = path.resolve(cwd, srcPath);
    if (!fs.existsSync(srcPath))
      throw new Error(`file could not be loaded: ${ srcPath }`);

    return fs.readFileSync(srcPath, 'utf-8');
  }));
}

let displayError = err => {
  console.error(chalk.red(`ERROR: ${ err.message }`));
  process.exit(1);
};

let startKeys = Object.keys(argv);

let finish = state => {
  var outputState = {};

  Object.keys(state).forEach(function(key) {
    // Ignore keys that were passed in
    if (startKeys.indexOf(key) === -1 && key !== 'html') {
      outputState[key] = state[key];
    }
  });

  if (argv.output === 'json') {
    console.log(JSON.stringify(outputState))
  } else {
    Object.keys(outputState).forEach(function(key) {
      console.log(key, '=', outputState[key]);
    });
  }

  // So that any process.stdin.once('data', ...) don't call the prcess to lock up.
  process.stdin.unref();
};
{'fraction_non_alphanumeric': 0.18385753931544865, 'fraction_numerical': 0.0030064754856614245, 'mean_word_length': 3.78429203539823, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5582807', 'n_tokens_mistral': 1444, 'n_tokens_neox': 1430, 'n_words': 307}