<commit_before>#!/usr/bin/env node
/*
 * mochify.js
 *
 * Copyright (c) 2014 <NAME> <<EMAIL>>
 *
 * @license MIT
 */
'use strict';

var watchify      = require('watchify');
var browserify    = require('browserify');
var coverify      = require('coverify');
var through       = require('through');
var resolve       = require('resolve');
var glob          = require('glob');
var path          = require('path');
var mocaccino     = require('mocaccino');
var phantomic     = require('phantomic');
var webdriver     = require('min-wd/lib/driver');
var webdriverOpts = require('min-wd/lib/options');
var spawn         = require('child_process').spawn;

var argv     = process.argv.slice(2);
var cwd      = process.cwd();
var reporter = 'dot';
var watch    = false;
var wd       = false;
var cover    = false;
var node     = false;
var failure  = false;
var debug    = false;
var port     = 0;
var yields   = 0;
var ps;

while (argv.length && argv[0].indexOf('-') === 0) {
  if (argv[0] === '--watch') {
    argv.shift();
    watch = true;
  } else if (argv[0] === '--cover') {
    argv.shift();
    cover = true;
  } else if (argv[0] === '--node') {
    argv.shift();
    node = true;
  } else if (argv[0] === '--wd') {
    argv.shift();
    wd = true;
  } else if (argv[0] === '--reporter' || argv[0] === '-R') {
    argv.shift();
    reporter = argv.shift();
  } else if (argv[0] === '--yields' || argv[0] === '-y') {
    argv.shift();
    yields = parseInt(argv.shift(), 10);
  } else if (argv[0] === '--debug') {
    argv.shift();
    debug = true;
  } else if (argv[0] === '--port') {
    argv.shift();
    port = parseInt(argv.shift(), 10);
  } else if (argv[0] === '--help' || argv[0] === '-h') {
    argv.shift();
    console.log(require('fs').readFileSync(__dirname + '/help.txt', 'utf8'));
    process.exit(0);
  } else if (argv[0] === '--version' || argv[0] === '-v') {
    argv.shift();
    console.log(require('../package.json').version);
    process.exit(0);
  } else {
    console.log('Unknown argument: ' + argv[0]);
    console.log('Run `mochify --help` for usage.\n');
    process.exit(1);
  }
}

if (cover && reporter === 'dot') {
  reporter = 'spec';
}

if (debug) {
  if (node) {
    console.log('--debug does not work with --node\n');
    process.exit(1);
  }
  if (wd) {
    console.log('--debug does not work with --wd\n');
    process.exit(1);
  }
}

var entries = [];
if (argv.length) {
  argv.forEach(function (arg) {
    if (arg.indexOf('*') === -1) {
      entries.push(arg);
    } else {
      Array.prototype.push.apply(entries, glob.sync(arg));
    }
  });
} else {
  entries = glob.sync("./test/*.js");
}

function error(err) {
  console.error(String(err));
}

var TRACE_RE  = /^\s+at [^:]+:[0-9]+\)?\s*$/;
var IGNORE_RE = /\/node_modules\/(browserify|mocha)\//;
var SOURCE_RE = /\/[^:]+/;

function relativePath(source) {
  return path.relative(cwd, source);
}

function tracebackFormatter() {
  var buf = '';
  return through(function (chunk) {
    buf += chunk;
    var l, p = buf.indexOf('\n');
    while (p !== -1) {
      l = buf.substring(0, p);
      if (TRACE_RE.test(l)) {
        if (!IGNORE_RE.test(l)) {
          l = l.replace(SOURCE_RE, relativePath);
          this.queue(l + '\n');
        }
      } else {
        this.queue(l + '\n');
      }
      buf = buf.substring(p + 1);
      p = buf.indexOf('\n');
    }
  });
}

function launcherCallback(callback) {
  return function (err) {
    if (!watch && !cover) {
      process.nextTick(function () {
        process.exit(err ? 1 : 0);
      });
    } else if (callback) {
      callback();
    }
  };
}
<commit_msg>Stream partial lines of output that don't look like a stack<commit_after>#!/usr/bin/env node
/*
 * mochify.js
 *
 * Copyright (c) 2014 <NAME> <<EMAIL>>
 *
 * @license MIT
 */
'use strict';

var watchify      = require('watchify');
var browserify    = require('browserify');
var coverify      = require('coverify');
var through       = require('through');
var resolve       = require('resolve');
var glob          = require('glob');
var path          = require('path');
var mocaccino     = require('mocaccino');
var phantomic     = require('phantomic');
var webdriver     = require('min-wd/lib/driver');
var webdriverOpts = require('min-wd/lib/options');
var spawn         = require('child_process').spawn;

var argv     = process.argv.slice(2);
var cwd      = process.cwd();
var reporter = 'dot';
var watch    = false;
var wd       = false;
var cover    = false;
var node     = false;
var failure  = false;
var debug    = false;
var port     = 0;
var yields   = 0;
var ps;

while (argv.length && argv[0].indexOf('-') === 0) {
  if (argv[0] === '--watch') {
    argv.shift();
    watch = true;
  } else if (argv[0] === '--cover') {
    argv.shift();
    cover = true;
  } else if (argv[0] === '--node') {
    argv.shift();
    node = true;
  } else if (argv[0] === '--wd') {
    argv.shift();
    wd = true;
  } else if (argv[0] === '--reporter' || argv[0] === '-R') {
    argv.shift();
    reporter = argv.shift();
  } else if (argv[0] === '--yields' || argv[0] === '-y') {
    argv.shift();
    yields = parseInt(argv.shift(), 10);
  } else if (argv[0] === '--debug') {
    argv.shift();
    debug = true;
  } else if (argv[0] === '--port') {
    argv.shift();
    port = parseInt(argv.shift(), 10);
  } else if (argv[0] === '--help' || argv[0] === '-h') {
    argv.shift();
    console.log(require('fs').readFileSync(__dirname + '/help.txt', 'utf8'));
    process.exit(0);
  } else if (argv[0] === '--version' || argv[0] === '-v') {
    argv.shift();
    console.log(require('../package.json').version);
    process.exit(0);
  } else {
    console.log('Unknown argument: ' + argv[0]);
    console.log('Run `mochify --help` for usage.\n');
    process.exit(1);
  }
}

if (cover && reporter === 'dot') {
  reporter = 'spec';
}

if (debug) {
  if (node) {
    console.log('--debug does not work with --node\n');
    process.exit(1);
  }
  if (wd) {
    console.log('--debug does not work with --wd\n');
    process.exit(1);
  }
}

var entries = [];
if (argv.length) {
  argv.forEach(function (arg) {
    if (arg.indexOf('*') === -1) {
      entries.push(arg);
    } else {
      Array.prototype.push.apply(entries, glob.sync(arg));
    }
  });
} else {
  entries = glob.sync("./test/*.js");
}

function error(err) {
  console.error(String(err));
}

var TRACE_RE  = /^\s+at [^:]+:[0-9]+\)?\s*$/;
var IGNORE_RE = /\/node_modules\/(browserify|mocha)\//;
var SOURCE_RE = /\/[^:]+/;

function relativePath(source) {
  return path.relative(cwd, source);
}

function tracebackFormatter() {
  var buf = '';
  return through(function (chunk) {
    buf += chunk;
    var l, p = buf.indexOf('\n');
    while (p !== -1) {
      l = buf.substring(0, p);
      if (TRACE_RE.test(l)) {
        if (!IGNORE_RE.test(l)) {
          l = l.replace(SOURCE_RE, relativePath);
          this.queue(l + '\n');
        }
      } else {
        this.queue(l + '\n');
      }
      buf = buf.substring(p + 1);
      p = buf.indexOf('\n');
    }
    if (!/^\s+/.test(buf) || (buf.length > 3 && !/^\s+at /.test(buf))) {
      this.queue(buf);
      buf = '';
    }
  });
}

function launcherCallback(callback) {
  return function (err) {
    if (!watch && !cover) {
      process.nextTick(function () {
        process.exit(err ? 1 : 0);
      });
    } else if (callback) {
      callback();
    }
  };
}

{'fraction_non_alphanumeric': 0.21745602165087957, 'fraction_numerical': 0.011502029769959404, 'mean_word_length': 2.499526515151515, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 10, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5947949', 'n_tokens_mistral': 2688, 'n_tokens_neox': 2653, 'n_words': 596}