<commit_before>#!/usr/bin/env node
require('shelljs/global');

var TEST_DIR = 'scripts';

var shouldLog = (process.argv[2] === 'log');
var log = [];

function writeLog(msg, link) {
  console.log(msg);
  if (link) {
    msg = msg.replace(']', '](' + link + ')');
  }
  log.push(msg);
}

cd(__dirname + '/' + TEST_DIR);
var prefix;
var shellJSWins = [];
ls().forEach(function (dir) {
  prefix = TEST_DIR + '/' + dir;
  writeLog('### [' + dir + ']', prefix)
  cd(dir)

  // Find the files
  var jsfile = ls('*.js')[0];
  var shfile = ls('*.sh')[0];

  if (!jsfile || !shfile) {
    console.error('Could not find your files. Please use proper extensions.');
    return;
  }

  // execute it
  var start_time,
      end_time,
      shell_time,
      shell_output,
      js_time,
      js_output;

  config.silent = true;

  start_time = new Date().getTime();
  js_output = exec('node ' + jsfile).output;
  end_time = new Date().getTime();
  js_time = end_time - start_time;
  writeLog(' - [ShellJS] took `' + js_time + '` milliseconds', prefix + '/' + jsfile);

  start_time = new Date().getTime();
  shell_output = exec('bash ' + shfile).output;
  end_time = new Date().getTime();
  shell_time = end_time - start_time;
  writeLog(' - [Bash] took `' + shell_time + '` milliseconds', prefix + '/' + shfile);

  if (shell_time < js_time) {
    writeLog('Bash was `' + (js_time/shell_time).toFixed(3) + '` times faster than ShellJS');
  } else {
    writeLog('ShellJS was `' + (shell_time/js_time).toFixed(3) + '` times faster than Bash');
    shellJSWins.push(dir);
  }

  if (shell_output !== js_output)
    writeLog('Output differs');

  // Clean up
  echo('=======================');
  echo();
  config.silent = false;
  cd('..')
});

if (shouldLog) {
  cd(__dirname);
  var text = '### ShellJS performance wins\n\n';
  text += shellJSWins.map(x => ' - [' + x + '](' + TEST_DIR + '/' + x + ')').join('\n');
  text += '\n\n' + log.join('\n\n');

  // Wipe out the old results
  cat('README.md').replace(/## Results(.|\n)*/, '## Results').to('README.md');

  // Append new docs to README
  sed('-i', /## Results/, '## Results\n\n' + text, 'README.md');
}<commit_msg>Print system information after tests<commit_after>#!/usr/bin/env node
require('shelljs/global');

var TEST_DIR = 'scripts';

var shouldLog = (process.argv[2] === 'log');
var log = [];

function writeLog(msg, link) {
  console.log(msg);
  if (link) {
    msg = msg.replace(']', '](' + link + ')');
  }
  log.push(msg);
}

function printSystemInfo() {
  var os = require('os');
  var cpus = os.cpus();
  var spawn = require('child_process').spawn;
  console.log('System Information:');
  console.log(os.type(), os.release(), os.arch());
  console.log('Node.js ' + process.versions.node);
  console.log('V8 ' + process.versions.v8);
  console.log(cpus[0].model + ' × ' + cpus.length);
  console.log();
  console.log('Shell Information:');
  spawn('sh', ['-c', '${SHELL} --version'], { stdio: 'inherit' });
}

cd(__dirname + '/' + TEST_DIR);
var prefix;
var shellJSWins = [];
ls().forEach(function (dir) {
  prefix = TEST_DIR + '/' + dir;
  writeLog('### [' + dir + ']', prefix)
  cd(dir)

  // Find the files
  var jsfile = ls('*.js')[0];
  var shfile = ls('*.sh')[0];

  if (!jsfile || !shfile) {
    console.error('Could not find your files. Please use proper extensions.');
    return;
  }

  // execute it
  var start_time,
      end_time,
      shell_time,
      shell_output,
      js_time,
      js_output;

  config.silent = true;

  start_time = new Date().getTime();
  js_output = exec('node ' + jsfile).output;
  end_time = new Date().getTime();
  js_time = end_time - start_time;
  writeLog(' - [ShellJS] took `' + js_time + '` milliseconds', prefix + '/' + jsfile);

  start_time = new Date().getTime();
  shell_output = exec('bash ' + shfile).output;
  end_time = new Date().getTime();
  shell_time = end_time - start_time;
  writeLog(' - [Bash] took `' + shell_time + '` milliseconds', prefix + '/' + shfile);

  if (shell_time < js_time) {
    writeLog('Bash was `' + (js_time/shell_time).toFixed(3) + '` times faster than ShellJS');
  } else {
    writeLog('ShellJS was `' + (shell_time/js_time).toFixed(3) + '` times faster than Bash');
    shellJSWins.push(dir);
  }

  if (shell_output !== js_output)
    writeLog('Output differs');

  // Clean up
  echo('=======================');
  echo();
  config.silent = false;
  cd('..')
});

if (shouldLog) {
  cd(__dirname);
  var text = '### ShellJS performance wins\n\n';
  text += shellJSWins.map(x => ' - [' + x + '](' + TEST_DIR + '/' + x + ')').join('\n');
  text += '\n\n' + log.join('\n\n');

  // Wipe out the old results
  cat('README.md').replace(/## Results(.|\n)*/, '## Results').to('README.md');

  // Append new docs to README
  sed('-i', /## Results/, '## Results\n\n' + text, 'README.md');
}
printSystemInfo();
{'fraction_non_alphanumeric': 0.21931091396740252, 'fraction_numerical': 0.002682071384361461, 'mean_word_length': 3.320855614973262, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5341732', 'n_tokens_mistral': 1799, 'n_tokens_neox': 1758, 'n_words': 385}