<commit_before>#!/usr/bin/env node

'use strict';

var child_process = require('child_process'),
    fs = require('fs'),
    path = require('path');

var urls = require('./benchmarks');
var fileNames = Object.keys(urls).sort();

function all(tasks, callback) {
  var remaining = tasks.length;

  function done() {
    if (!--remaining) {
      callback();
    }
  }

  tasks.forEach(function(task) {
    task(done);
  });
}

function git() {
  var args = [].slice.call(arguments, 0, -1);
  var callback = arguments[arguments.length - 1];
  var task = child_process.spawn('git', args, { stdio: [ 'ignore', 'pipe', 'ignore' ] });
  var output = '';
  task.stdout.setEncoding('utf8');
  task.stdout.on('data', function(data) {
    output += data;
  }).on('end', function() {
    callback(output);
  });
}

function setupConf(hash, callback) {
  var output = path.join('benchmarks/generated/', hash + '.conf');

  function done(data) {
    fs.writeFile(output, data, function(err) {
      if (err) {
        console.error(hash, ': error writing backtest.csv');
        callback();
      }
      else {
        callback(output);
      }
    });
  }

  fs.readFile('benchmark.conf', function(err, data) {
    if (err) {
      fs.readFile('sample-cli-config-file.conf', function(err, data) {
        if (err) {
          console.error(hash, ': failed to extract configuration');
          process.send({});
        }
        else {
          done(data);
        }
      });
    }
    else {
      done(data);
    }
  });
}

function minify(hash) {
  setupConf(hash, function(conf) {
    var results = {};
    process.send('ready');
    all(fileNames.map(function(fileName) {
      return function(done) {
        var input = path.join('benchmarks/', fileName + '.html');
        var output = path.join('benchmarks/generated/', fileName + '.' + hash + '.html');
        var args = [input, '-c', conf, '--minify-urls', urls[fileName], '-o', output];
        var task = child_process.fork('./cli', args, { silent: true }).on('exit', function () {
          fs.stat(output, function(err, stats) {
            if (err) {
              console.error(hash, ': failed to minify "' + fileName + '"');
            }
            else {
              results[fileName] = stats.size;
            }
            done();
          });
        });
        task.stdout.resume();
        task.stderr.resume();
        setTimeout(function() {
          task.kill();
        }, 10000);
      };
    }), function() {
      process.send(results);
    });
  });
}

function print(table) {
  var output = [];
  var row = fileNames.slice(0);
  row.unshift('hash', 'date');
  output.push(row.join(','));
  for (var hash in table) {
    var data = table[hash];
    row = [ hash, '"' + data.date + '"' ];
    for (var i = 0; i < fileNames.length; i++) {
      row.push(data[fileNames[i]]);
    }
    output.push(row.join(','));
  }
  fs.writeFile('backtest.csv', output.join('\n'), { encoding: 'utf8' }, function(err) {
    if (err) {
      throw new Error('There was an error writing backtest.csv');
    }
  });
}

if (process.argv.length > 2) {
  var count = +process.argv[2];
  if (count) {
    git('log', '--date=iso', '--pretty=format:%h %cd', '-' + count, function(data) {
      var table = {};
      var commits = data.split(/\s*?\n/).map(function(line) {
        var index = line.indexOf(' ');
        var hash = line.substr(0, index);
        table[hash] = {
          date: line.substr(index + 1).replace('+', '').replace(/ 0000$/, '')
        };
        return hash;
      });
      var running = 0, ready = true;

      function fork() {
        if (commits.length && running < 8) {
          var hash = commits.shift();
          var task = child_process.fork('./backtest');
          task.on('message', function(data) {
            if (data === 'ready') {
              ready = true;
            }
            else {
              var date = table[hash].date;
              table[hash] = data;
              table[hash].date = date;
              task.disconnect();
              if (!--running && !commits.length) {
                print(table);
              }
            }
            fork();<commit_msg>adjust magic constants<commit_after>#!/usr/bin/env node

'use strict';

var child_process = require('child_process'),
    fs = require('fs'),
    path = require('path');

var urls = require('./benchmarks');
var fileNames = Object.keys(urls).sort();

function all(tasks, callback) {
  var remaining = tasks.length;

  function done() {
    if (!--remaining) {
      callback();
    }
  }

  tasks.forEach(function(task) {
    task(done);
  });
}

function git() {
  var args = [].slice.call(arguments, 0, -1);
  var callback = arguments[arguments.length - 1];
  var task = child_process.spawn('git', args, { stdio: [ 'ignore', 'pipe', 'ignore' ] });
  var output = '';
  task.stdout.setEncoding('utf8');
  task.stdout.on('data', function(data) {
    output += data;
  }).on('end', function() {
    callback(output);
  });
}

function setupConf(hash, callback) {
  var output = path.join('benchmarks/generated/', hash + '.conf');

  function done(data) {
    fs.writeFile(output, data, function(err) {
      if (err) {
        console.error(hash, ': error writing backtest.csv');
        callback();
      }
      else {
        callback(output);
      }
    });
  }

  fs.readFile('benchmark.conf', function(err, data) {
    if (err) {
      fs.readFile('sample-cli-config-file.conf', function(err, data) {
        if (err) {
          console.error(hash, ': failed to extract configuration');
          process.send({});
        }
        else {
          done(data);
        }
      });
    }
    else {
      done(data);
    }
  });
}

function minify(hash) {
  setupConf(hash, function(conf) {
    var results = {};
    process.send('ready');
    all(fileNames.map(function(fileName) {
      return function(done) {
        var input = path.join('benchmarks/', fileName + '.html');
        var output = path.join('benchmarks/generated/', fileName + '.' + hash + '.html');
        var args = [input, '-c', conf, '--minify-urls', urls[fileName], '-o', output];
        var task = child_process.fork('./cli', args, { silent: true }).on('exit', function () {
          fs.stat(output, function(err, stats) {
            if (err) {
              console.error(hash, ': failed to minify "' + fileName + '"');
            }
            else {
              results[fileName] = stats.size;
            }
            done();
          });
        });
        task.stdout.resume();
        task.stderr.resume();
        setTimeout(function() {
          task.kill();
        }, 15000);
      };
    }), function() {
      process.send(results);
    });
  });
}

function print(table) {
  var output = [];
  var row = fileNames.slice(0);
  row.unshift('hash', 'date');
  output.push(row.join(','));
  for (var hash in table) {
    var data = table[hash];
    row = [ hash, '"' + data.date + '"' ];
    for (var i = 0; i < fileNames.length; i++) {
      row.push(data[fileNames[i]]);
    }
    output.push(row.join(','));
  }
  fs.writeFile('backtest.csv', output.join('\n'), { encoding: 'utf8' }, function(err) {
    if (err) {
      throw new Error('There was an error writing backtest.csv');
    }
  });
}

if (process.argv.length > 2) {
  var count = +process.argv[2];
  if (count) {
    git('log', '--date=iso', '--pretty=format:%h %cd', '-' + count, function(data) {
      var table = {};
      var commits = data.split(/\s*?\n/).map(function(line) {
        var index = line.indexOf(' ');
        var hash = line.substr(0, index);
        table[hash] = {
          date: line.substr(index + 1).replace('+', '').replace(/ 0000$/, '')
        };
        return hash;
      });
      var running = 0, ready = true;

      function fork() {
        if (commits.length && running < 4) {
          var hash = commits.shift();
          var task = child_process.fork('./backtest');
          task.on('message', function(data) {
            if (data === 'ready') {
              ready = true;
            }
            else {
              var date = table[hash].date;
              table[hash] = data;
              table[hash].date = date;
              task.disconnect();
              if (!--running && !commits.length) {
                print(table);
              }
            }
            fork();
{'fraction_non_alphanumeric': 0.18351032096408543, 'fraction_numerical': 0.00524997017062403, 'mean_word_length': 2.0468920392584513, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4897378', 'n_tokens_mistral': 2659, 'n_tokens_neox': 2621, 'n_words': 555}