<commit_before>var jobMap = {};

var jobs = [
  // box2d. build instructions: let emscripten benchmark suite generate it for you
  {
    benchmark: 'box2d-throughput',
    description: 'Box2D physics: average frame rate',
    scale: 'milliseconds (lower numbers are better)',
    args: ['3'],
    createWorker: function() {
      return new Worker('box2d/benchmark-worker.js')
    },
    calculate: function() {
      // output format is:         frame averages: 31.675 +- 7.808, range: 22.000 to 63.000
      var m = /frame averages: (\d+\.\d+) \+- (\d+\.\d+), range: (\d+\.\d+) to (\d+\.\d+)/.exec(this.msg.output);
      this.parsed = {
        average: parseFloat(m[1]),
        variance: parseFloat(m[2]),
        lowest: parseFloat(m[3]),
        highest: parseFloat(m[4])
      };
      return this.parsed.average;
    },
    normalized: function() {
      return (20.308/this.calculate());
    },
  },
  {
    benchmark: 'box2d-latency',
    description: 'Box2D physics: frame variability and worst case',
    scale: 'milliseconds (lower numbers are better)',
    createWorker: function() {
      return {
        postMessage: function() {
          this.onmessage({ data: {
            benchmark: 'box2d-latency'
          }});
        }
      };
    },
    calculate: function() {
      var parsed = jobMap['box2d-throughput'].parsed;
      return (2*parsed.variance + (parsed.highest - parsed.average))/3;
    },
    normalized: function() {
      return (20.308/this.calculate());
    },
  },
  {
    benchmark: 'box2d-warm-startup',
    description: 'how long a warm startup takes for Box2D',
    scale: 'seconds (lower numbers are better)',
    args: ['0'],
    createWorker: function() {
      return new Worker('box2d/benchmark-worker.js')
    },
    calculate: function() {
      return this.msg.runtime/1000;
    },
    normalized: function() {
      return 0.10/Math.max(this.calculate(), 1/60);
    },
  },
  {
    benchmark: 'box2d-cold-startup',
    description: 'how long a cold startup takes for Box2D',
    scale: 'seconds (lower numbers are better)',
    args: 'cold',
    createWorker: function() {
      return new Worker('box2d/benchmark-worker.js')
    },
    calculate: function() {
      return this.msg.runtime/1000;
    },
    normalized: function() {
      return 0.10/Math.max(this.calculate(), 1/60);
    },
  },

  // lua. build instructions: use lua.vm.js project build system
  {
    benchmark: 'lua-binarytrees',
    description: 'GC performance in compiled Lua VM',
    scale: 'seconds (lower numbers are better)',
    args: ['binarytrees.lua'],
    createWorker: function() {
      return new Worker('lua/benchmark-worker.js')
    },
    calculate: function() {
      return this.msg.runtime/1000;
    },
    normalized: function() {
      return (20.308/this.calculate());
    },
  },
  {
    benchmark: 'lua-scimark',
    description: 'numeric computation performance in compiled Lua VM',
    scale: 'MFLOPS (higher numbers are better)',
    args: ['scimark.lua'],
    createWorker: function() {
      return new Worker('lua/benchmark-worker.js')
    },
    calculate: function() {
      return parseFloat(/\nSciMark +([\d\.]+)/.exec(this.msg.output)[1]);
    },
    normalized: function() {
      return (this.calculate()/3.19);
    },
  },
  { // do startup last so there is no network access
    benchmark: 'lua-warm-startup',
    description: 'how long a warm startup takes the compiled Lua VM',
    scale: 'seconds (lower numbers are better)',
    args: null,
    createWorker: function() {
      return new Worker('lua/benchmark-worker.js')
    },
    calculate: function() {
      return this.msg.startup/1000;
    },
    normalized: function() {
      return 0.10/Math.max(this.calculate(), 1/60); // resolution: 1 frame
    },
  },

  // sqlite. build instructions: run asm3.test_sqlite in emscripten test suite
  {
    benchmark: 'sqlite',
    description: 'sqlite operations performance (create, inserts, selects)',
    scale: 'seconds (lower numbers are better)',
    args: ['5000', '9'],
    createWorker: function() {
      return new Worker('sqlite/benchmark-worker.js')
    },
    calculate: function() {
      return this.msg.runtime/1000;
    },
    normalized: function() {
      return 20.308/Math.max(this.calculate(), 1/60);
    },
  },
];

var ran = false;
function run() {
  if (ran) return;
  ran = true;

  document.getElementById('results_area').hidden = false;

  var tableBody = document.getElementById('table_body');
  tableBody.innerHTML = '';

  var theButton = document.getElementById('the_button');
  theButton.innerHTML = 'Running benchmarks... (this can take a while)';
  theButton.classList.remove('btn-primary');
  theButton.classList.add('btn-warning');

  function finalCalculation() {
    // normalize based on experimental data
    var normalized = jobs.map(function(job) { return 100*job.normalized() });
    console.log('normalized values: ' + normalized);
    return Math.round(normalized.reduce(function(x, y) { return x + y }, 0));<commit_msg>fix sqlite processing to focus on computation only<commit_after>var jobMap = {};

var jobs = [
  // box2d. build instructions: let emscripten benchmark suite generate it for you
  {
    benchmark: 'box2d-throughput',
    description: 'Box2D physics: average frame rate',
    scale: 'milliseconds (lower numbers are better)',
    args: ['3'],
    createWorker: function() {
      return new Worker('box2d/benchmark-worker.js')
    },
    calculate: function() {
      // output format is:         frame averages: 31.675 +- 7.808, range: 22.000 to 63.000
      var m = /frame averages: (\d+\.\d+) \+- (\d+\.\d+), range: (\d+\.\d+) to (\d+\.\d+)/.exec(this.msg.output);
      this.parsed = {
        average: parseFloat(m[1]),
        variance: parseFloat(m[2]),
        lowest: parseFloat(m[3]),
        highest: parseFloat(m[4])
      };
      return this.parsed.average;
    },
    normalized: function() {
      return (20.308/this.calculate());
    },
  },
  {
    benchmark: 'box2d-latency',
    description: 'Box2D physics: frame variability and worst case',
    scale: 'milliseconds (lower numbers are better)',
    createWorker: function() {
      return {
        postMessage: function() {
          this.onmessage({ data: {
            benchmark: 'box2d-latency'
          }});
        }
      };
    },
    calculate: function() {
      var parsed = jobMap['box2d-throughput'].parsed;
      return (2*parsed.variance + (parsed.highest - parsed.average))/3;
    },
    normalized: function() {
      return (20.308/this.calculate());
    },
  },
  {
    benchmark: 'box2d-warm-startup',
    description: 'how long a warm startup takes for Box2D',
    scale: 'seconds (lower numbers are better)',
    args: ['0'],
    createWorker: function() {
      return new Worker('box2d/benchmark-worker.js')
    },
    calculate: function() {
      return this.msg.runtime/1000;
    },
    normalized: function() {
      return 0.10/Math.max(this.calculate(), 1/60);
    },
  },
  {
    benchmark: 'box2d-cold-startup',
    description: 'how long a cold startup takes for Box2D',
    scale: 'seconds (lower numbers are better)',
    args: 'cold',
    createWorker: function() {
      return new Worker('box2d/benchmark-worker.js')
    },
    calculate: function() {
      return this.msg.runtime/1000;
    },
    normalized: function() {
      return 0.10/Math.max(this.calculate(), 1/60);
    },
  },

  // lua. build instructions: use lua.vm.js project build system
  {
    benchmark: 'lua-binarytrees',
    description: 'GC performance in compiled Lua VM',
    scale: 'seconds (lower numbers are better)',
    args: ['binarytrees.lua'],
    createWorker: function() {
      return new Worker('lua/benchmark-worker.js')
    },
    calculate: function() {
      return this.msg.runtime/1000;
    },
    normalized: function() {
      return (20.308/this.calculate());
    },
  },
  {
    benchmark: 'lua-scimark',
    description: 'numeric computation performance in compiled Lua VM',
    scale: 'MFLOPS (higher numbers are better)',
    args: ['scimark.lua'],
    createWorker: function() {
      return new Worker('lua/benchmark-worker.js')
    },
    calculate: function() {
      return parseFloat(/\nSciMark +([\d\.]+)/.exec(this.msg.output)[1]);
    },
    normalized: function() {
      return (this.calculate()/3.19);
    },
  },
  { // do startup last so there is no network access
    benchmark: 'lua-warm-startup',
    description: 'how long a warm startup takes the compiled Lua VM',
    scale: 'seconds (lower numbers are better)',
    args: null,
    createWorker: function() {
      return new Worker('lua/benchmark-worker.js')
    },
    calculate: function() {
      return this.msg.startup/1000;
    },
    normalized: function() {
      return 0.10/Math.max(this.calculate(), 1/60); // resolution: 1 frame
    },
  },

  // sqlite. build instructions: run asm3.test_sqlite in emscripten test suite
  {
    benchmark: 'sqlite',
    description: 'sqlite operations performance (create, inserts, selects)',
    scale: 'seconds (lower numbers are better)',
    args: ['5000', '9'],
    createWorker: function() {
      return new Worker('sqlite/benchmark-worker.js')
    },
    calculate: function() {
      var m = /create table : took (\d+) ms\n25,000 inserts : took (\d+) ms\ncommit : took (\d+) ms\ncount\(\*\) = 25000\n\ncount\(\*\) = 5000\n\ncount\(\*\) = 15000\n\ncount\(\*\) = 5000\n\nselects : took (\d+) ms\ncreate indexes : took (\d+) ms\ncount\(\*\) = 5000\n\ncount\(\*\) = 15000\n\nselects with indexes : took (\d+) ms/.exec(this.msg.output);
      if (!m) throw 'invalid sqlite output: ' + this.msg.output;
      m = m.map(parseFloat);
      return (m[1]+m[2]+m[3]+m[4]+m[5]+m[6])/1000;
    },
    normalized: function() {
      return 4.0/Math.max(this.calculate(), 1/60);
    },
  },
];

var ran = false;
function run() {
  if (ran) return;
  ran = true;

  document.getElementById('results_area').hidden = false;

  var tableBody = document.getElementById('table_body');
  tableBody.innerHTML = '';

  var theButton = document.getElementById('the_button');
  theButton.innerHTML = 'Running benchmarks... (this can take a while)';
  theButton.classList.remove('btn-primary');
  theButton.classList.add('btn-warning');

  function finalCalculation() {
    // normalize based on experimental data
    var normalized = jobs.map(function(job) { return 100*job.normalized() });
    console.log('normalized values: ' + normalized);
    return Math.round(normalized.reduce(function(x, y) { return x + y }, 0));
{'fraction_non_alphanumeric': 0.1579445443361408, 'fraction_numerical': 0.025456610201570928, 'mean_word_length': 2.7878136200716845, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '438324', 'n_tokens_mistral': 3509, 'n_tokens_neox': 3321, 'n_words': 821}