<commit_before><head>
  <title>Page Benchmark Options</title>

  <script src="jst/util.js" type="text/javascript"></script>
  <script src="jst/jsevalcontext.js" type="text/javascript"></script>
  <script src="jst/jstemplate.js" type="text/javascript"></script>
  <script src="jquery/jquery-1.4.2.min.js" type="text/javascript"></script>
  <script src="jquery/jquery.flot.min.js" type="text/javascript"></script>
  <script src="jquery/jquery.flot.dashes.js" type="text/javascript"></script>
  <script src="util/table2CSV.js" type="text/javascript"></script> 
  <script src="util/sorttable.js" type="text/javascript"></script>

<style>
body {
  font-size: 84%;
  font-family: Helvetica, Arial, sans-serif;
  padding: 0.75em;
  margin: 0;
  min-width: 45em;
}

h1 {
  font-size: 110%;
  font-weight: bold;
  color: #4a8ee6;
  letter-spacing: -1px;
  padding: 0;
  margin: 0;
}

div#header {
  padding: 0.75em 1em;
  padding-top: 0.6em;
  padding-left: 10;
  margin-bottom: 0.75em;
  position: relative;
  overflow: hidden;
  background: #5296de;
  background-size: 100%;
  border: 1px solid #3a75bd;
  border-radius: 6px;
  color: white;
  text-shadow: 0 0 2px black;
}
div#header h1 {
  padding-left: 37px;
  margin: 0;
  display: inline;
  color: white;
}
div#header p {
  font-size: 84%;
  font-style: italic;
  padding: 0;
  margin: 0;
  color: white;
  padding-left: 0.4em;
  display: inline;
}

table.sortable {
  font-size: 84%;
  table-layout: fixed;
}

table.sortable:not([class*='filtered']) tr:nth-child(even) td:not([class*='filtered']) {
  background: #eff3ff;
}

.nobg {
  padding: 0 0.5em;
  vertical-align: bottom;
  font-weight: bold;
  color: #315d94;
  color: black;
  text-align: center;
}

.bg{
  padding: 0 0.5em;
  vertical-align: bottom;
  font-weight: bold;
  color: #315d94;
  color: black;
  text-align: center;
  cursor: pointer;
}

.bg:hover {
  background: #eff3aa;
}

.avg {
  font-weight: bold;
  text-align: center;
}

.data {
  text-align: left;
  white-space: nowrap;
}

.bggraph {
  white-space: nowrap;
}

.file_input
{
  position: absolute;
  width: 140px;
  height: 26px;
  overflow: hidden;
}

.file_input_button
{
  width: 140px;
  position: absolute;
  top: 0px;
}

.file_input_hidden
{
  font-size: 25px;
  position: absolute;
  right: 0px;
  top: 0px;
  opacity: 0;
}
</style>

<script>
var max_sample = 0;

Array.max = function(array) {
  return Math.max.apply( Math, array );
}

Array.min = function(array) {
  return Math.min.apply( Math, array );
};

// Compute the average of an array, removing the min/max.
Array.avg = function(array) {
  var count = array.length;
  var sum = 0;
  var min = array[0];
  var max = array[0];
  for (var i = 0; i < count; i++) {
    sum += array[i];
    if (array[i] < min) {
      min = array[i];
    }
    if (array[i] > max) {
      max = array[i];
    }
  }
  if (count >= 3) {
    sum = sum - min - max;
    count -= 2;
  }
  return sum / count;
}

// Compute the standard deviation of an array
Array.stddev = function(array) {
  var count = array.length;
  var mean = Array.avg(array);
  var variance = 0;
  for (var i = 0; i < count; i++) {
    var deviation = mean - array[i];
    variance = variance + deviation * deviation;
  }
  variance = variance / count;
  return Math.sqrt(variance);
}

function handleFileSelect(evt) {
  var files = evt.target.files;
  for (var i = 0, f; f = files[i]; i++) {<commit_msg>The benchmarking tool calculates the population variance rather than the sample variance. Since benchmarking results represent only a sample of the entire population of results, it would be better to calculate the sample standard deviation, which is an unbiased estimator of the population variance.  Also, since the Array.avg function throws out the min and max, this function should not be used when calculating the variance.<commit_after><head>
  <title>Page Benchmark Options</title>

  <script src="jst/util.js" type="text/javascript"></script>
  <script src="jst/jsevalcontext.js" type="text/javascript"></script>
  <script src="jst/jstemplate.js" type="text/javascript"></script>
  <script src="jquery/jquery-1.4.2.min.js" type="text/javascript"></script>
  <script src="jquery/jquery.flot.min.js" type="text/javascript"></script>
  <script src="jquery/jquery.flot.dashes.js" type="text/javascript"></script>
  <script src="util/table2CSV.js" type="text/javascript"></script> 
  <script src="util/sorttable.js" type="text/javascript"></script>

<style>
body {
  font-size: 84%;
  font-family: Helvetica, Arial, sans-serif;
  padding: 0.75em;
  margin: 0;
  min-width: 45em;
}

h1 {
  font-size: 110%;
  font-weight: bold;
  color: #4a8ee6;
  letter-spacing: -1px;
  padding: 0;
  margin: 0;
}

div#header {
  padding: 0.75em 1em;
  padding-top: 0.6em;
  padding-left: 10;
  margin-bottom: 0.75em;
  position: relative;
  overflow: hidden;
  background: #5296de;
  background-size: 100%;
  border: 1px solid #3a75bd;
  border-radius: 6px;
  color: white;
  text-shadow: 0 0 2px black;
}
div#header h1 {
  padding-left: 37px;
  margin: 0;
  display: inline;
  color: white;
}
div#header p {
  font-size: 84%;
  font-style: italic;
  padding: 0;
  margin: 0;
  color: white;
  padding-left: 0.4em;
  display: inline;
}

table.sortable {
  font-size: 84%;
  table-layout: fixed;
}

table.sortable:not([class*='filtered']) tr:nth-child(even) td:not([class*='filtered']) {
  background: #eff3ff;
}

.nobg {
  padding: 0 0.5em;
  vertical-align: bottom;
  font-weight: bold;
  color: #315d94;
  color: black;
  text-align: center;
}

.bg{
  padding: 0 0.5em;
  vertical-align: bottom;
  font-weight: bold;
  color: #315d94;
  color: black;
  text-align: center;
  cursor: pointer;
}

.bg:hover {
  background: #eff3aa;
}

.avg {
  font-weight: bold;
  text-align: center;
}

.data {
  text-align: left;
  white-space: nowrap;
}

.bggraph {
  white-space: nowrap;
}

.file_input
{
  position: absolute;
  width: 140px;
  height: 26px;
  overflow: hidden;
}

.file_input_button
{
  width: 140px;
  position: absolute;
  top: 0px;
}

.file_input_hidden
{
  font-size: 25px;
  position: absolute;
  right: 0px;
  top: 0px;
  opacity: 0;
}
</style>

<script>
var max_sample = 0;

Array.max = function(array) {
  return Math.max.apply( Math, array );
}

Array.min = function(array) {
  return Math.min.apply( Math, array );
};

// Compute the average of an array, removing the min/max.
Array.avg = function(array) {
  var count = array.length;
  var sum = 0;
  var min = array[0];
  var max = array[0];
  for (var i = 0; i < count; i++) {
    sum += array[i];
    if (array[i] < min) {
      min = array[i];
    }
    if (array[i] > max) {
      max = array[i];
    }
  }
  if (count >= 3) {
    sum = sum - min - max;
    count -= 2;
  }
  return sum / count;
}

// Compute the sample standard deviation of an array
Array.stddev = function(array) {
  var count = array.length;
  var mean = 0;
  for (var i = 0; i < count; i++) {
    mean += array[i];
  }
  mean /= count;
  var variance = 0;
  for (var i = 0; i < count; i++) {
    var deviation = mean - array[i];
    variance = variance + deviation * deviation;
  }
  variance = variance / (count - 1);
  return Math.sqrt(variance);
}

function handleFileSelect(evt) {
  var files = evt.target.files;
  for (var i = 0, f; f = files[i]; i++) {
{'fraction_non_alphanumeric': 0.1575408261287224, 'fraction_numerical': 0.02840675174969123, 'mean_word_length': 3.330362448009507, 'pattern_counts': {'":': 0, '<': 54, '<?xml version=': 0, '>': 51, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5045624', 'n_tokens_mistral': 2923, 'n_tokens_neox': 2749, 'n_words': 711}