<commit_before>var A = 'A'.charCodeAt(0);
var Z = 'Z'.charCodeAt(0);
var a = 'a'.charCodeAt(0);
var z = 'z'.charCodeAt(0);


var distribution = null;
$.get("distribution/trigrams.json").done(function(d){
  window.distribution = d;
});


function scoreString(s){
  s = s.replace(/\s+/g, ""); 
  if (distribution ==- null  || s.length < 3){
    return null;
  }

  var N = 3; // use trigrams

  var scores = toNgrams(s, N).map(scoreNgram);
  var raw = scores.reduce(sum);
  var penaltyAmount = distribution['ING']; // common trigram for big penalty
  var penalty = scores.filter(isCrappy).length * penaltyAmount;

  return raw - penalty;
}

function sum(v,n){return v+n;}

function isCrappy(t){
  return t < distribution['EVV']; // uncommon trigram as reference point for crappiness
}

function toNgrams(s, N){<commit_msg>Get JSON explicitly, in case the server serves .json with the wrong MIME type.<commit_after>var A = 'A'.charCodeAt(0);
var Z = 'Z'.charCodeAt(0);
var a = 'a'.charCodeAt(0);
var z = 'z'.charCodeAt(0);


var distribution = null;
$.getJSON("distribution/trigrams.json").done(function(d){
  window.distribution = d;
});


function scoreString(s){
  s = s.replace(/\s+/g, ""); 
  if (distribution ==- null  || s.length < 3){
    return null;
  }

  var N = 3; // use trigrams

  var scores = toNgrams(s, N).map(scoreNgram);
  var raw = scores.reduce(sum);
  var penaltyAmount = distribution['ING']; // common trigram for big penalty
  var penalty = scores.filter(isCrappy).length * penaltyAmount;

  return raw - penalty;
}

function sum(v,n){return v+n;}

function isCrappy(t){
  return t < distribution['EVV']; // uncommon trigram as reference point for crappiness
}

function toNgrams(s, N){
{'fraction_non_alphanumeric': 0.16208308952603862, 'fraction_numerical': 0.007021650087770626, 'mean_word_length': 4.0, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3985346', 'n_tokens_mistral': 622, 'n_tokens_neox': 581, 'n_words': 160}