<commit_before>/*!
 Based on ndef.parser, by <NAME>(<EMAIL>)
 http://www.undefined.ch/mparser/index.html

 Ported to JavaScript and modified by Matthew Crumley (<EMAIL>, http://silentmatt.com/)

 You are free to use and modify this code in anyway you find useful. Please leave this comment in the code
 to acknowledge its original source. If you feel like it, I enjoy hearing about projects that use my code,
 but don't feel like you have to let me know or ask permission.
*/

var Parser = (function (scope) { // eslint-disable-line no-unused-vars
  //  Added by stlsmiths 6/13/2011
  //  re-define Array.indexOf, because IE doesn't know it ...
  function indexOf(array, obj, start) {
    for (var i = (start || 0); i < array.length; i++) {
      if (array[i] === obj) {
        return i;
      }
    }
    return -1;
  }

  function object(o) {
    function F() {}
    F.prototype = o;
    return new F();
  }

  var TNUMBER = 0;
  var TOP1 = 1;
  var TOP2 = 2;
  var TVAR = 3;
  var TFUNCALL = 4;

  function Token(type, index, prio, number) {<commit_msg>Use strings for token types.<commit_after>/*!
 Based on ndef.parser, by <NAME>(<EMAIL>)
 http://www.undefined.ch/mparser/index.html

 Ported to JavaScript and modified by <NAME> (<EMAIL>, http://silentmatt.com/)

 You are free to use and modify this code in anyway you find useful. Please leave this comment in the code
 to acknowledge its original source. If you feel like it, I enjoy hearing about projects that use my code,
 but don't feel like you have to let me know or ask permission.
*/

var Parser = (function (scope) { // eslint-disable-line no-unused-vars
  //  Added by stlsmiths 6/13/2011
  //  re-define Array.indexOf, because IE doesn't know it ...
  function indexOf(array, obj, start) {
    for (var i = (start || 0); i < array.length; i++) {
      if (array[i] === obj) {
        return i;
      }
    }
    return -1;
  }

  function object(o) {
    function F() {}
    F.prototype = o;
    return new F();
  }

  var TNUMBER = 'TNUMBER';
  var TOP1 = 'TOP1';
  var TOP2 = 'TOP2';
  var TVAR = 'TVAR';
  var TFUNCALL = 'TFUNCALL';

  function Token(type, index, prio, number) {
{'fraction_non_alphanumeric': 0.12407063197026022, 'fraction_numerical': 0.013475836431226766, 'mean_word_length': 2.8106194690265487, 'pattern_counts': {'":': 0, '<': 12, '<?xml version=': 0, '>': 10, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2784935', 'n_tokens_mistral': 732, 'n_tokens_neox': 694, 'n_words': 273}