<commit_before>var CONDITION = 0;
var STREAM = 1;
var SUCCESS = 2;
var FAIL = 3;
var PAYLOAD = 4;

var EOF = {};

Function.prototype.desc = function(d) {
  this.toString = function() { return d; };
  return this;
}

var StringPS = {
  create: function(str) {
    var o = Object.create(this);
    o.pos = 0;
    o.str_ = [str];
    o.tail_ = [];
    return o;
  },
  set str(str) { this.str_[0] = str; },
  get head() { return this.pos >= this.str_[0].length ? EOF : this.str_[0].charAt(this.pos); },
  // TODO(kgr): next line is slow because it can't bet JITed, fix.
  get tail() {
    if ( ! this.tail_[0] ) {
      var tail = Object.create(this.__proto__);
      tail.str_ = this.str_;
      tail.pos = this.pos+1;
      tail.tail_ = [];
      this.tail_[0] = tail;
    }
    return this.tail_[0];
  },
  toString: function() {
    return this.str_[0].substring(this.pos);
  }
};

function HEAD(state) {
  return state[STREAM].head;
}

function COPY_STATE(state, ps) {
  if ( state[CONDITION] === "END" ||
       state[CONDITION] === "END_FAIL" )
    return state;
  return [
    state[CONDITION],
    ps,
    state[SUCCESS],
    state[FAIL],
    state[PAYLOAD]
  ];
}

function TAIL(state) {
  return state[STREAM].tail;
}

function VALUE(state, value) {
  state[PAYLOAD] && state[PAYLOAD](value);
}

function COLLECT(state, f) {
  if ( state[PAYLOAD] ) {
    return f;
  }
}

function DO_SUCCESS(state, ps) {
//  if ( ! state[SUCCESS][STREAM] )
  //    state[SUCCESS][STREAM] = ps;
  if ( typeof state[SUCCESS] === "function" )
    return state[SUCCESS](ps);
  if ( ! state[SUCCESS][STREAM] ) state[SUCCESS][STREAM] = ps;
  return state[SUCCESS];
}

function DO_FAIL(state, ps) {
//  if ( ! state[FAIL][STREAM] )
//    state[FAIL][STREAM] = ps;
  if ( typeof state[FAIL] === "function" )
    return state[FAIL](ps);
  if ( ! state[FAIL][STREAM] ) state[FAIL][STREAM] = ps;
  return state[FAIL];
}

function char(c) {
  return function(state) {
    if ( HEAD(state) == c ) {
      VALUE(state, c);
      return DO_SUCCESS(state, TAIL(state));
    }
    return DO_FAIL(state, state[STREAM])
  }
}

function char_ic(c) {
  c = c.toLowerCase();
  return function(state) {
    if ( HEAD(state).toLowerCase() == c ) {
      VALUE(state, HEAD(state));
      return DO_SUCCESS(state, TAIL(state));
    }
    return DO_FAIL(state, state[STREAM]);
  }
}

function literal(str, opt_value) {
  var s = [];
  for ( var i = 0; i < str.length ; i++ ) {
    s.push(char(str[i]));
  }
  return seq.apply(null, s);
}

function literal_ic(str, opt_value) {
  var s = [];
  for ( var i = 0 ; i < str.length ; i++ ) {
    s.push(char_ic(str[i]));
  }
  return seq.apply(null, s);
}

function range(c1, c2) {
  return function(state) {<commit_msg>Add optimized literal to core/experimental/parse3.js<commit_after>var CONDITION = 0;
var STREAM = 1;
var SUCCESS = 2;
var FAIL = 3;
var PAYLOAD = 4;

var EOF = {};

Function.prototype.desc = function(d) {
  this.toString = function() { return d; };
  return this;
}

var StringPS = {
  create: function(str) {
    var o = Object.create(this);
    o.pos = 0;
    o.str_ = [str];
    o.tail_ = [];
    return o;
  },
  set str(str) { this.str_[0] = str; },
  get head() { return this.pos >= this.str_[0].length ? EOF : this.str_[0].charAt(this.pos); },
  // TODO(kgr): next line is slow because it can't bet JITed, fix.
  get tail() {
    if ( ! this.tail_[0] ) {
      var tail = Object.create(this.__proto__);
      tail.str_ = this.str_;
      tail.pos = this.pos+1;
      tail.tail_ = [];
      this.tail_[0] = tail;
    }
    return this.tail_[0];
  },
  toString: function() {
    return this.str_[0].substring(this.pos);
  }
};

function HEAD(state) {
  return state[STREAM].head;
}

function COPY_STATE(state, ps) {
  if ( state[CONDITION] === "END" ||
       state[CONDITION] === "END_FAIL" )
    return state;
  return [
    state[CONDITION],
    ps,
    state[SUCCESS],
    state[FAIL],
    state[PAYLOAD]
  ];
}

function TAIL(state) {
  return state[STREAM].tail;
}

function VALUE(state, value) {
  state[PAYLOAD] && state[PAYLOAD](value);
}

function COLLECT(state, f) {
  if ( state[PAYLOAD] ) {
    return f;
  }
}

function DO_SUCCESS(state, ps) {
//  if ( ! state[SUCCESS][STREAM] )
  //    state[SUCCESS][STREAM] = ps;
  if ( typeof state[SUCCESS] === "function" )
    return state[SUCCESS](ps);
  if ( ! state[SUCCESS][STREAM] ) state[SUCCESS][STREAM] = ps;
  return state[SUCCESS];
}

function DO_FAIL(state, ps) {
//  if ( ! state[FAIL][STREAM] )
//    state[FAIL][STREAM] = ps;
  if ( typeof state[FAIL] === "function" )
    return state[FAIL](ps);
  if ( ! state[FAIL][STREAM] ) state[FAIL][STREAM] = ps;
  return state[FAIL];
}

function char(c) {
  return function(state) {
    if ( HEAD(state) == c ) {
      VALUE(state, c);
      return DO_SUCCESS(state, TAIL(state));
    }
    return DO_FAIL(state, state[STREAM])
  }
}

function char_ic(c) {
  c = c.toLowerCase();
  return function(state) {
    if ( HEAD(state).toLowerCase() == c ) {
      VALUE(state, HEAD(state));
      return DO_SUCCESS(state, TAIL(state));
    }
    return DO_FAIL(state, state[STREAM]);
  }
}

function xxliteral(str, opt_value) {
  var s = [];
  for ( var i = 0; i < str.length ; i++ ) {
    s.push(char(str[i]));
  }
  return seq.apply(null, s);
}

function literal(str, opt_value) {
  return function(state) {
    var i = 0;
    var ps = state[STREAM];
    var c = ps.head;
    while ( c != null && i < str.length ) {
      if ( str.charAt(i) !== c) {
        return DO_FAIL(state, state[STREAM]);
      }
      ps = ps.tail;
      c = ps.head;
      i++;
    }
    if ( c == null ) {
      debugger;
      return [
        literal(str.subtring(i), opt_value || str),
        ps,
        state[SUCCESS],
        state[FAIL],
        state[PAYLOAD]
      ];
    }
    VALUE(state, opt_value || str);
    return DO_SUCCESS(state, ps);
  };
}

function literal_ic(str, opt_value) {
  var s = [];
  for ( var i = 0 ; i < str.length ; i++ ) {
    s.push(char_ic(str[i]));
  }
  return seq.apply(null, s);
}

function range(c1, c2) {
  return function(state) {
{'fraction_non_alphanumeric': 0.1758043335521996, 'fraction_numerical': 0.00623768877216021, 'mean_word_length': 2.8272613065326633, 'pattern_counts': {'":': 0, '<': 8, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2237839', 'n_tokens_mistral': 2283, 'n_tokens_neox': 2258, 'n_words': 500}