<commit_before>// Options: --free-variable-checker --require --validate
module.exports = (function(){
  "use strict";

  const {def, confine} = require('./sesshim.es6');
  const sc = require('./scanner.es6');

  function simple(prefix, list) {
    if (list.length === 0) { return ['empty']; }
    if (list.length === 1) { return list[0]; }
    return [prefix, ...list];
  }

  function indent(str, newnewline) {
    return str.replace(/\n/g, newnewline);
  }

  function compile(sexp) {
    let numSubs = 0;
    const tokenTypes = new Set();

    // generated names
    // act_${i}      action parameter
    // rule_${name}  method from bnf rule
    // seq_${i}      sequence failure label
    // or_${i}       choice success label
    // pos_${i}      backtrack token index
    // s_${i}        accumulated list of values
    // v_${i}        set to s_${i} on fall thru path

    let alphaCount = 0;
    const vars = ['let value = FAIL'];
    function nextVar(prefix) {
      const result = `${prefix}_${alphaCount++}`;
      vars.push(result);
      return result;
    }
    function takeVarsSrc() {
      const result = `${vars.join(', ')};`;
      vars.length = 1;
      return result;
    }
    function nextLabel(prefix) {
      return `${prefix}_${alphaCount++}`;
    }


    function peval(sexp) {
      const vtable = Object.freeze({
        bnf(...rules) {
          // The following line also initializes tokenTypes and numSubs
          const rulesSrc = rules.map(peval).join('\n');

          const paramSrcs = [];
          for (let i = 0; i < numSubs; i++) {
            paramSrcs.push(`act_${i}`);
          }
          const tokenTypeListSrc =
                `[${[...tokenTypes].map(tt => JSON.stringify(tt)).join(', ')}]`;
          // rules[0] is the ast of the first rule, which has the form
          // ["def", ruleName, body], so rules[0][1] is the name of
          // the start rule. We prepend "rule_" to get the name of the
          // JS method that implements the start rule. We invoke it
          // with (0) so that it will parse starting at position 0. It
          // returns a pair of the final position (after the last
          // non-EOF token parsed), and the semantic value. On failure
          // to parse, the semantic value will be FAIL.
          const name = rules[0][1];
          return (
`(function(${paramSrcs.join(', ')}) {
  return BaseParser => class extends BaseParser {
    constructor(template, tokenTypeList=[]) {
      super(template, ${tokenTypeListSrc}.concat(tokenTypeList));
    }
    start() {
      const pair = this.run(this.rule_${name}, 0, ${JSON.stringify(name)});
      if (pair[1] === FAIL) {
        this.syntaxError();
      }
      return pair[1];
    }
    ${indent(rulesSrc,`
    `)}
  }
})
`);
        },
        def(name, body) {
          // The following line also initializes vars
          const bodySrc = peval(body);
          return (
`rule_${name}(pos) {
  ${takeVarsSrc()}
  ${indent(bodySrc,`
  `)}
  return [pos, value];
}`);
        },
        empty() {
          return `value = [];`;
        },
        fail() {
          return `value = FAIL;`;
        },
        or(...choices) {
          const labelSrc = nextLabel('or');
          const choicesSrc = choices.map(peval).map(cSrc =>
`${cSrc}
if (value !== FAIL) break ${labelSrc};`).join('\n');

          return (
`${labelSrc}: {
  ${indent(choicesSrc,`
  `)}
}`);
        },
        seq(...terms) {
          const posSrc = nextVar('pos');
          const labelSrc = nextLabel('seq');
          const sSrc = nextVar('s');
          const vSrc = nextVar('v');
          const termsSrc = terms.map(peval).map(termSrc =>
`${termSrc}
if (value === FAIL) break ${labelSrc};
${sSrc}.push(value);`).join('\n');

          return (
`${sSrc} = [];
${vSrc} = FAIL;
${posSrc} = pos;
${labelSrc}: {
  ${indent(termsSrc,`
  `)}
  ${vSrc} = ${sSrc};
}
if ((value = ${vSrc}) === FAIL) pos = ${posSrc};`);
        },
        act(terms, hole) {
          numSubs = Math.max(numSubs, hole + 1);
          const termsSrc = vtable.seq(...terms);
          return (
`${termsSrc}
if (value !== FAIL) value = act_${hole}(...value);`);
        },
        '**'(patt, sep) {
          // for backtracking
          const posSrc = nextVar('pos');
          // a non-advancing success only repeats once.
          const startSrc = nextVar('pos');
          const sSrc = nextVar('s');
          const pattSrc = peval(patt);
          const sepSrc = peval(sep);
          return (
// after first iteration, backtrack to before the separator
`${sSrc} = [];
${posSrc} = pos;
while (true) {
  ${startSrc} = pos;
  ${indent(pattSrc,`
  `)}
  if (value === FAIL) {
    pos = ${posSrc};
    break;
  }
  ${sSrc}.push(value);
  ${posSrc} = pos;
  ${indent(sepSrc,`
  `)}
  if (value === FAIL) break;
  if (pos === ${startSrc}) break;
}
value = ${sSrc};`);
        },
        '++'(patt, sep) {
          const starSrc = vtable['**'](patt, sep);
          return (
`${starSrc}
if (value.length === 0) value = FAIL;`);
        },
        '?'(patt) {
          return vtable['**'](patt, ['fail']);
        },
        '*'(patt) {
          return vtable['**'](patt, ['empty']);
        },
        '+'(patt) {
          return vtable['++'](patt, ['empty']);
        },
        'super'(ident) {
          return `[pos, value] = this.run(super.rule_${ident}, pos, ${
              JSON.stringify(ident)});`;
        },
        apply(hole) {
          numSubs = Math.max(numSubs, hole + 1);
          return `[pos, value] = act_${hole}.call(this, pos);`;
        }
      });

      if (typeof sexp === 'string') {
        // If sexp is already a string literal, this will doubly quote it,
        // which is what we want.
        const nameStr = JSON.stringify(sexp);
        if (sc.allRE(sc.STRING_RE).test(sexp)) {
          tokenTypes.add(sexp);
          return `[pos, value] = this.run(${sexp}, pos, ${nameStr});`;
        }
        if (sc.allRE(sc.IDENT_RE).test(sexp)) {
          // Assume a standalone identifier is a rule name.
          return `[pos, value] = this.run(this.rule_${sexp}, pos, ${nameStr});`;
        }
        throw new Error('unexpected: ' + sexp);
      }
      return vtable[sexp[0]](...sexp.slice(1));
    }

    return peval(sexp);
  }


  function quasiMemo(quasiCurry) {
    const wm = new WeakMap();
    return function(template, ...subs) {
      let quasiRest = wm.get(template);
      if (!quasiRest) {<commit_msg>Hoist vtable for efficiency, so it's not rebuilt every time through peval()<commit_after>// Options: --free-variable-checker --require --validate
module.exports = (function(){
  "use strict";

  const {def, confine} = require('./sesshim.es6');
  const sc = require('./scanner.es6');

  function simple(prefix, list) {
    if (list.length === 0) { return ['empty']; }
    if (list.length === 1) { return list[0]; }
    return [prefix, ...list];
  }

  function indent(str, newnewline) {
    return str.replace(/\n/g, newnewline);
  }

  function compile(sexp) {
    let numSubs = 0;
    const tokenTypes = new Set();

    // generated names
    // act_${i}      action parameter
    // rule_${name}  method from bnf rule
    // seq_${i}      sequence failure label
    // or_${i}       choice success label
    // pos_${i}      backtrack token index
    // s_${i}        accumulated list of values
    // v_${i}        set to s_${i} on fall thru path

    let alphaCount = 0;
    const vars = ['let value = FAIL'];
    function nextVar(prefix) {
      const result = `${prefix}_${alphaCount++}`;
      vars.push(result);
      return result;
    }
    function takeVarsSrc() {
      const result = `${vars.join(', ')};`;
      vars.length = 1;
      return result;
    }
    function nextLabel(prefix) {
      return `${prefix}_${alphaCount++}`;
    }


    const vtable = Object.freeze({
      bnf(...rules) {
        // The following line also initializes tokenTypes and numSubs
        const rulesSrc = rules.map(peval).join('\n');

        const paramSrcs = [];
        for (let i = 0; i < numSubs; i++) {
          paramSrcs.push(`act_${i}`);
        }
        const tokenTypeListSrc =
              `[${[...tokenTypes].map(tt => JSON.stringify(tt)).join(', ')}]`;
        // rules[0] is the ast of the first rule, which has the form
        // ["def", ruleName, body], so rules[0][1] is the name of
        // the start rule. We prepend "rule_" to get the name of the
        // JS method that implements the start rule. We invoke it
        // with (0) so that it will parse starting at position 0. It
        // returns a pair of the final position (after the last
        // non-EOF token parsed), and the semantic value. On failure
        // to parse, the semantic value will be FAIL.
        const name = rules[0][1];
        return (
`(function(${paramSrcs.join(', ')}) {
  return BaseParser => class extends BaseParser {
    constructor(template, tokenTypeList=[]) {
      super(template, ${tokenTypeListSrc}.concat(tokenTypeList));
    }
    start() {
      const pair = this.run(this.rule_${name}, 0, ${JSON.stringify(name)});
      if (pair[1] === FAIL) {
        this.syntaxError();
      }
      return pair[1];
    }
    ${indent(rulesSrc,`
    `)}
  }
})
`);
      },
      def(name, body) {
        // The following line also initializes vars
        const bodySrc = peval(body);
        return (
`rule_${name}(pos) {
  ${takeVarsSrc()}
  ${indent(bodySrc,`
  `)}
  return [pos, value];
}`);
      },
      empty() {
        return `value = [];`;
      },
      fail() {
        return `value = FAIL;`;
      },
      or(...choices) {
        const labelSrc = nextLabel('or');
        const choicesSrc = choices.map(peval).map(cSrc =>
`${cSrc}
if (value !== FAIL) break ${labelSrc};`).join('\n');

        return (
`${labelSrc}: {
  ${indent(choicesSrc,`
  `)}
}`);
      },
      seq(...terms) {
        const posSrc = nextVar('pos');
        const labelSrc = nextLabel('seq');
        const sSrc = nextVar('s');
        const vSrc = nextVar('v');
        const termsSrc = terms.map(peval).map(termSrc =>
`${termSrc}
if (value === FAIL) break ${labelSrc};
${sSrc}.push(value);`).join('\n');

        return (
`${sSrc} = [];
${vSrc} = FAIL;
${posSrc} = pos;
${labelSrc}: {
  ${indent(termsSrc,`
  `)}
  ${vSrc} = ${sSrc};
}
if ((value = ${vSrc}) === FAIL) pos = ${posSrc};`);
      },
      act(terms, hole) {
        numSubs = Math.max(numSubs, hole + 1);
        const termsSrc = vtable.seq(...terms);
        return (
`${termsSrc}
if (value !== FAIL) value = act_${hole}(...value);`);
      },
      '**'(patt, sep) {
        // for backtracking
        const posSrc = nextVar('pos');
        // a non-advancing success only repeats once.
        const startSrc = nextVar('pos');
        const sSrc = nextVar('s');
        const pattSrc = peval(patt);
        const sepSrc = peval(sep);
        return (
// after first iteration, backtrack to before the separator
`${sSrc} = [];
${posSrc} = pos;
while (true) {
  ${startSrc} = pos;
  ${indent(pattSrc,`
  `)}
  if (value === FAIL) {
    pos = ${posSrc};
    break;
  }
  ${sSrc}.push(value);
  ${posSrc} = pos;
  ${indent(sepSrc,`
  `)}
  if (value === FAIL) break;
  if (pos === ${startSrc}) break;
}
value = ${sSrc};`);
      },
      '++'(patt, sep) {
        const starSrc = vtable['**'](patt, sep);
        return (
`${starSrc}
if (value.length === 0) value = FAIL;`);
      },
      '?'(patt) {
        return vtable['**'](patt, ['fail']);
      },
      '*'(patt) {
        return vtable['**'](patt, ['empty']);
      },
      '+'(patt) {
        return vtable['++'](patt, ['empty']);
      },
      'super'(ident) {
        return `[pos, value] = this.run(super.rule_${ident}, pos, ${
            JSON.stringify(ident)});`;
      },
      apply(hole) {
        numSubs = Math.max(numSubs, hole + 1);
        return `[pos, value] = act_${hole}.call(this, pos);`;
      }
    });

    function peval(sexp) {
      if (typeof sexp === 'string') {
        // If sexp is already a string literal, this will doubly quote it,
        // which is what we want.
        const nameStr = JSON.stringify(sexp);
        if (sc.allRE(sc.STRING_RE).test(sexp)) {
          tokenTypes.add(sexp);
          return `[pos, value] = this.run(${sexp}, pos, ${nameStr});`;
        }
        if (sc.allRE(sc.IDENT_RE).test(sexp)) {
          // Assume a standalone identifier is a rule name.
          return `[pos, value] = this.run(this.rule_${sexp}, pos, ${nameStr});`;
        }
        throw new Error('unexpected: ' + sexp);
      }
      return vtable[sexp[0]](...sexp.slice(1));
    }

    return peval(sexp);
  }


  function quasiMemo(quasiCurry) {
    const wm = new WeakMap();
    return function(template, ...subs) {
      let quasiRest = wm.get(template);
      if (!quasiRest) {
{'fraction_non_alphanumeric': 0.18069654849289396, 'fraction_numerical': 0.0037482430110885524, 'mean_word_length': 2.1374326310632044, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7271356', 'n_tokens_mistral': 4260, 'n_tokens_neox': 4185, 'n_words': 1055}