<commit_before>// --------------------------------------------------------------------
// Imports
// --------------------------------------------------------------------

var common = require('./common.js');
var pexprs = require('./pexprs.js');
var PosInfo = require('./PosInfo.js');

// --------------------------------------------------------------------
// Private stuff
// --------------------------------------------------------------------

function State(grammar, inputStream) {
  this.grammar = grammar;
  this.inputStreamStack = [];
  this.posInfosStack = [];
  this.pushInputStream(inputStream);
  this.ruleStack = [];
  this.bindings = [];
  this.failures = [];
  this.failuresPos = -1;
}

State.prototype = {
  pushInputStream: function(inputStream) {
    this.inputStreamStack.push(this.inputStream);
    this.posInfosStack.push(this.posInfos);
    this.inputStream = inputStream;
    this.posInfos = [];
  },

  popInputStream: function() {
    this.inputStream = this.inputStreamStack.pop();
    this.posInfos = this.posInfosStack.pop();
  },

  getCurrentPosInfo: function() {
    return this.getPosInfo(this.inputStream.pos);
  },

  getPosInfo: function(pos) {
    var posInfo = this.posInfos[pos];
    return posInfo || (this.posInfos[pos] = new PosInfo(this.ruleStack));
  },

  recordFailure: function(pos, expr) {
    if (pos < this.failuresPos) {
      return;
    } else if (pos > this.failuresPos) {
      this.failures = {};
      this.failuresPos = pos;
    }

    var fs = this.failures;
    for (var idx = 0; idx < this.ruleStack.length; idx++) {
      var ruleName = this.ruleStack[idx];
      if (idx === this.ruleStack.length - 1) {
        if (Array.isArray(fs[ruleName])) {
          fs[ruleName].push(expr);
        } else {
          // This failure subsumes others whose paths have this failure's path as a prefix.
          fs[ruleName] = [expr];
        }
      } else if (Array.isArray(fs[ruleName])) {
        // Don't record this failure, it's subsumed by another failure
        // whose path is a prefix of this failure's path.
        break;
      } else {
        if (fs[ruleName] === undefined) {<commit_msg>added important TODO about problems w/ the current failure-recording scheme<commit_after>// --------------------------------------------------------------------
// Imports
// --------------------------------------------------------------------

var common = require('./common.js');
var pexprs = require('./pexprs.js');
var PosInfo = require('./PosInfo.js');

// --------------------------------------------------------------------
// Private stuff
// --------------------------------------------------------------------

function State(grammar, inputStream) {
  this.grammar = grammar;
  this.inputStreamStack = [];
  this.posInfosStack = [];
  this.pushInputStream(inputStream);
  this.ruleStack = [];
  this.bindings = [];
  this.failures = [];
  this.failuresPos = -1;
}

State.prototype = {
  pushInputStream: function(inputStream) {
    this.inputStreamStack.push(this.inputStream);
    this.posInfosStack.push(this.posInfos);
    this.inputStream = inputStream;
    this.posInfos = [];
  },

  popInputStream: function() {
    this.inputStream = this.inputStreamStack.pop();
    this.posInfos = this.posInfosStack.pop();
  },

  getCurrentPosInfo: function() {
    return this.getPosInfo(this.inputStream.pos);
  },

  getPosInfo: function(pos) {
    var posInfo = this.posInfos[pos];
    return posInfo || (this.posInfos[pos] = new PosInfo(this.ruleStack));
  },

  recordFailure: function(pos, expr) {
    if (pos < this.failuresPos) {
      return;
    } else if (pos > this.failuresPos) {
      this.failures = {};
      this.failuresPos = pos;
    }

    var fs = this.failures;
    for (var idx = 0; idx < this.ruleStack.length; idx++) {
      var ruleName = this.ruleStack[idx];
      if (idx === this.ruleStack.length - 1) {
        if (Array.isArray(fs[ruleName])) {
          fs[ruleName].push(expr);
        } else {
          // This failure subsumes others whose paths have this failure's path as a prefix.
          fs[ruleName] = [expr];
        }
      } else if (Array.isArray(fs[ruleName])) {
        // Don't record this failure, it's subsumed by another failure
        // whose path is a prefix of this failure's path.
        break;

        /*
          TODO: think about this some more, it's not quite right.

          The problem is that this scheme excludes failures that you would want, e.g., in

            SeqExpr
              = NoSeqExpr ';' SeqExpr  -- seq
              | NoSeqExpr

          it's not good to only complain about the missing ';', because failures from the NoSeqExpr in the second
          case would be equally relevant, since the continuation of that application is nullable.
        */
      } else {
        if (fs[ruleName] === undefined) {
{'fraction_non_alphanumeric': 0.2317826265953067, 'fraction_numerical': 0.0012350761630300535, 'mean_word_length': 1.790924755887421, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4459082', 'n_tokens_mistral': 1373, 'n_tokens_neox': 1344, 'n_words': 328}