<commit_before>//
// ### function AbstractParser (source)
// #### @source {Array|String} source code
// Abstract Parser constructor
//
function AbstractParser(source) {
  // Allocate rule cache for left recursion and overall speedup
  this._cache = new Array(source.length);

  // Store source and current position
  this.source = source;
  this.offset = 0;

  // Remember type of stream (string, array or simulate)
  this.type = this._type(source);

  // Allocate history for traversing through nested lists
  this.history = [];

  // Prepary property for future usage
  this.intermediate = undefined;
  this.result = undefined;

  // Some state info
  this.simulated = false;
};
module.exports = AbstractParser;

//
// ### function _type (source)
// #### @source {any} Input source
// Returns type of source
//
AbstractParser.prototype._type = function _type(source) {
  if (Array.isArray(source)) return 'array';
  if (typeof source === 'string') return 'string';
  if (source instanceof Simulate) return 'simulate';

  throw new Error('Non-sequencable source! (source: ' + source + ' )');
};

//
// ### function _save ()
// Saves parser's state
//
AbstractParser.prototype._save = function _save() {
  return {
    source: this.source,
    offset: this.offset,
    type: this.type,
    history: this.history.length > 0 ? this.history.slice() : [],
    intermediate: this.intermediate,
    result: this.result,
    simulated: this.simulated
  };
};

//
// ### function _load (state)
// #### @state {Object} state
// #### @values {Boolean} should load intermediate and result?
// Loads parser's state
//
AbstractParser.prototype._load = function _load(state, values) {
  this.source = state.source;
  this.offset = state.offset;
  this.type = state.type;
  this.history = state.history;
  if (values) {
    this.intermediate = state.intermediate;
    this.result = state.result;
  }
  this.simulated = state.simulated;
};

//
// ### function cache (grammar, rule, body)
// #### @grammar {String} grammar name
// #### @rule {String} rule name
// #### @body {Function} rule's body
// Caches rule results and allows left recursion
//
AbstractParser.prototype.cache = function cache(grammar, rule, body) {
  // Simulates can't be cached
  // And left recursion isn't supported here too
  if (this.simulated || grammar === 'AbstractGrammar') return body.call(this);

  var key = this.history.length > 0 ? this.history.concat(
        grammar,
        rule
      ).join(':') : grammar + ':' + rule,
      // Fast offset level
      cacheLevel = this._cache[this.offset] || (this._cache[this.offset] = {}),
      cache,
      res;

  // Slower history level
  cache = cacheLevel[key];


  if (cache) {
    // Indicate that left recursion was met
    if (cache.lr) {
      cache.detected = true;
    }

    // If result is positive - move position to cached one
    if (res = cache.result) this._load(cache.state, true);
  } else {
    cacheLevel[key] = cache = {
      lr: true,
      detected: false,
      result: false,
      state: this._save()
    };

    res = body.call(this);

    cacheLevel[key] = {
      lr: false,
      result: res,
      state: this._save()
    };

    // Left recursion detected
    if (res && cache.detected) {
      var source = this.source,
          offset = this.offset;

      do {
        // Return to previous position and start seeding
        this._load(cache.state);

        res = body.call(this);

        if (source === this.source && offset === this.offset) res = false;

        if (res) {
          cacheLevel[key] = {
            lr: false,
            result: res,
            state: this._save()
          }
        }
      } while (res);

      var state = cacheLevel[key].state;
      res = true;

      this._load(state, true);
    }

  }

  return res;
};

//<commit_msg>[core/parser] minimize GC time<commit_after>//
// ### function AbstractParser (source)
// #### @source {Array|String} source code
// Abstract Parser constructor
//
function AbstractParser(source) {
  // Allocate rule cache for left recursion and overall speedup
  this._cache = new Array(source.length);

  // Store source and current position
  this.source = source;
  this.offset = 0;

  // Remember type of stream (string, array or simulate)
  this.type = this._type(source);

  // Allocate history for traversing through nested lists
  this.history = [];

  // Prepary property for future usage
  this.intermediate = undefined;
  this.result = undefined;

  // Some state info
  this.simulated = false;
};
module.exports = AbstractParser;

//
// ### function _type (source)
// #### @source {any} Input source
// Returns type of source
//
AbstractParser.prototype._type = function _type(source) {
  if (Array.isArray(source)) return 'array';
  if (typeof source === 'string') return 'string';
  if (source instanceof Simulate) return 'simulate';

  throw new Error('Non-sequencable source! (source: ' + source + ' )');
};

//
// ### function _save ()
// Saves parser's state
//
AbstractParser.prototype._save = function _save() {
  return {
    source: this.source,
    offset: this.offset,
    type: this.type,
    history: this.history.length > 0 ? this.history.slice() : [],
    intermediate: this.intermediate,
    result: this.result,
    simulated: this.simulated
  };
};

//
// ### function _load (state)
// #### @state {Object} state
// #### @values {Boolean} should load intermediate and result?
// Loads parser's state
//
AbstractParser.prototype._load = function _load(state, values) {
  this.source = state.source;
  this.offset = state.offset;
  this.type = state.type;
  this.history = state.history;
  if (values) {
    this.intermediate = state.intermediate;
    this.result = state.result;
  }
  this.simulated = state.simulated;
};

//
// ### function cache (grammar, rule, body)
// #### @grammar {String} grammar name
// #### @rule {String} rule name
// #### @body {Function} rule's body
// Caches rule results and allows left recursion
//
AbstractParser.prototype.cache = function cache(grammar, rule, body) {
  // Simulates can't be cached
  // And left recursion isn't supported here too
  if (this.simulated || grammar === 'AbstractGrammar') return body.call(this);

  var key = this.history.length > 0 ? this.history.concat(
        grammar,
        rule
      ).join(':') : grammar + ':' + rule,
      // Fast offset level
      cacheLevel = this._cache[this.offset] || (this._cache[this.offset] = {}),
      cache,
      res;

  // Slower history level
  cache = cacheLevel[key];

  if (cache) {
    // Indicate that left recursion was met
    if (cache.lr) {
      cache.detected = true;
    }

    // If result is positive - move position to cached one
    if (res = cache.result) this._load(cache.state, true);
  } else {
    var state = this._save();

    cacheLevel[key] = cache = {
      lr: true,
      detected: false,
      result: false,
      state: state
    };

    res = body.call(this);

    cache.lr = false;
    cache.result = res;
    cache.state = this._save();

    // Left recursion detected
    if (res && cache.detected) {
      var source = this.source,
          offset = this.offset;

      do {
        // Return to previous position and start seeding
        this._load(state);

        res = body.call(this);

        if (source === this.source && offset === this.offset) res = false;

        if (res) {
          cache.result = res;
          cache.state = this._save();
        }
      } while (res);

      res = true;
      this._load(cache.state, true);
    }

  }

  return res;
};

//
{'fraction_non_alphanumeric': 0.14264958393871352, 'fraction_numerical': 0.0007924976885484084, 'mean_word_length': 2.6212338593974174, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1652226', 'n_tokens_mistral': 2315, 'n_tokens_neox': 2260, 'n_words': 657}