<commit_before>// [VexFlow](http://vexflow.com) - Copyright (c) <NAME> 2010.
//
// A generic text parsing class for VexFlow.

import { Vex } from './vex';
import { Grammar } from './easyscore';
import { Rule, Match, RuleFunction, TriggerFunction, TriggerState } from './types/common';

// To enable logging for this class. Set `Vex.Flow.Parser.DEBUG` to `true`.
function L(...args: any[]) {
  if (Parser.DEBUG) Vex.L('Vex.Flow.Parser', args);
}

export const X = Vex.MakeException('ParserError');

const NO_ERROR_POS = -1;

interface Result {
  success: boolean;

  // Lexer Results
  pos?: number;
  incrementPos?: number;
  matchedString?: string;

  // Parser Results
  matches?: Match[];
  numMatches?: number;
  results?: Result[];
  errorPos?: number; // Set to NO_ERROR if successful. N if there is an error in the string.
}

// Converts parser results into an easy to reference list that can be
// used in triggers. This function returns:
// - nested array in which the leaf elements are string or null
// - string (including empty strings)
// - null
function flattenMatches(r: Result | Result[]): Match {
  if ('matchedString' in r) return r.matchedString as string; // string
  if ('results' in r) return flattenMatches(r.results as Result[]);
  const results = r as Result[];
  if (results.length === 1) return flattenMatches(results[0]);
  if (results.length === 0) return null;
  return results.map(flattenMatches); // nested array
}

// This is the base parser class. Given an arbitrary context-free grammar, it
// can parse any line and execute code when specific rules are met (e.g.,
// when a string is terminated.)
export class Parser {
  static DEBUG: boolean = false;

  // For an example of a simple grammar, take a look at tests/parser_tests.js or
  // the EasyScore grammar in easyscore.js.
  constructor(grammar) {
    this.grammar = grammar;
  }

  // Parse `line` using current grammar. Returns {success: true} if the
  // line parsed correctly, otherwise returns `{success: false, errorPos: N}`
  // where `errorPos` is the location of the error in the string.
  parse(line) {
    this.line = line;
    this.pos = 0;
    this.errorPos = -1;
    const results = this.expect(this.grammar.begin());
    results.errorPos = this.errorPos;
    return results;
  }

  matchFail(returnPos) {
    if (this.errorPos === -1) this.errorPos = this.pos;
    this.pos = returnPos;
  }

  matchSuccess() {
    this.errorPos = -1;
  }

  // Look for `token` in this.line[this.pos], and return success
  // if one is found. `token` is specified as a regular expression.
  matchToken(token, noSpace = false) {
    const regexp = noSpace ? new RegExp('^((' + token + '))') : new RegExp('^((' + token + ')\\s*)');
    const workingLine = this.line.slice(this.pos);
    const result = workingLine.match(regexp);
    if (result !== null) {
      return {
        success: true,
        matchedString: result[2],
        incrementPos: result[1].length,
        pos: this.pos,
      };
    } else {
      return {
        success: false,
        pos: this.pos,
      };
    }
  }

  // Execute rule to match a sequence of tokens (or rules). If `maybe` is
  // set, then return success even if the token is not found, but reset
  // the position before exiting.
  expectOne(rule, maybe = false) {<commit_msg>Initialize instance variables in the constructor().<commit_after>// [VexFlow](http://vexflow.com) - Copyright (c) <NAME> 2010.
//
// A generic text parsing class for VexFlow.

import { Vex } from './vex';
import { Grammar } from './easyscore';
import { Rule, Match, RuleFunction, TriggerFunction, TriggerState } from './types/common';

// To enable logging for this class. Set `Vex.Flow.Parser.DEBUG` to `true`.
function L(...args: any[]) {
  if (Parser.DEBUG) Vex.L('Vex.Flow.Parser', args);
}

export const X = Vex.MakeException('ParserError');

const NO_ERROR_POS = -1;

interface Result {
  success: boolean;

  // Lexer Results
  pos?: number;
  incrementPos?: number;
  matchedString?: string;

  // Parser Results
  matches?: Match[];
  numMatches?: number;
  results?: Result[];
  errorPos?: number; // Set to NO_ERROR if successful. N if there is an error in the string.
}

// Converts parser results into an easy to reference list that can be
// used in triggers. This function returns:
// - nested array in which the leaf elements are string or null
// - string (including empty strings)
// - null
function flattenMatches(r: Result | Result[]): Match {
  if ('matchedString' in r) return r.matchedString as string; // string
  if ('results' in r) return flattenMatches(r.results as Result[]);
  const results = r as Result[];
  if (results.length === 1) return flattenMatches(results[0]);
  if (results.length === 0) return null;
  return results.map(flattenMatches); // nested array
}

// This is the base parser class. Given an arbitrary context-free grammar, it
// can parse any line and execute code when specific rules are met (e.g.,
// when a string is terminated.)
export class Parser {
  static DEBUG: boolean = false;

  protected grammar: Grammar;

  protected line: string; // Use RegExp to extract tokens from this line.
  protected pos: number;
  protected errorPos: number;

  // For an example of a simple grammar, take a look at tests/parser_tests.js or
  // the EasyScore grammar in easyscore.ts.
  constructor(grammar: Grammar) {
    this.grammar = grammar;
    this.line = '';
    this.pos = 0;
    this.errorPos = NO_ERROR_POS;
  }

  // Parse `line` using current grammar. Returns `{success: true}` if the
  // line parsed correctly, otherwise returns `{success: false, errorPos: N}`
  // where `errorPos` is the location of the error in the string.
  parse(line: string): Result {
    this.line = line;
    this.pos = 0;
    this.errorPos = NO_ERROR_POS;
    const result = this.expect(this.grammar.begin());
    result.errorPos = this.errorPos;
    return result;
  }

  matchFail(returnPos: number): void {
    if (this.errorPos === NO_ERROR_POS) this.errorPos = this.pos;
    this.pos = returnPos;
  }

  matchSuccess(): void {
    this.errorPos = NO_ERROR_POS;
  }

  // Look for `token` in this.line[this.pos], and return success
  // if one is found. `token` is specified as a regular expression.
  matchToken(token, noSpace = false) {
    const regexp = noSpace ? new RegExp('^((' + token + '))') : new RegExp('^((' + token + ')\\s*)');
    const workingLine = this.line.slice(this.pos);
    const result = workingLine.match(regexp);
    if (result !== null) {
      return {
        success: true,
        matchedString: result[2],
        incrementPos: result[1].length,
        pos: this.pos,
      };
    } else {
      return {
        success: false,
        pos: this.pos,
      };
    }
  }

  // Execute rule to match a sequence of tokens (or rules). If `maybe` is
  // set, then return success even if the token is not found, but reset
  // the position before exiting.
  expectOne(rule, maybe = false) {
{'fraction_non_alphanumeric': 0.12868016659485854, 'fraction_numerical': 0.003734022691368663, 'mean_word_length': 3.072514619883041, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1945024', 'n_tokens_mistral': 2156, 'n_tokens_neox': 2111, 'n_words': 768}