<commit_before>class SCIM; class Query; class Filter; class Parser; end; end; end; end

class SCIM::Query::Filter::Parser
  attr_accessor :rpn

  #----------------------------------------------------------------------------
  # Operator Precedence:
  Ops = {
    'pr' => 4,
    'eq' => 3,
    'co' => 3,
    'sw' => 3,
    'gt' => 3,
    'ge' => 3,
    'lt' => 3,
    'le' => 3,
    'and' => 2,
    'or' => 1,
  }
  Unary = {
    'pr' => 1,
  }

  # Tokenizing regexen:
  Paren = /[\(\)]/
  Str = /"(?:\\"|[^"])*"/
  Op = /#{Ops.keys.join'|'}/
  Word = /[\w\.]+/
  Sep = /\s?/
  NextToken = /^(#{Paren}|#{Str}|#{Op}|#{Word})#{Sep}/
  IsOperator = /^(?:#{Op})$/

  #----------------------------------------------------------------------------
  # Parse SCIM filter query into RPN stack:
  def parse input
    @input = input.clone            # Save for error msgs
    @tokens = lex input
    @rpn = parse_expr
    assert_eos
    self
  end

  def parse_expr
    ast = []
    expect_op = false
    while not eos and peek != ')'
      expect_op && assert_op || assert_not_op
      ast.push(start_group ? parse_group : pop)
      expect_op ^= true unless Unary[ast.last]
    end
    to_rpn ast
  end

  def parse_group
    pop                 # pop '(' token
    ast = parse_expr
    assert_close && pop # pop ')' token
    ast
  end

  # Split input into tokens
  def lex input
    tokens = []
    while ! input.empty? do
      input.sub! NextToken, '' \
        or fail "Can't lex input here '#{input}'"
      tokens.push $1
    end
    tokens
  end

  # Turn parsed tokens into an RPN stack<commit_msg>lex() should not destroy argument<commit_after>class SCIM; class Query; class Filter; class Parser; end; end; end; end

class SCIM::Query::Filter::Parser
  attr_accessor :rpn

  #----------------------------------------------------------------------------
  # Operator Precedence:
  Ops = {
    'pr' => 4,
    'eq' => 3,
    'co' => 3,
    'sw' => 3,
    'gt' => 3,
    'ge' => 3,
    'lt' => 3,
    'le' => 3,
    'and' => 2,
    'or' => 1,
  }
  Unary = {
    'pr' => 1,
  }

  # Tokenizing regexen:
  Paren = /[\(\)]/
  Str = /"(?:\\"|[^"])*"/
  Op = /#{Ops.keys.join'|'}/
  Word = /[\w\.]+/
  Sep = /\s?/
  NextToken = /^(#{Paren}|#{Str}|#{Op}|#{Word})#{Sep}/
  IsOperator = /^(?:#{Op})$/

  #----------------------------------------------------------------------------
  # Parse SCIM filter query into RPN stack:
  def parse input
    @input = input.clone            # Save for error msgs
    @tokens = lex input
    @rpn = parse_expr
    assert_eos
    self
  end

  def parse_expr
    ast = []
    expect_op = false
    while not eos and peek != ')'
      expect_op && assert_op || assert_not_op
      ast.push(start_group ? parse_group : pop)
      expect_op ^= true unless Unary[ast.last]
    end
    to_rpn ast
  end

  def parse_group
    pop                 # pop '(' token
    ast = parse_expr
    assert_close && pop # pop ')' token
    ast
  end

  # Split input into tokens
  def lex input
    input = input.clone
    tokens = []
    while ! input.empty? do
      input.sub! NextToken, '' \
        or fail "Can't lex input here '#{input}'"
      tokens.push $1
    end
    tokens
  end

  # Turn parsed tokens into an RPN stack
{'fraction_non_alphanumeric': 0.24290998766954378, 'fraction_numerical': 0.007398273736128237, 'mean_word_length': 1.5411119812059515, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 25, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5978502', 'n_tokens_mistral': 1153, 'n_tokens_neox': 1102, 'n_words': 303}