<commit_before>module FunctionParser
  class Parser

    def initialize(config)
      @config = config
    end

    def parse(source)
      lexer = Lexer.new(source)
      grammars = @config.grammars
      pt = ParseTree.new
      while token = lexer.advance
        grammar = grammars.find{ |g| g.tokens.include?(token) }
        if grammar
          grammar.parse(token, lexer, pt)
        else
          raise UnexpectedToken, %{
            The chosen configuration of FunctionParser
            does not support #{token.class}
            (see position #{token.seek})
          }.squish
        end
      end
      raise ParenthesesMismatch, %{
        Expression has an unclosed parentheses!
      }.squish unless pt.closed?
      return pt
    rescue RubyLex::SyntaxError => se
      raise ParseError, se.message
    end
<commit_msg>should be looking for the token's class<commit_after>module FunctionParser
  class Parser

    def initialize(config)
      @config = config
    end

    def parse(source)
      lexer = Lexer.new(source)
      grammars = @config.grammars
      pt = ParseTree.new
      while token = lexer.advance
        grammar = grammars.find{ |g| g.tokens.include?(token.class) }
        if grammar
          grammar.parse(token, lexer, pt)
        else
          raise UnexpectedToken, %{
            The chosen configuration of FunctionParser
            does not support #{token.class}
            (see position #{token.seek})
          }.squish
        end
      end
      raise ParenthesesMismatch, %{
        Expression has an unclosed parentheses!
      }.squish unless pt.closed?
      return pt
    rescue RubyLex::SyntaxError => se
      raise ParseError, se.message
    end

{'fraction_non_alphanumeric': 0.07710280373831775, 'fraction_numerical': 0.0, 'mean_word_length': 2.015845070422535, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2859447', 'n_tokens_mistral': 499, 'n_tokens_neox': 469, 'n_words': 138}