<commit_before>require 'radish/token'

module Radish
  class Parser
    END_TOKEN_TYPE = :"(end)"
    
    def self.inherited(klass)
      klass.deftoken END_TOKEN_TYPE, 0
    end
    
    # TODO: should use class_inheritable_accessor
    def self.symbol_table
      @symbol_table ||= {}
    end
    
    def self.deftoken(type, lbp, &blk)
      tok_module = Module.new do
        extend TokenClassMethods
        include TokenInstanceMethods
        define_method(:type) do
          type
        end
        define_method(:lbp) do
          lbp
        end
      end
      tok_module.module_eval(&blk) if block_given?
      symbol_table[type] = tok_module
    end
    
    attr_reader :lexer
    
    def initialize(source_lexer)
      @lexer = source_lexer
    end
    
    def symbol_table
      self.class.symbol_table
    end
    
    def parse
      # TODO: use returning
      s = expression
      advance_if_looking_at END_TOKEN_TYPE
      s
    end
    
    # We use the names 'prefix' and 'infix' instead of Pratt's 'nud' and 'led'
    # (following suggestion from <NAME> here: http://eli.thegreenplace.net/2010/01/02/top-down-operator-precedence-parsing/#comment-247017)
    
    def expression(rbp=0)
      start_expression = take_token.prefix
      extend_with_infixes(rbp, start_expression)
    end
    
    def advance_if_looking_at(type)
      raise next_token, "Expected '#{type}'" if next_token.type != type
      take_token
    end

    protected
    
    attr_writer :next_token
    
    def extend_with_infixes(rbp, sub_expression)
      loop do
        return sub_expression if rbp >= next_token.lbp
        sub_expression = take_token.infix(sub_expression)
      end
    end
    
    def next_token
      @next_token ||= symbolize(lexer.take_token || LexerToken.new(END_TOKEN_TYPE, ''))
    end
    
    def take_token<commit_msg>Small readability change to #extend_with_infixes<commit_after>require 'radish/token'

module Radish
  class Parser
    END_TOKEN_TYPE = :"(end)"
    
    def self.inherited(klass)
      klass.deftoken END_TOKEN_TYPE, 0
    end
    
    # TODO: should use class_inheritable_accessor
    def self.symbol_table
      @symbol_table ||= {}
    end
    
    def self.deftoken(type, lbp, &blk)
      tok_module = Module.new do
        extend TokenClassMethods
        include TokenInstanceMethods
        define_method(:type) do
          type
        end
        define_method(:lbp) do
          lbp
        end
      end
      tok_module.module_eval(&blk) if block_given?
      symbol_table[type] = tok_module
    end
    
    attr_reader :lexer
    
    def initialize(source_lexer)
      @lexer = source_lexer
    end
    
    def symbol_table
      self.class.symbol_table
    end
    
    def parse
      # TODO: use returning
      s = expression
      advance_if_looking_at END_TOKEN_TYPE
      s
    end
    
    # We use the names 'prefix' and 'infix' instead of Pratt's 'nud' and 'led'
    # (following suggestion from <NAME> here: http://eli.thegreenplace.net/2010/01/02/top-down-operator-precedence-parsing/#comment-247017)
    
    def expression(rbp=0)
      start_expression = take_token.prefix
      extend_with_infixes(rbp, start_expression)
    end
    
    def advance_if_looking_at(type)
      raise next_token, "Expected '#{type}'" if next_token.type != type
      take_token
    end

    protected
    
    attr_writer :next_token
    
    def extend_with_infixes(rbp, sub_expression)
      sub_expression = take_token.infix(sub_expression) while next_token.lbp > rbp
      sub_expression
    end
    
    def next_token
      @next_token ||= symbolize(lexer.take_token || LexerToken.new(END_TOKEN_TYPE, ''))
    end
    
    def take_token
{'fraction_non_alphanumeric': 0.0738544474393531, 'fraction_numerical': 0.00862533692722372, 'mean_word_length': 2.3462578899909827, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4471347', 'n_tokens_mistral': 1225, 'n_tokens_neox': 1206, 'n_words': 263}