<commit_before>module Twostroke
  class ParseError < Error
  end
  
  class Parser
    attr_reader :statements
    
    def initialize(tokens)
      @i = -1
      @tokens = tokens + [Token.new(type: :SEMICOLON)]
      @statements = []
    end
    
    def parse
      while @i + 1 < @tokens.length
        statements.push statement
      end
    end
  
  private
    def error!(msg)
      raise ParseError, "Syntax error at line #{token.line}, col #{token.col}. #{msg}"
    end
    def assert_type(tok, *types)
      error! "Found #{tok.type}#{"<#{tok.val}>" if tok.val}, expected #{types.join ", "}" unless types.include? tok.type
    end
    def stack
      @stack
    end
    def stack_top
      @stack.last
    end
    def token
      @tokens[@i] or raise ParseError, "unexpected end of input"
    end
    def next_token
      @i += 1
      token
    end
    def try_peek_token
      @i + 1 < @tokens.length ? peek_token : nil
    end
    def peek_token
      @tokens[@i + 1] or raise ParseError, "unexpected end of input"
    end
  
    ####################
    
    def statement
      st = case peek_token.type
      when :RETURN; send :return
      when :VAR; var
      when :IF; send :if
      when :OPEN_BRACE; body
      when :SEMICOLON; nil
      else; expression
      end
      next_token if try_peek_token && peek_token.type == :SEMICOLON
      st
    end
    
    def expression
      expr = case peek_token.type
      when :FUNCTION; function
      when :STRING; string
      when :NUMBER; number
      when :BAREWORD; bareword
      when :OPEN_BRACE; object_literal
      when :OPEN_BRACKET; array
      else error! "Unexpected #{peek_token.type}"
      end
      if [:PLUS, :MINUS, :ASTERISK, :SLASH, :GT, :LT, :GTE, :LTE].include? peek_token.type
        op = next_token.type
        AST::UnsortedBinop.new left: expr, op: op, right: expression
      else
        expr
      end
    end
    
    def body
      assert_type next_token, :OPEN_BRACE
      body = AST::Body.new
      while peek_token.type != :CLOSE_BRACE
        body.statements.push statement
      end
      assert_type next_token, :CLOSE_BRACE<commit_msg>more operators can be used to create an unsorted binop<commit_after>module Twostroke
  class ParseError < Error
  end
  
  class Parser
    attr_reader :statements
    
    def initialize(tokens)
      @i = -1
      @tokens = tokens + [Token.new(type: :SEMICOLON)]
      @statements = []
    end
    
    def parse
      while @i + 1 < @tokens.length
        statements.push statement
      end
    end
  
  private
    def error!(msg)
      raise ParseError, "Syntax error at line #{token.line}, col #{token.col}. #{msg}"
    end
    def assert_type(tok, *types)
      error! "Found #{tok.type}#{"<#{tok.val}>" if tok.val}, expected #{types.join ", "}" unless types.include? tok.type
    end
    def stack
      @stack
    end
    def stack_top
      @stack.last
    end
    def token
      @tokens[@i] or raise ParseError, "unexpected end of input"
    end
    def next_token
      @i += 1
      token
    end
    def try_peek_token
      @i + 1 < @tokens.length ? peek_token : nil
    end
    def peek_token
      @tokens[@i + 1] or raise ParseError, "unexpected end of input"
    end
  
    ####################
    
    def statement
      st = case peek_token.type
      when :RETURN; send :return
      when :VAR; var
      when :IF; send :if
      when :OPEN_BRACE; body
      when :SEMICOLON; nil
      else; expression
      end
      next_token if try_peek_token && peek_token.type == :SEMICOLON
      st
    end
    
    def expression
      expr = case peek_token.type
      when :FUNCTION; function
      when :STRING; string
      when :NUMBER; number
      when :BAREWORD; bareword
      when :OPEN_BRACE; object_literal
      when :OPEN_BRACKET; array
      else error! "Unexpected #{peek_token.type}"
      end
      if [:PLUS, :MINUS, :ASTERISK, :SLASH, :GT, :LT,
          :GTE, :LTE, :DOUBLE_EQUALS, :TRIPLE_EQUALS,
          :NOT_EQUALS, :NOT_DOUBLE_EQUALS, :AND, :OR,
          :AMPERSAND, :PIPE, :TILDE, :CARET, :MOD   ].include? peek_token.type
        op = next_token.type
        AST::UnsortedBinop.new left: expr, op: op, right: expression
      else
        expr
      end
    end
    
    def body
      assert_type next_token, :OPEN_BRACE
      body = AST::Body.new
      while peek_token.type != :CLOSE_BRACE
        body.statements.push statement
      end
      assert_type next_token, :CLOSE_BRACE
{'fraction_non_alphanumeric': 0.10194500335345406, 'fraction_numerical': 0.0022356360384529397, 'mean_word_length': 2.1286713286713286, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2191051', 'n_tokens_mistral': 1485, 'n_tokens_neox': 1449, 'n_words': 424}