<commit_before><commit_msg>Implement parser. Probably won't work yet<commit_after>import ast
import token

LOWEST      = 0  #
ASSIGN      = 1  # =
OR          = 2  # ||
AND         = 3  # &&
BIT_OR      = 4  # |
BIT_AND     = 5  # &
EQUALS      = 6  # == or !==
LESSGREATER = 7  # < or >
SUM         = 8  # + or -
PRODUCT     = 9  # * or /
PREFIX      = 10 # -x

precedences = {
    token.EQ:     EQUALS,
    token.N_EQ:   EQUALS,
    token.LT:     LESSGREATER,
    token.GT:     LESSGREATER,
    token.PLUS:   SUM,
    token.MINUS:  SUM,
    token.STAR:   PRODUCT,
    token.SLASH:  PRODUCT,
    token.ASSIGN: ASSIGN,
    token.OR:     OR,
    token.AND:    AND,
    token.B_OR:   BIT_OR,
    token.B_AND:  BIT_AND
}

class Parser(object):
    """parses a stream of tokens into an abstract syntax tree (AST)"""
    def __init__(self, tokens):
        self.tokens   = tokens
        self.errors  = []
        self.cur_tok  = None
        self.peek_tok = None
        
        self.prefixes = {
            # Literals
            token.ID:      self.parse_id,
            token.NUM:     self.parse_num,
            token.TRUE:    self.parse_bool,
            token.FALSE:   self.parse_bool,
            token.NULL:    self.parse_null,
            token.LSQUARE: self.parse_array,
            token.STRING:  self.parse_string,
            
            # Prefix operators
            token.MINUS: self.parse_prefix,
            token.PLUS:  self.parse_prefix,
            
            # Constructs
            token.LPAREN: self.parse_grouped_expr
        }
        
        self.infixes  = {
            token.PLUS:   self.parse_infix,
            token.MINUS:  self.parse_infix,
            token.STAR:   self.parse_infix,
            token.SLASH:  self.parse_infix,
            token.EQ:     self.parse_infix,
            token.N_EQ:   self.parse_infix,
            token.LT:     self.parse_infix,
            token.GT:     self.parse_infix,
            token.OR:     self.parse_infix,
            token.AND:    self.parse_infix,
            token.B_OR:   self.parse_infix,
            token.B_AND:  self.parse_infix,
            token.ASSIGN: self.parse_assign_expr
        }
        
        self.next()
        self.next()
        
    def peek_precedence(self):
        return precedences.get(self.peek_tok.type, LOWEST)
        
    def cur_precedence(self):
        return precedences.get(self.cur_tok.type, LOWEST)
        
    def peek_err(self, t):
        msg = "expected next token to be %s, but got %s" % (t, self.peek_tok.type)
        self.errors.append(msg)
        
    def no_prefix_fn_error(self, t):
        msg = "unexpected token: %s" % t
        self.errors.append(msg)
        
    def next(self):
        self.cur_tok = self.peek_tok
        self.peek_tok = next(self.tokens)
        
    def parse_program(self):
        program = ast.Program()
        
        while not self.cur_tok_is(token.EOF):
            stmt = self.parse_stmt()
            
            if stmt != None:
                program.statements.append(stmt)
                
            self.next()
            
        return program
        
    def parse_statement(self):
        stmt = None
        
        if self.cur_tok_is(token.SEMI):
            return None
        elif self.cur_tok_is(token.RETURN):
            stmt = self.parse_return_stmt()
        else:
            stmt = self.parse_expr_stmt()
            
        if not self.expect(token.SEMI):
            return None
            
        return stmt
        
    def parse_block_statement(self):
        block = ast.BlockStatement(self.cur_tok)
        
        self.next()
        
        while not self.cur_tok_is(token.RBRACE) and not self.cur_tok_is(token.EOF):
            stmt = self.parse_statement()
            
            if stmt != None:
                block.statements.append(stmt)
                
            self.next()
            
        return block
        
    def parse_expr_stmt(self):
        return ast.ExpressionStatement(self.cur_tok, self.parse_expr(LOWEST))
        
    def parse_return_stmt(self):
        if self.peek_tok.type == token.SEMI:
            return ast.ReturnStatement(self.cur_tok, None)
            
        stmt = ast.ReturnStatement(self.cur_tok, None)
        self.next()
        stmt.value = self.parse_expr(LOWEST)
        
        return stmt
        
    def parse_expr(self, precedence):
        prefix = self.prefixes.get(self.cur_tok.type, None)
        
        if prefix == None:
            self.no_prefix_fn_error(self.cur_tok.type)
            return None
            
        left = prefix()
        
        while not self.peek_is(token.SEMI) and precedence < self.peek_precedence():
            infix = self.infixes.get(self.peek_tok.type, None)
            
            if infix == None:
                return left
                
            self.next()
            
            left = infix(left)
            
        return left
        
    def parse_id(self):
        return ast.Identifier(self.cur_tok)
        
    def parse_num(self):
        lit = ast.Number(self.cur_tok)
        
        try:
            lit.value = float(self.cur_tok.literal)
        except ValueError:
            msg = "could not parse %s as a number" % self.cur_tok.literal
            self.errors.append(msg)
            return None
            
        return lit
        
    def parse_bool(self):
        return ast.Boolean(self.cur_tok, self.cur_tok.type == token.TRUE)
        
    def parse_null(self):
        return ast.Null(self.cur_tok)
        
    def parse_string(self):
        return ast.String(self.cur_tok)
        
    def parse_prefix(self):
        expr = ast.PrefixExpression(self.cur_tok, self.cur_tok.literal, None)
        
        self.next()
        
        expr.right = self.parse_expr(PREFIX)
        
        return expr
        
    def parse_grouped_expr(self):
        self.next()
        
        expr = self.parse_expr(LOWEST)
        
        if not self.expect(token.RPAREN):
            return None
            
        return expr
        
    def parse_array(self):
        return ast.Array(self.cur_tok, self.parse_expr_list(token.RSQUARE))
        
    def parse_infix(self, left):
        expr = ast.InfixExpression(self.cur_tok, self.cur_tok.literal, left, None)
        
        precedence = self.cur_precedence()
        self.next()
        expr.right = self.parse_expr(precedence)
        
        return expr
        
    def parse_assign_expr(self, left):
        expr = ast.AssignExpression(self.cur_precedence, left, None)
        
        self.next()
        expr.value = self.parse_expr(LOWEST)
        
        return expr
        
    def parse_expr_list(end):
        exprs = []

        if self.peek_tok_is(end):
            self.next()
            return exprs
            
        self.next()
        list.append(self.parse_expr(LOWEST))
        
        while self.peek_tok_is(token.COMMA):
            self.next()
            self.next()
            exprs.append(self.parse_expr(LOWEST))
            
        if not self.expect(end):
            return None
            
        return exprs
        
    def cur_tok_is(t):
        return self.cur_tok.type == t
        
    def peek_tok_is(t):
        return self.peek_tok.type == t
        
    def expect(t):
        if self.peek_tok_is(t):
            self.next()
            return True
        else:
            self.peek_err(t)
            return False
{'fraction_non_alphanumeric': 0.09145527369826435, 'fraction_numerical': 0.0016021361815754338, 'mean_word_length': 1.39712, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 4, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1907118', 'n_tokens_mistral': 2352, 'n_tokens_neox': 2296, 'n_words': 432}