<commit_before>from Lexeme import Lexeme

class Node():
    Error = []
    def __init__(self, lx):
        self.lexemes = lx
        self.nextLexeme = self.lexemes.pop(0)
        self.backtrack = None
        return

    def parseError(self, err):
        self.Error.append(err)
        return

    def lookahead(self):
        self.backtrack = self.nextLexeme
        self.nextLexeme = self.lexemes.pop(0)
        return

    def parse(self):
        #if token in PTree.grammar:
        #    print("wow")
        self.codeblock()
        return

    def codeblock(self):
        #<CODEBLOCK>:= <STATEMENT>
        self.statement()

        #<CODEBLOCK>:= <STATEMENT><CODEBLOCK>
        if len(self.lexemes) != 0:
            self.lookahead()
            self.codeblock()
        return

    def statement(self):
        #<STATEMENT>:= <EXPRESSION>|<VARDEC>|<ARRAYDEC>|<ASSIGN>|<IF-THEN>|<FOR-LOOP>|<WHILE-LOOP>|<DO-WHILE-LOOP>|<FUNCTIONDEC>|<FUNCTIONCALL>|<OUTPUT>|<INPUT>
        if self.nextLexeme.label == 'new':
            self.lookahead()
            self.varDec()

        if self.operand():
            if expression():
        return

    def varDec(self):
        #<VARDEC>:= new <VARIDENT><LINE-DELIMITER>
        if self.nextLexeme.label == 'Variable Identifier':
            self.lookahead()
        else:
            self.parseError('Expected Variable Identifier at line '+str(self.backtrack.lineNumber))

        if self.nextLexeme.label == ';':
            self.lookahead()
        else:
            self.parseError('Expected \';\' at line '+str(self.backtrack.lineNumber))
        return

    def expression():
        if not self.operand()
            self.parseError('Expected operand at line '+str(self.backtrack.lineNumber))
            return False

        return True

    def operation():
        if self.nextLexeme.label in ['+', '-', '*', '/', '%']:
            self.lookahead()
            if self.operand():
        else:
            return False
        return True

    def operand():
        if self.nextLexeme.label in ['Integer Literal', 'Float Literal', 'String Literal', 'Variable Identifier']:


def parser(tokens):
    global PTree
    PTree = Node(tokens)
    PTree.parse()

    for s in PTree.Error:
        print("Syntax Error: "+s)
    return
<commit_msg>Updated Implementation for expression parsing<commit_after>from Lexeme import Lexeme

class Node():
    Error = []
    def __init__(self, lx):
        self.index = 0
        self.lexemes = lx
        self.nextLexeme = self.lexemes[0]
        self.prevLexeme = None
        return

    def parseError(self, err):
        self.Error.append(err)
        return

    def lookahead(self):
        self.index += 1
        if self.index < len(self.lexemes):
            self.prevLexeme = self.nextLexeme
            self.nextLexeme = self.lexemes[self.index]
        return

    def backtrack(self):
        self.index -= 1
        self.nextLexeme = self.lexemes[self.index]
        return

    def parse(self):
        #if token in PTree.grammar:
        #    print("wow")
        self.codeblock()
        return

    def codeblock(self):
        #<CODEBLOCK>:= <STATEMENT>
        self.statement()

        #<CODEBLOCK>:= <STATEMENT><CODEBLOCK>
        if self.index < len(self.lexemes):
            self.lookahead()
            self.codeblock()
        return

    def statement(self):
        #<STATEMENT>:= <EXPRESSION>|<VARDEC>|<ARRAYDEC>|<ASSIGN>|<IF-THEN>|<FOR-LOOP>|<WHILE-LOOP>|<DO-WHILE-LOOP>|<FUNCTIONDEC>|<FUNCTIONCALL>|<OUTPUT>|<INPUT>
        if self.nextLexeme.label == 'new':
            self.lookahead()
            self.varDec()

        #if self.operand():
            #if expression():
        return

    def varDec(self):
        #<VARDEC>:= new <VARIDENT><LINE-DELIMITER>
        if self.nextLexeme.label == 'Variable Identifier':
            self.lookahead()
        else:
            self.parseError('Expected Variable Identifier at line '+str(self.prevLexeme.lineNumber))

        if self.nextLexeme.label == ';':
            self.lookahead()
        else:
            self.parseError('Expected \';\' at line '+str(self.prevLexeme.lineNumber))
        return

    def expression():
        if not self.operand():

            return False
        if self.operation():
            True
        return True

    def operation():
        if self.operand():
            self.lookahead()
            if self.nextLexeme.label in ['+', '-', '*', '/', '%']:
                self.lookahead()
                if self.operand():
                    self.lookahead()
                    return True
                else:
                    self.backtrack()
            else:
                self.backtrack()
        return False

    def operand():
        if self.nextLexeme.label in ['Integer Literal', 'Float Literal', 'String Literal', 'Variable Identifier']:
            return True
        else:
            self.parseError('Expected operand at line '+str(self.prevLexeme.lineNumber))
        return False

def parser(tokens):
    global PTree
    PTree = Node(tokens)
    PTree.parse()

    for s in PTree.Error:
        print("Syntax Error: "+s)
    return

{'fraction_non_alphanumeric': 0.12780355761794276, 'fraction_numerical': 0.0013534416086620263, 'mean_word_length': 1.8931767337807606, 'pattern_counts': {'":': 0, '<': 47, '<?xml version=': 0, '>': 45, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3307815', 'n_tokens_mistral': 1617, 'n_tokens_neox': 1509, 'n_words': 325}