<commit_before># Example usage of the Parser. The test defines simple calculator
# language MATABC and demonstrates how programs written in that
# language are parsed.
#
# Copyright, 2014, <NAME> (<EMAIL>)
#

import sys
sys.path.append("..")

from pylangparser import *

# define all tokens in the language
IF = Keyword(r'if')

KEYWORDS = IF

PLUS = Operator(r'+')
MINUS = Operator(r'-')
ASSIGNMENT = Operator(r'=')
SEMICOLON = Operator(r';')
EQ = Operator(r'==')
LE = Operator(r'<')
GT = Operator(r'>')
LPAR = Operator(r'(')
RPAR = Operator(r')')

# order is important as first operator that matches will be considered
# so it is important that '<=' is taken before '<'
OPERATORS = EQ & PLUS & MINUS & ASSIGNMENT & LE & GT & SEMICOLON & LPAR & RPAR

IGNORE_CHARS = Ignore(r'[ \t\v\f\n]+')

COMMENTS = Ignore(r'\#.*\n')

IDENTIFIER = Symbols(r'[A-Za-z_]+[A-Za-z0-9_]*')

CONSTANT = Symbols(r'[0-9]+')

TOKENS = KEYWORDS & OPERATORS & CONSTANT & IDENTIFIER & COMMENTS & IGNORE_CHARS

IgnoreTokensInAST(SEMICOLON & LPAR & RPAR)

# define our grammar

arthm_operator = \
    OperatorParser(PLUS) | \
    OperatorParser(MINUS)

comp_operator = \
    OperatorParser(LE) | \
    OperatorParser(GT) | \
    OperatorParser(EQ)

operand = \
    SymbolsParser(CONSTANT) | \
    SymbolsParser(IDENTIFIER)

arthm_expression = \
    SymbolsParser(IDENTIFIER) & \
    OperatorParser(ASSIGNMENT) & \
    operand & \
    Optional(arthm_operator & operand) & \
    OperatorParser(SEMICOLON)

condition = \
    operand & \
    comp_operator & \
    operand

statement = RecursiveParser()

if_statement = \
    KeywordParser(IF) & \
    OperatorParser(LPAR) & \
    condition & \
    OperatorParser(RPAR) & \
    statement

# not the usage of the '+=' operator below
statement += \
    if_statement | arthm_expression

program = AllTokensConsumed(ZeroOrMore(statement))

# our source code
source = """

# example program written in ABCMATH

p = 12;

if (p == 12)
  if (p == 5)
    p = 3 + 2;

"""

# obtain list of tokens present in the source
lexer = Lexer(TOKENS)
tokens = lexer.parseTokens(source)
print(tokens)

# build AST
result = program(tokens, 0)
result.pretty_print()
<commit_msg>simple_calc_language example: improve grammar<commit_after># Example usage of the Parser. The test defines simple calculator
# language MATABC and demonstrates how programs written in that
# language are parsed.
#
# Copyright, 2014, <NAME> (otonchev at gmail com)
#

import sys
sys.path.append("..")

from pylangparser import *

# define all tokens in the language
IF = Keyword(r'if')

KEYWORDS = IF

PLUS = Operator(r'+')
MINUS = Operator(r'-')
ASSIGNMENT = Operator(r'=')
SEMICOLON = Operator(r';')
EQ = Operator(r'==')
LE = Operator(r'<')
GT = Operator(r'>')
LPAR = Operator(r'(')
RPAR = Operator(r')')

# order is important as first operator that matches will be considered
# so it is important that '<=' is taken before '<'
OPERATORS = EQ & PLUS & MINUS & ASSIGNMENT & LE & GT & SEMICOLON & LPAR & RPAR

IGNORE_CHARS = Ignore(r'[ \t\v\f\n]+')

COMMENTS = Ignore(r'\#.*\n')

IDENTIFIER = Symbols(r'[A-Za-z_]+[A-Za-z0-9_]*')

CONSTANT = Symbols(r'[0-9]+')

TOKENS = KEYWORDS & OPERATORS & CONSTANT & IDENTIFIER & COMMENTS & IGNORE_CHARS

IgnoreTokensInAST(SEMICOLON & LPAR & RPAR)

# define our grammar

arthm_operator = \
    OperatorParser(PLUS) | \
    OperatorParser(MINUS)

comp_operator = \
    OperatorParser(LE) | \
    OperatorParser(GT) | \
    OperatorParser(EQ)

operand = \
    SymbolsParser(CONSTANT) | \
    SymbolsParser(IDENTIFIER)

arthm_expression = \
    SymbolsParser(IDENTIFIER) & \
    OperatorParser(ASSIGNMENT) & \
    (operand << Optional(arthm_operator << operand)) & \
    OperatorParser(SEMICOLON)

condition = \
    operand << \
    comp_operator << \
    operand

statement = RecursiveParser()

if_statement = \
    KeywordParser(IF) & \
    OperatorParser(LPAR) & \
    condition & \
    OperatorParser(RPAR) & \
    statement

# not the usage of the '+=' operator below
statement += \
    if_statement | arthm_expression

program = AllTokensConsumed(ZeroOrMore(statement))

# our source code
source = """

# example program written in ABCMATH

p = 12;

if (p == 12)
  if (p == 5)
    p = 3 + 2;

"""

# obtain list of tokens present in the source
lexer = Lexer(TOKENS)
tokens = lexer.parseTokens(source)
print(tokens)

# build AST
result = program(tokens, 0)
result.pretty_print()

{'fraction_non_alphanumeric': 0.1340018231540565, 'fraction_numerical': 0.007292616226071103, 'mean_word_length': 3.7551462621885157, 'pattern_counts': {'":': 0, '<': 20, '<?xml version=': 0, '>': 8, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4916438', 'n_tokens_mistral': 1645, 'n_tokens_neox': 1546, 'n_words': 390}