<commit_before># calc_lexer.py  17/01/2016  D.J.Whale

# token list
tokens = (
    'NAME','NUMBER', 'EQUALS', 'LPAREN', 'RPAREN', 'PLUS', 'MINUS', 'TIMES', 'DIVIDE'
    )

# yply generator generates invalid code if you use ')' at the end of a line
#literals = ['=','+','-','*','/', '(',')']
#TODO: see note in pcode_lexer, probably just fix and test this here
#then propagate fix to pcode_lexer

# token definitions

t_NAME    = r'[a-zA-Z_][a-zA-Z0-9_]*'

def t_NUMBER(t):
    r'\d+'
    t.value = int(t.value)
    return t

t_EQUALS    = r'='
t_LPAREN    = r'\('
t_RPAREN    = r'\)'
t_PLUS      = r'\+'
t_MINUS     = r'\-'
t_TIMES     = r'\*'
t_DIVIDE    = r'/'

t_ignore = " \t"

def t_newline(t):
    r'\n+'
    t.lexer.lineno += t.value.count("\n")


# END
<commit_msg>Attempt to use literals. Doesn't seem to work (it generates a runtime error). Literals are limited to a single character anyway.<commit_after># calc_lexer.py  17/01/2016  D.J.Whale

# token list
tokens = (
    'NAME','NUMBER', 'EQUALS', 'LPAREN', 'RPAREN', 'PLUS', 'MINUS', 'TIMES', 'DIVIDE'
    )

# token definitions

t_EQUALS    = r'='
t_LPAREN    = r'\('
t_RPAREN    = r'\)'
t_PLUS      = r'\+'
t_MINUS     = r'\-'
t_TIMES     = r'\*'
t_DIVIDE    = r'/'

t_NAME    = r'[a-zA-Z_][a-zA-Z0-9_]*'

def t_NUMBER(t):
    r'\d+'
    t.value = int(t.value)
    return t

t_ignore = " \t"

def t_newline(t):
    r'\n+'
    t.lexer.lineno += t.value.count("\n")


# END

{'fraction_non_alphanumeric': 0.19972164231036882, 'fraction_numerical': 0.013917884481558803, 'mean_word_length': 2.7447916666666665, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '719876', 'n_tokens_mistral': 654, 'n_tokens_neox': 637, 'n_words': 148}