<commit_before>import ply.lex as lex
from bsi_string import BsiString

tokens = (
    'KEY',
    'STRING',
    'NUM',
    'EQ',
    'L_SQ_BR',
    'R_SQ_BR',
    'L_BRACE',
    'R_BRACE'
)

t_KEY = r'[<KEY>'

def t_STRING(t):
    r'".*"'
    t.value = BsiString(t.value.strip('"'))
    return t

def t_NUM(t):
    r'\d+'
    t.value = int(t.value)
    return t

t_EQ = r'='

t_L_SQ_BR = r'\['
t_R_SQ_BR = r'\]'
t_L_BRACE = r'{'
t_R_BRACE = r'}'

def t_newline(t):
    r'\n+'
    t.lexer.lineno += len(t.value)

t_ignore  = ' \t'

def t_error(t):
    print "Illegal character '%s'" % t.value[0]
    t.lexer.skip(1)

bsi_lexer = lex.lex()
<commit_msg>commas are optional<commit_after>import ply.lex as lex
from bsi_string import BsiString

tokens = (
    'KEY',
    'STRING',
    'NUM',
    'EQ',
    'L_SQ_BR',
    'R_SQ_BR',
    'L_BRACE',
    'R_BRACE'
)

t_KEY = r'[<KEY>'

def t_STRING(t):
    r'".*"'
    t.value = BsiString(t.value.strip('"'))
    return t

def t_NUM(t):
    r'\d+'
    t.value = int(t.value)
    return t

t_EQ = r'='

t_L_SQ_BR = r'\['
t_R_SQ_BR = r'\]'
t_L_BRACE = r'{'
t_R_BRACE = r'}'

def t_newline(t):
    r'\n+'
    t.lexer.lineno += len(t.value)

t_ignore  = ' \t,'

def t_error(t):
    print "Illegal character '%s'" % t.value[0]
    t.lexer.skip(1)

bsi_lexer = lex.lex()

{'fraction_non_alphanumeric': 0.19693486590038314, 'fraction_numerical': 0.0030651340996168583, 'mean_word_length': 2.637883008356546, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3898988', 'n_tokens_mistral': 648, 'n_tokens_neox': 641, 'n_words': 122}