<commit_before>class Token:
    ELSE = 'ELSE'
    END = 'END'
    IF = 'IF'
    INPUT = 'INPUT'
    LET = 'LET'
    MACRO = 'MACRO'
    PRINT = 'PRINT'
    THEN = 'THEN'
    ASSIGN = 'ASSIGN'
    ADD = 'ADD'
    DIV = 'DIV'
    MUL = 'MUL'
    SUB = 'SUB'
    LPAR = 'LPAR'
    RPAR = 'RPAR'
    LSBRA = 'LSBRA'
    RSBRA = 'RSBRA'
    ID = 'ID'
    MACROID = 'MACROID'
    NUM = 'NUM'
    CHAR = 'CHAR'
    STR = 'STR'

    regexes = [
        (ELSE,     'ELSE'                             ),
        (END,      'END'                              ),
        (IF,       'IF'                               ),
        (INPUT,    'INPUT'                            ),
        (LET,      'LET'                              ),
        (MACRO,    'MACRO'                            ),
        (PRINT,    'PRINT'                            ),
        (THEN,     'THEN'                             ),
        (ASSIGN,   '='                                ),
        (ADD,      '[+]'                              ),
        (DIV,      '[/]'                              ),
        (MUL,      '[*]'                              ),
        (SUB,      '-'                                ),
        (LPAR,     '[(]'                              ),
        (RPAR,     '[)]'                              ),
        (LSBRA,    '\\['                              ),
        (RSBRA,    '\\]'                              ),
        (ID,       '[a-z_][a-z0-9_]*'                 ),
        (MACROID,  '@[a-z_][a-z0-9_]*'                ),
        (NUM,      '[0-9]+'                           ),
        (CHAR,     "'.'"                              ),
        (STR,      '"([^"]|\\\\")+"'                  ),
    ]
<commit_msg>Add macroparam token<commit_after>class Token:
    ELSE = 'ELSE'
    END = 'END'
    IF = 'IF'
    INPUT = 'INPUT'
    LET = 'LET'
    MACRO = 'MACRO'
    PRINT = 'PRINT'
    THEN = 'THEN'
    ASSIGN = 'ASSIGN'
    ADD = 'ADD'
    DIV = 'DIV'
    MUL = 'MUL'
    SUB = 'SUB'
    LPAR = 'LPAR'
    RPAR = 'RPAR'
    LSBRA = 'LSBRA'
    RSBRA = 'RSBRA'
    ID = 'ID'
    MACROID = 'MACROID'
    MACROPARAM = 'MACROPARAM'
    NUM = 'NUM'
    CHAR = 'CHAR'
    STR = 'STR'

    regexes = [
        (ELSE,        'ELSE'                 ),
        (END,         'END'                  ),
        (IF,          'IF'                   ),
        (INPUT,       'INPUT'                ),
        (LET,         'LET'                  ),
        (MACRO,       'MACRO'                ),
        (PRINT,       'PRINT'                ),
        (THEN,        'THEN'                 ),
        (ASSIGN,      '='                    ),
        (ADD,         '[+]'                  ),
        (DIV,         '[/]'                  ),
        (MUL,         '[*]'                  ),
        (SUB,         '-'                    ),
        (LPAR,        '[(]'                  ),
        (RPAR,        '[)]'                  ),
        (LSBRA,       '\\['                  ),
        (RSBRA,       '\\]'                  ),
        (ID,          '[a-z_][a-z0-9_]*'     ),
        (MACROID,     '@[a-z_][a-z0-9_]*'    ),
        (MACROPARAM,  '[$][a-z_][a-z0-9_]*'  ),
        (NUM,         '[0-9]+'               ),
        (CHAR,        "'.'"                  ),
        (STR,         '"([^"]|\\\\")+"'      ),
    ]


{'fraction_non_alphanumeric': 0.1675741076830006, 'fraction_numerical': 0.004234724742891712, 'mean_word_length': 0.610813443740867, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4948824', 'n_tokens_mistral': 1088, 'n_tokens_neox': 1056, 'n_words': 166}