<commit_before>
import sys
import ply.lex as lex

# Reserved
reserved = {
    # Reserved words
    'array': 'ARRAY',
    'by': 'BY',
    'chars': 'CHARS',
    'dcl': 'DCL',
    'do': 'DO',
    'down': 'DOWN',
    'else': 'ELSE',
    'elsif': 'ELSIF',
    'end': 'END',
    'exit': 'EXIT',
    'fi': 'FI',
    'for': 'FOR',
    'if': 'IF',
    'in': 'IN',
    'loc': 'LOC',
    'type': 'TYPE',
    'od': 'OD',
    'proc': 'PROC',
    'ref': 'REF',
    'result': 'RESULT',
    'return': 'RETURN',
    'returns': 'RETURNS',
    'syn': 'SYN',
    'then': 'THEN',
    'to': 'TO',
    'while': 'WHILE',

    # Predefined words
    'abs': 'ABS',
    'asc': 'ASC',
    'bool': 'BOOL',
    'char': 'CHAR',
    'false': 'FALSE',
    'int': 'INT',
    'length': 'LENGTH',
    'lower': 'LOWER',
    'null': 'NULL',
    'num': 'NUM',
    'print': 'PRINT',
    'read': 'READ',
    'true': 'TRUE',
    'upper': 'UPPER'
}

# Tokens
tokens = [
        # Identifier
        'ID',

        # Operations and Delimiters
        'PLUS', 'MINUS', 'TIMES', 'DIVIDE',
        'ASSIGN', 'COMMA', 'COLON', 'SEMI', 'ARROW',
        'LPAREN', 'RPAREN', 'LBRACKET', 'RBRACKET',
        'LESS', 'LESSEQ', 'GREATER', 'GREATEREQ', 'EQUAL',
        'INCREASE', 'DECREASE', 'MULVAL', 'DIVVAL',

        # Literals
        'ICONST', 'CCONST', 'SCONST'
        ] + list(reserved.values())

# Operations and Delimiters
t_PLUS = r'\+'
t_MINUS = r'-'
t_TIMES = r'\*'
t_DIVIDE = r'/'
t_ASSIGN = r'='
t_COMMA = r','
t_COLON = r':'
t_SEMI = r';'
t_ARROW = r'->'
t_LPAREN = r'\('
t_RPAREN = r'\)'
t_LBRACKET = r'\['
t_RBRACKET = r'\]'
t_LESS = r'<'
t_LESSEQ = r'<='
t_GREATER = r'>'
t_GREATEREQ = r'>='
t_EQUAL = r'=='
t_INCREASE = r'\+='
t_DECREASE = r'-='
t_MULVAL = r'\*='
t_DIVVAL = r'/='

# Comments
t_ignore_COMMENNT = r'(/\*(.*\n?)\*/|//.*)'

# Identifier
def t_ID(t):
    r'[A-Za-z_][a-zA-Z0-9_]*'
    t.type = reserved.get(t.value, 'ID') # Check for reserved words
    return t


def t_ICONST(t):
    r'-?\d+'
    t.value = int(t.value)
    return t


def t_CCONST(t):
    r'\'(\\\"|\\\'|[^\'\"])\''
    t.value = chr(t.value)
    return t


def t_SCONST(t):
    r'\'(\\\"|\\\'|[^\'\"])*\''
    t.value = str(t.value)
    return t


# Ignored characters
t_ignore = " \t"


def t_newline(t):
    r'\n+'
    t.lexer.lineno += t.value.count("\n")


# Error
# def t_error(t):
#     print("Error at " + str(t.lexer.lineno))
#     t.lexer.skip(1)

def t_error(t):
    print("Illegal character '%s'" % t.value[0])
    t.lexer.skip(1)


def main():
    file_name = sys.argv[1]

    # Read given file
    file = open(file_name, "r")<commit_msg>String<commit_after>
import sys
import ply.lex as lex

# Reserved
reserved = {
    # Reserved words
    'array': 'ARRAY',
    'by': 'BY',
    'chars': 'CHARS',
    'dcl': 'DCL',
    'do': 'DO',
    'down': 'DOWN',
    'else': 'ELSE',
    'elsif': 'ELSIF',
    'end': 'END',
    'exit': 'EXIT',
    'fi': 'FI',
    'for': 'FOR',
    'if': 'IF',
    'in': 'IN',
    'loc': 'LOC',
    'type': 'TYPE',
    'od': 'OD',
    'proc': 'PROC',
    'ref': 'REF',
    'result': 'RESULT',
    'return': 'RETURN',
    'returns': 'RETURNS',
    'syn': 'SYN',
    'then': 'THEN',
    'to': 'TO',
    'while': 'WHILE',

    # Predefined words
    'abs': 'ABS',
    'asc': 'ASC',
    'bool': 'BOOL',
    'char': 'CHAR',
    'false': 'FALSE',
    'int': 'INT',
    'length': 'LENGTH',
    'lower': 'LOWER',
    'null': 'NULL',
    'num': 'NUM',
    'print': 'PRINT',
    'read': 'READ',
    'true': 'TRUE',
    'upper': 'UPPER'
}

# Tokens
tokens = [
        # Identifier
        'ID',

        # Operations and Delimiters
        'PLUS', 'MINUS', 'TIMES', 'DIVIDE',
        'ASSIGN', 'COMMA', 'COLON', 'SEMI', 'ARROW',
        'LPAREN', 'RPAREN', 'LBRACKET', 'RBRACKET',
        'LESS', 'LESSEQ', 'GREATER', 'GREATEREQ', 'EQUAL',
        'INCREASE', 'DECREASE', 'MULVAL', 'DIVVAL','DIFF',
        'MOD','MODVAL','NEGATE',

        # Literals
        'ICONST', 'CCONST', 'SCONST'
        ] + list(reserved.values())

# Operations and Delimiters
t_PLUS = r'\+'
t_MINUS = r'-'
t_TIMES = r'\*'
t_DIVIDE = r'/'
t_ASSIGN = r'='
t_COMMA = r','
t_COLON = r':'
t_SEMI = r';'
t_ARROW = r'->'
t_LPAREN = r'\('
t_RPAREN = r'\)'
t_LBRACKET = r'\['
t_RBRACKET = r'\]'
t_LESS = r'<'
t_LESSEQ = r'<='
t_GREATER = r'>'
t_GREATEREQ = r'>='
t_EQUAL = r'=='
t_INCREASE = r'\+='
t_DECREASE = r'-='
t_MULVAL = r'\*='
t_DIVVAL = r'/='
t_DIFF = r'!='
t_MODVAL = r'%='
t_NEGATE = r'!'
t_MOD = r'%'

# Comments
t_ignore_COMMENNT = r'(/\*(.*\n?)\*/|//.*)'

# Identifier
def t_ID(t):
    r'[A-Za-z_][a-zA-Z0-9_]*'
    t.type = reserved.get(t.value, 'ID') # Check for reserved words
    return t


def t_ICONST(t):
    r'-?\d+'
    t.value = int(t.value)
    return t


def t_CCONST(t):
    r'\'(\\\"|\\\'|[^\'\"])\''
    t.value = chr(t.value)
    return t


def t_SCONST(t):
    r'\"(\\\"|\\\'|[^\'\"])*\"'
    t.value = str(t.value)
    return t


# Ignored characters
t_ignore = " \t"


def t_newline(t):
    r'\n+'
    t.lexer.lineno += t.value.count("\n")


# Error
# def t_error(t):
#     print("Error at " + str(t.lexer.lineno))
#     t.lexer.skip(1)

def t_error(t):
    x
    print("Illegal character '%s'" % t.value[0])
    t.lexer.skip(1)


def main():
    file_name = sys.argv[1]

    # Read given file
    file = open(file_name, "r")
{'fraction_non_alphanumeric': 0.25056603773584907, 'fraction_numerical': 0.0022641509433962265, 'mean_word_length': 2.5129224652087476, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4563382', 'n_tokens_mistral': 2402, 'n_tokens_neox': 2357, 'n_words': 497}