<commit_before>#
# -*- coding: utf-8 -*-
# Zmeya's Lexer
# <NAME>
# <NAME>
#

import ply.lex as lex

# Reserved words
reserved = {
  'main'   : 'MAIN',
  'return' : 'RETURN',
  'if'     : 'IF',
  'else'   : 'ELSE',
  'write'  : 'WRITE',
  'read'   : 'READ',
  'while'  : 'WHILE',
  'repeat' : 'REPEAT',
  'int'    : 'INT',
  'float'  : 'FLOAT',
  'string' : 'STRING',
  'void'   : 'VOID',
  'bool'   : 'BOOL',
  'char'   : 'CHAR',
  'T'      : 'TRUE',
  'F'      : 'FALSE',
  '&&'     : 'AND',
  '||'     : 'OR'
}

tokens = [
  'SUM', 'MINUS','MULT', 'DIV', 'MOD', 'EQUALS',
  'EQUALITY', 'GREATER', 'LESS', 'L_EQUAL', 'G_EQUAL',
  'N_EQUAL', 'SEMICOLON', 'COMMA', 'L_PAREN', 'R_PAREN',
  'L_BRACE', 'R_BRACE', 'L_BRACKET', 'R_BRACKET', 'INT_CONST',
  'STRING_CONST', 'FLOAT_CONST', 'ID', 'POS_INT_CONST'
  ] + list(reserved.values())

# Token definitions
t_ignore = ' \t'
t_SUM = '\+'
t_MINUS = '-'
t_MULT = '\*'
t_DIV = '/'
t_MOD = '%'
t_EQUALS = '='
t_EQUALITY = '=='
t_GREATER = '>'
t_LESS = '<'
t_L_EQUAL = '<='
t_G_EQUAL = '>='
t_N_EQUAL = '<>'
t_SEMICOLON = ';'
t_COMMA = ','
t_L_PAREN = '\('
t_R_PAREN = '\)'
t_L_BRACE = '\{'
t_R_BRACE = '\}'
t_L_BRACKET = '\['
t_R_BRACKET = '\]'
t_INT_CONST = '[-][0-9]+'
t_POS_INT_CONST = '[0-9]+'
t_STRING_CONST = '"[a-zA-Z_].*"'
t_FLOAT_CONST = '[-]([0-9]+[.])?[0-9]+'

# LOC counter
def t_newline(t):
    r'\n+'
    t.lexer.lineno += len(t.value)

# ID recognition
# Checks if id is not a reserved word
def t_ID(t):
  '[a-zA-Z_][a-zA-Z0-9_]*'
  t.type = reserved.get(t.value, 'ID')
  return t

# Error
def t_error(t):
  print("Illegal character '%s'" % t.value[0])
  exit(-1)
  t.lexer.skip(1)

lexer = lex.lex()<commit_msg>Added regexp for function ids starting with underscore<commit_after>#
# -*- coding: utf-8 -*-
# Zmeya's Lexer
# <NAME>
# <NAME>
#

import ply.lex as lex

# Reserved words
reserved = {
  'main'   : 'MAIN',
  'return' : 'RETURN',
  'if'     : 'IF',
  'else'   : 'ELSE',
  'write'  : 'WRITE',
  'read'   : 'READ',
  'while'  : 'WHILE',
  'repeat' : 'REPEAT',
  'int'    : 'INT',
  'float'  : 'FLOAT',
  'string' : 'STRING',
  'void'   : 'VOID',
  'bool'   : 'BOOL',
  'char'   : 'CHAR',
  'T'      : 'TRUE',
  'F'      : 'FALSE',
  '&&'     : 'AND',
  '||'     : 'OR'
}

tokens = [
  'SUM', 'MINUS','MULT', 'DIV', 'MOD', 'EQUALS',
  'EQUALITY', 'GREATER', 'LESS', 'L_EQUAL', 'G_EQUAL',
  'N_EQUAL', 'SEMICOLON', 'COMMA', 'L_PAREN', 'R_PAREN',
  'L_BRACE', 'R_BRACE', 'L_BRACKET', 'R_BRACKET', 'INT_CONST',
  'STRING_CONST', 'FLOAT_CONST', 'ID', 'POS_INT_CONST', 'ID_FUN'
  ] + list(reserved.values())

# Token definitions
t_ignore = ' \t'
t_SUM = '\+'
t_MINUS = '-'
t_MULT = '\*'
t_DIV = '/'
t_MOD = '%'
t_EQUALS = '='
t_EQUALITY = '=='
t_GREATER = '>'
t_LESS = '<'
t_L_EQUAL = '<='
t_G_EQUAL = '>='
t_N_EQUAL = '<>'
t_SEMICOLON = ';'
t_COMMA = ','
t_L_PAREN = '\('
t_R_PAREN = '\)'
t_L_BRACE = '\{'
t_R_BRACE = '\}'
t_L_BRACKET = '\['
t_R_BRACKET = '\]'
t_INT_CONST = '[-][0-9]+'
t_POS_INT_CONST = '[0-9]+'
t_STRING_CONST = '"[a-zA-Z_].*"'
t_FLOAT_CONST = '[-]([0-9]+.)?[0-9]+'

# LOC counter
def t_newline(t):
    r'\n+'
    t.lexer.lineno += len(t.value)

# ID recognition
# Checks if id is not a reserved word
def t_ID(t):
  '[a-zA-Z_][a-zA-Z0-9_]*'
  t.type = reserved.get(t.value, 'ID')
  return t

def t_ID_FUN(t):
  '_[a-zA-Z_][a-zA-Z0-9_]*'
  t.type = reserved.get(t.value, 'ID')
  return t

# Error
def t_error(t):
  print("Illegal character '%s'" % t.value[0])
  exit(-1)
  t.lexer.skip(1)

lexer = lex.lex()
{'fraction_non_alphanumeric': 0.2555205047318612, 'fraction_numerical': 0.008603383997705764, 'mean_word_length': 3.269277845777234, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '992793', 'n_tokens_mistral': 1735, 'n_tokens_neox': 1696, 'n_words': 299}