<commit_before>import ply.lex as lex

class Lexer(object):

  reserved = {
    'and'     : 'AND',
    'break'   : 'BREAK',
    'do'      : 'DO',
    'else'    : 'ELSE',
    'elseif'  : 'ELSEIF',
    'end'     : 'END',
    'false'   : 'FALSE',
    'for'     : 'FOR',
    'function': 'FUNCTION',
    'if'      : 'IF',
    'in'      : 'IN',
    'local'   : 'LOCAL',
    'nil'     : 'NIL',
    'not'     : 'NOT',
    'or'      : 'OR',
    'repeat'  : 'REPEAT',
    'return'  : 'RETURN',
    'then'    : 'THEN',
    'true'    : 'TRUE',
    'until'   : 'UNTIL',
    'while'   : 'WHILE'
  }

  # List of token names
  tokens = [
    'ID'        , 'NUMBER'      , 'STRING',
    'PLUS'      , 'MINUS'       , 'TIMES',
    'DIVIDE'    , 'PERCENT'     , 'CIRCUMFLEX',
    'SHARP'     , 'EQUAL'       , 'NOTEQUAL',
    'LESSEQUAL' , 'GREATEREQUAL', 'LESS',
    'GREATER'   , 'ASSIGN'      , 'LPAREN',
    'RPAREN'    , 'LCURLY'      , 'RCURLY',
    'LSQUARE'   , 'RSQUARE'     , 'SEMICOLON',
    'COLON'     , 'COMMA'       , 'DOT',
    'TWODOTS'   , 'THREEDOTS'
  ] + list(reserved.values())

  t_STRING        = r'(\"([^\\\n]|(\\.))*?\")|(\'([^\\\n]|(\\.))*?\')'
  t_PLUS          = r'\+'
  t_MINUS         = r'-'
  t_TIMES         = r'\*'
  t_DIVIDE        = r'/'
  t_PERCENT       = r'%'
  t_CIRCUMFLEX    = r'\^'
  t_SHARP         = r'\#'<commit_msg>Adding a regex to match char literal<commit_after>import ply.lex as lex

class Lexer(object):

  reserved = {
    'and'     : 'AND',
    'break'   : 'BREAK',
    'do'      : 'DO',
    'else'    : 'ELSE',
    'elseif'  : 'ELSEIF',
    'end'     : 'END',
    'false'   : 'FALSE',
    'for'     : 'FOR',
    'function': 'FUNCTION',
    'if'      : 'IF',
    'in'      : 'IN',
    'local'   : 'LOCAL',
    'nil'     : 'NIL',
    'not'     : 'NOT',
    'or'      : 'OR',
    'repeat'  : 'REPEAT',
    'return'  : 'RETURN',
    'then'    : 'THEN',
    'true'    : 'TRUE',
    'until'   : 'UNTIL',
    'while'   : 'WHILE'
  }

  # List of token names
  tokens = [
    'ID'        , 'NUMBER'      , 'CHAR',
    'STRING'    , 'PLUS'        , 'MINUS',
    'TIMES'     , 'DIVIDE'      , 'PERCENT',
    'CIRCUMFLEX', 'SHARP'       , 'EQUAL',
    'NOTEQUAL'  , 'LESSEQUAL'   , 'GREATEREQUAL',
    'LESS'      , 'GREATER'     , 'ASSIGN',
    'LPAREN'    , 'RPAREN'      , 'LCURLY',
    'RCURLY'    , 'LSQUARE'     , 'RSQUARE',
    'SEMICOLON' , 'COLON'       , 'COMMA',
    'DOT'       , 'TWODOTS'     , 'THREEDOTS'
  ] + list(reserved.values())

  t_CHAR          = r'(\"([^\\\n]|(\\.))?\")|(\'([^\\\n]|(\\.))?\')'
  t_STRING        = r'(\"([^\\\n]|(\\.))*?\")|(\'([^\\\n]|(\\.))*?\')'
  t_PLUS          = r'\+'
  t_MINUS         = r'-'
  t_TIMES         = r'\*'
  t_DIVIDE        = r'/'
  t_PERCENT       = r'%'
  t_CIRCUMFLEX    = r'\^'
  t_SHARP         = r'\#'
{'fraction_non_alphanumeric': 0.24015748031496062, 'fraction_numerical': 0.0, 'mean_word_length': 1.5642201834862386, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1125939', 'n_tokens_mistral': 1270, 'n_tokens_neox': 1215, 'n_words': 208}