<commit_before>package polyglot.pth;

import java_cup.runtime.Symbol;

@SuppressWarnings("all")
%%
%cup
%public
%class Lexer_c

%{
   StringBuffer string = new StringBuffer();
%}

%eofval{
return (new Symbol(sym.EOF));
%eofval}

LINE_TERMINATOR = [\r]|[\n]|[\r\n]
WHITE_SPACE     = {LINE_TERMINATOR} | [ \t\f]

/* comments */
COMMENT = {C_COMMENT} | {LINE_COMMENT} | {END_OF_LINE_COMMENT}

C_COMMENT            = "/*" ~"*/"
END_OF_LINE_COMMENT  = "//" ~{LINE_TERMINATOR}
LINE_COMMENT         = "#" ~{LINE_TERMINATOR}

IDENT                = [a-zA-Z0-9\:\.\$\/\\\-]*

%state STRING_LIT

%%
<YYINITIAL> {
   /* identifiers */
   {IDENT}                     { return new Symbol(sym.IDENT, yytext()); }

   /* literals */
   \"                             { string.setLength(0); yybegin(STRING_LIT); }

   /* comments */
   {COMMENT}                      { /* ignore */ }

   /* whitespace */
   {WHITE_SPACE}                  { /* ignore white space. */ }

   /* symbols */
   "+"                            { return new Symbol(sym.PLUS); }
   ";"                            { return new Symbol(sym.SEMICOLON); }
   ","                            { return new Symbol(sym.COMMA); }
   "("                            { return new Symbol(sym.LPAREN); }
   ")"                            { return new Symbol(sym.RPAREN); }<commit_msg>Allows _ in PTH's identifier.<commit_after>package polyglot.pth;

import java_cup.runtime.Symbol;

@SuppressWarnings("all")
%%
%cup
%public
%class Lexer_c

%{
   StringBuffer string = new StringBuffer();
%}

%eofval{
return (new Symbol(sym.EOF));
%eofval}

LINE_TERMINATOR = [\r]|[\n]|[\r\n]
WHITE_SPACE     = {LINE_TERMINATOR} | [ \t\f]

/* comments */
COMMENT = {C_COMMENT} | {LINE_COMMENT} | {END_OF_LINE_COMMENT}

C_COMMENT            = "/*" ~"*/"
END_OF_LINE_COMMENT  = "//" ~{LINE_TERMINATOR}
LINE_COMMENT         = "#" ~{LINE_TERMINATOR}

IDENT                = [a-zA-Z0-9_\:\.\$\/\\\-]*

%state STRING_LIT

%%
<YYINITIAL> {
   /* identifiers */
   {IDENT}                     { return new Symbol(sym.IDENT, yytext()); }

   /* literals */
   \"                             { string.setLength(0); yybegin(STRING_LIT); }

   /* comments */
   {COMMENT}                      { /* ignore */ }

   /* whitespace */
   {WHITE_SPACE}                  { /* ignore white space. */ }

   /* symbols */
   "+"                            { return new Symbol(sym.PLUS); }
   ";"                            { return new Symbol(sym.SEMICOLON); }
   ","                            { return new Symbol(sym.COMMA); }
   "("                            { return new Symbol(sym.LPAREN); }
   ")"                            { return new Symbol(sym.RPAREN); }
{'fraction_non_alphanumeric': 0.16847622613253463, 'fraction_numerical': 0.0022463496817671283, 'mean_word_length': 1.6455445544554455, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1353036', 'n_tokens_mistral': 921, 'n_tokens_neox': 909, 'n_words': 145}