<commit_before>/* lexical grammar */
%options flex case-insensitive
%lex
%%
^\#.*                    return '#';
\n                       return "l";
\t                       return "t";
\s                       return "w";
"while"                  return "W";
"if"                     return "I";
"then"                   return "T";
"else"                   return "E";
"and"                    return "&";
"or"                     return "|";
"not"                    return "!";
"is"                     return "=";
"true"                   return "1";
"false"                  return "0";
"in"                     return "e";
"["                      return "[";
"]"                      return "]";
"_"                      return "_";
"("                      return "(";
")"                      return ")";
"{"                      return "{";
"}"                      return "}";
"nil"                    return "N";
\"[^\"]*\"               return "s";
[0-9]+("."[0-9]+)?\b     return "n";
[A-Za-z]([A-Za-z0-9])*   return "i";
"="                      return "=";
"^"                      return '^';
"*"                      return '*';
"/"                      return '/';
"-"                      return '-';
"+"                      return '+';
"$"                      return "$";
\:                       return ":";
\;                       return ";";
[\<\>][\=]?              return "c";                 
<<EOF>>                  return 'F';<commit_msg>updated symbol<commit_after>/* lexical grammar */
%options flex case-insensitive
%lex
%%
^\#.*                    return '#';
\n                       return "l";
\t                       return "t";
\s                       return "w";
"while"                  return "W";
"if"                     return "I";
"then"                   return "T";
"else"                   return "E";
"and"                    return "&";
"or"                     return "|";
"not"                    return "!";
"is"                     return "=";
"true"                   return "1";
"false"                  return "0";
"in"                     return ">";
"["                      return "[";
"]"                      return "]";
"_"                      return "_";
"("                      return "(";
")"                      return ")";
"{"                      return "{";
"}"                      return "}";
"nil"                    return "N";
\"[^\"]*\"               return "s";
[0-9]+("."[0-9]+)?\b     return "n";
[A-Za-z]([A-Za-z0-9])*   return "i";
"="                      return "=";
"^"                      return '^';
"*"                      return '*';
"/"                      return '/';
"-"                      return '-';
"+"                      return '+';
"$"                      return "$";
\:                       return ":";
\;                       return ";";
[\<\>][\=]?              return "c";                 
<<EOF>>                  return 'F';
{'fraction_non_alphanumeric': 0.1849338310145911, 'fraction_numerical': 0.005429250084832033, 'mean_word_length': 0.7060185185185185, 'pattern_counts': {'":': 2, '<': 9, '<?xml version=': 0, '>': 10, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2911827', 'n_tokens_mistral': 817, 'n_tokens_neox': 726, 'n_words': 155}