<commit_before>Definitions.

IDENTIFIER = [-A-Za-z0-9_]+
QUOTED = (\".*\"|\'.*\')
SYMBOL = [\[\]*]
W = [\s\t\r\n\f]

Rules.

{IDENTIFIER}     : {token, {identifier, TokenLine, TokenChars}}.
{QUOTED}         : {token, {quoted, TokenLine, remove_quotes(TokenChars)}}.
{SYMBOL}         : {token, {TokenChars, TokenLine}}.
#{IDENTIFIER}    : {token, {hash, TokenLine, tail(TokenChars)}}.
\.{IDENTIFIER}   : {token, {class, TokenLine, tail(TokenChars)}}.
~=               : {token, {includes, TokenLine}}.
\|=              : {token, {dash_match, TokenLine}}.<commit_msg>Using negative list in order to be more specific in regex<commit_after>Definitions.

IDENTIFIER = [-A-Za-z0-9_]+
QUOTED = (\"[^"]*\"|\'[^']*\')
SYMBOL = [\[\]*]
W = [\s\t\r\n\f]

Rules.

{IDENTIFIER}     : {token, {identifier, TokenLine, TokenChars}}.
{QUOTED}         : {token, {quoted, TokenLine, remove_quotes(TokenChars)}}.
{SYMBOL}         : {token, {TokenChars, TokenLine}}.
#{IDENTIFIER}    : {token, {hash, TokenLine, tail(TokenChars)}}.
\.{IDENTIFIER}   : {token, {class, TokenLine, tail(TokenChars)}}.
~=               : {token, {includes, TokenLine}}.
\|=              : {token, {dash_match, TokenLine}}.
{'fraction_non_alphanumeric': 0.22372881355932203, 'fraction_numerical': 0.003389830508474576, 'mean_word_length': 3.202846975088968, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7122607', 'n_tokens_mistral': 453, 'n_tokens_neox': 429, 'n_words': 85}