<commit_before>%%% -*- mode: erlang -*-
%%% @doc       Lexer for the riak Time Series Query Language.
%%% @author    <EMAIL>
%%% @copyright (C) 2015 Basho

Definitions.

AND = (A|a)(N|n)(D|d)
AS = (A|a)(S|s)
CREATE_TABLE = (C|c)(R|r)(E|e)(A|a)(T|t)(E|e)\s(T|t)(A|a)(B|b)(L|l)(E|e)
DELETE = (D|d)(E|e)(L|l)(E|e)(T|t)(E|e)
DROP = (D|d)(R|r)(O|o)(P|p)
FLOAT = (F|f)(L|l)(O|o)(A|a)(T|t)
FROM = (F|f)(R|r)(O|o)(M|m)
GLOBAL = (G|g)(L|l)(O|o)(B|b)(A|a)(L|l)
GROUPBY = (G|g)(R|r)(O|o)(U|u)(P|p)(B|b)(Y|y)
INNER = (I|i)(N|n)(N|n)(E|e)(R|r)
JOIN = (J|j)(O|o)(I|i)(N|n)
LIMIT = (L|l)(I|i)(M|m)(I|i)(T|t)
LOCAL = (L|l)(O|o)(C|c)(A|a)(L|l)
MERGE = (M|m)(E|e)(R|r)(G|g)(E|e)
NOT_NULL = (N|n)(O|o)(T|t)\s(N|n)(U|u)(L|l)(L|l)
OF = (O|o)(F|f)
ON_COMMIT = (O|o)(N|n)\s(C|c)(O|o)(M|m)(M|m)(I|i)(T|t)
OR = (O|o)(R|r)
PARTITION_KEY = (P|p)(A|a)(R|r)(T|t)(I|i)(T|t)(I|i)(O|o)(N|n)\s(K|k)(E|e)(Y|y)
PRESERVE = (P|p)(R|r)(E|e)(S|s)(E|e)(R|r)(V|v)(E|e)
ROWS = (R|r)(O|o)(W|w)(S|s)
SELECT = (S|s)(E|e)(L|l)(E|e)(C|c)(T|t)
SYSTEM_VERSIONING = (S|s)(Y|y)(S|s)(T|t)(E|e)(M|m)\s(V|v)(E|e)(R|r)(S|s)(I|i)(O|o)(N|n)(I|i)(N|n)(G|g)
TEMPORARY = (T|t)(E|e)(M|m)(P|p)(O|o)(R|r)(A|a)(R|r)(Y|y)
TIMESTAMP = (T|t)(I|i)(M|m)(E|e)(S|s)(T|t)(A|a)(M|m)(P|p)
WHERE = (W|w)(H|h)(E|e)(R|r)(E|e)
WITH = (W|w)(I|i)(T|t)(H|h)

DATETIME = ('[0-9a-zA-Z\s:\-\.]*')

REGEX = (/[^/][a-zA-Z0-9\*\.]+/i?)

QUOTED = ("(.*(\")*)")

WHITESPACE = ([\000-\s]*)

INT      = (\-*[0-9]+)
FLOATDEC = (\-*([0-9]+)?\.[0-9]+)
FLOATSCI = (\-*([0-9]+)?(\.)?[0-9]+(E|e)(\+|\-)?[0-9]+)

APPROXMATCH = (=\~)
EQ          = (=)
GT          = (>)
LT          = (<)
NE          = (<>)
NOMATCH     = (!=)
NOTAPPROX   = (!\~)
OPEN        = \(
CLOSE       =\)

PLUS  = (\+)
MINUS = (\-)
TIMES = (\*)
DIV   = (/)

COMMA = (,)

Rules.

{AND} : {token, {and_, TokenChars}}.
{AS} : {token, {as, TokenChars}}.
{CREATE_TABLE} : {token, {create_table, TokenChars}}.
{DELETE} : {token, {delete, TokenChars}}.
{DROP} : {token, {drop, TokenChars}}.
{FLOAT} : {token, {float, TokenChars}}.
{FROM} : {token, {from, TokenChars}}.
{GLOBAL} : {token, {global, TokenChars}}.
{GROUPBY} : {token, {groupby, TokenChars}}.
{INNER} : {token, {inner, TokenChars}}.
{JOIN} : {token, {join, TokenChars}}.
{LIMIT} : {token, {limit, TokenChars}}.
{LOCAL} : {token, {local, TokenChars}}.
{MERGE} : {token, {merge, TokenChars}}.
{NOT_NULL} : {token, {not_null, TokenChars}}.
{OF} : {token, {of_, TokenChars}}.
{ON_COMMIT} : {token, {on_commit, TokenChars}}.
{OR} : {token, {or_, TokenChars}}.
{PARTITION_KEY} : {token, {partition_key, TokenChars}}.
{PRESERVE} : {token, {preserve, TokenChars}}.
{ROWS} : {token, {rows, TokenChars}}.
{SELECT} : {token, {select, TokenChars}}.
{SYSTEM_VERSIONING} : {token, {system_versioning, TokenChars}}.
{TEMPORARY} : {token, {temporary, TokenChars}}.
{TIMESTAMP} : {token, {timestamp, TokenChars}}.
{WHERE} : {token, {where, TokenChars}}.
{WITH} : {token, {with, TokenChars}}.

{INT}      : {token, {int,   list_to_integer(TokenChars)}}.
{FLOATDEC} : {token, {float, list_to_float(TokenChars)}}.
{FLOATSCI} : {token, {float, list_to_float(TokenChars)}}.

{EQ}          : {token, {eq,     TokenChars}}.
{APPROXMATCH} : {token, {approx, TokenChars}}.
{GT}          : {token, {gt,        TokenChars}}.
{LT}          : {token, {lt,        TokenChars}}.
{NE}          : {token, {ne,        TokenChars}}.
{NOMATCH}     : {token, {nomatch,   TokenChars}}.
{NOTAPPROX}   : {token, {notapprox,   TokenChars}}.

{OPEN}  :  {token, {openb,  TokenChars}}.
{CLOSE} :  {token, {closeb, TokenChars}}.

{PLUS}  : {token, {plus,       TokenChars}}.
{MINUS} : {token, {minus,      TokenChars}}.
{TIMES} : {token, {maybetimes, TokenChars}}.
{DIV}   : {token, {div_,       TokenChars}}.<commit_msg>lex local key and varchar<commit_after>%%% -*- mode: erlang -*-
%%% @doc       Lexer for the riak Time Series Query Language.
%%% @author    <EMAIL>
%%% @copyright (C) 2015 Basho

Definitions.

AND = (A|a)(N|n)(D|d)
AS = (A|a)(S|s)
CREATE_TABLE = (C|c)(R|r)(E|e)(A|a)(T|t)(E|e)\s(T|t)(A|a)(B|b)(L|l)(E|e)
DELETE = (D|d)(E|e)(L|l)(E|e)(T|t)(E|e)
DROP = (D|d)(R|r)(O|o)(P|p)
FLOAT = (F|f)(L|l)(O|o)(A|a)(T|t)
FROM = (F|f)(R|r)(O|o)(M|m)
GLOBAL = (G|g)(L|l)(O|o)(B|b)(A|a)(L|l)
GROUPBY = (G|g)(R|r)(O|o)(U|u)(P|p)(B|b)(Y|y)
INNER = (I|i)(N|n)(N|n)(E|e)(R|r)
JOIN = (J|j)(O|o)(I|i)(N|n)
LIMIT = (L|l)(I|i)(M|m)(I|i)(T|t)
LOCAL = (L|l)(O|o)(C|c)(A|a)(L|l)
LOCAL_KEY = (L|l)(O|o)(C|c)(A|a)(L|l)\s(K|k)(E|e)(Y|y)
MERGE = (M|m)(E|e)(R|r)(G|g)(E|e)
NOT_NULL = (N|n)(O|o)(T|t)\s(N|n)(U|u)(L|l)(L|l)
OF = (O|o)(F|f)
ON_COMMIT = (O|o)(N|n)\s(C|c)(O|o)(M|m)(M|m)(I|i)(T|t)
OR = (O|o)(R|r)
PARTITION_KEY = (P|p)(A|a)(R|r)(T|t)(I|i)(T|t)(I|i)(O|o)(N|n)\s(K|k)(E|e)(Y|y)
PRESERVE = (P|p)(R|r)(E|e)(S|s)(E|e)(R|r)(V|v)(E|e)
ROWS = (R|r)(O|o)(W|w)(S|s)
SELECT = (S|s)(E|e)(L|l)(E|e)(C|c)(T|t)
SYSTEM_VERSIONING = (S|s)(Y|y)(S|s)(T|t)(E|e)(M|m)\s(V|v)(E|e)(R|r)(S|s)(I|i)(O|o)(N|n)(I|i)(N|n)(G|g)
TEMPORARY = (T|t)(E|e)(M|m)(P|p)(O|o)(R|r)(A|a)(R|r)(Y|y)
TIMESTAMP = (T|t)(I|i)(M|m)(E|e)(S|s)(T|t)(A|a)(M|m)(P|p)
VARCHAR = (V|v)(A|a)(R|r)(C|c)(H|h)(A|a)(R|r)
WHERE = (W|w)(H|h)(E|e)(R|r)(E|e)
WITH = (W|w)(I|i)(T|t)(H|h)

DATETIME = ('[0-9a-zA-Z\s:\-\.]*')

REGEX = (/[^/][a-zA-Z0-9\*\.]+/i?)

QUOTED = ("(.*(\")*)")

WHITESPACE = ([\000-\s]*)

INT      = (\-*[0-9]+)
FLOATDEC = (\-*([0-9]+)?\.[0-9]+)
FLOATSCI = (\-*([0-9]+)?(\.)?[0-9]+(E|e)(\+|\-)?[0-9]+)

APPROXMATCH = (=\~)
EQ          = (=)
GT          = (>)
LT          = (<)
NE          = (<>)
NOMATCH     = (!=)
NOTAPPROX   = (!\~)
OPEN        = \(
CLOSE       =\)

PLUS  = (\+)
MINUS = (\-)
TIMES = (\*)
DIV   = (/)

COMMA = (,)

Rules.

{AND} : {token, {and_, TokenChars}}.
{AS} : {token, {as, TokenChars}}.
{CREATE_TABLE} : {token, {create_table, TokenChars}}.
{DELETE} : {token, {delete, TokenChars}}.
{DROP} : {token, {drop, TokenChars}}.
{FLOAT} : {token, {float, TokenChars}}.
{FROM} : {token, {from, TokenChars}}.
{GLOBAL} : {token, {global, TokenChars}}.
{GROUPBY} : {token, {groupby, TokenChars}}.
{INNER} : {token, {inner, TokenChars}}.
{JOIN} : {token, {join, TokenChars}}.
{LIMIT} : {token, {limit, TokenChars}}.
{LOCAL} : {token, {local, TokenChars}}.
{LOCAL_KEY} : {token, {local_key, TokenChars}}.
{MERGE} : {token, {merge, TokenChars}}.
{NOT_NULL} : {token, {not_null, TokenChars}}.
{OF} : {token, {of_, TokenChars}}.
{ON_COMMIT} : {token, {on_commit, TokenChars}}.
{OR} : {token, {or_, TokenChars}}.
{PARTITION_KEY} : {token, {partition_key, TokenChars}}.
{PRESERVE} : {token, {preserve, TokenChars}}.
{ROWS} : {token, {rows, TokenChars}}.
{SELECT} : {token, {select, TokenChars}}.
{SYSTEM_VERSIONING} : {token, {system_versioning, TokenChars}}.
{TEMPORARY} : {token, {temporary, TokenChars}}.
{TIMESTAMP} : {token, {timestamp, TokenChars}}.
{VARCHAR} : {token, {varchar, TokenChars}}.
{WHERE} : {token, {where, TokenChars}}.
{WITH} : {token, {with, TokenChars}}.

{INT}      : {token, {int,   list_to_integer(TokenChars)}}.
{FLOATDEC} : {token, {float, list_to_float(TokenChars)}}.
{FLOATSCI} : {token, {float, list_to_float(TokenChars)}}.

{EQ}          : {token, {eq,     TokenChars}}.
{APPROXMATCH} : {token, {approx, TokenChars}}.
{GT}          : {token, {gt,        TokenChars}}.
{LT}          : {token, {lt,        TokenChars}}.
{NE}          : {token, {ne,        TokenChars}}.
{NOMATCH}     : {token, {nomatch,   TokenChars}}.
{NOTAPPROX}   : {token, {notapprox,   TokenChars}}.

{OPEN}  :  {token, {openb,  TokenChars}}.
{CLOSE} :  {token, {closeb, TokenChars}}.

{PLUS}  : {token, {plus,       TokenChars}}.
{MINUS} : {token, {minus,      TokenChars}}.
{TIMES} : {token, {maybetimes, TokenChars}}.
{DIV}   : {token, {div_,       TokenChars}}.
{'fraction_non_alphanumeric': 0.31902323749507683, 'fraction_numerical': 0.006039123014310096, 'mean_word_length': 4.528301886792453, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3268440', 'n_tokens_mistral': 3970, 'n_tokens_neox': 3746, 'n_words': 561}