<commit_before>from collections import Sequence

from pyparsing import nums
from pyparsing import Word, CaselessKeyword, QuotedString, Regex, Literal, Suppress
from pyparsing import Optional, ZeroOrMore, Or, Group, NotAny, Forward, StringEnd
from pyparsing import delimitedList
from enum import Enum


class OdhQLParser(object):
    """
    Parser for the OpenDataHub Query Language (OdhQL).

    Syntax borrows heavily from ANSI SQLs SELECT statement, but there are some differences.

    All keywords are case-insensitive.

    Identifiers: Everything that doesn't match r'[a-zA-Z_][a-zA-Z0-9_]*' needs to be quoted using "double quotes".
    If you need to include quote chars inside your strings, escape with a backslash ('\').

    All field references must be prefixed by the name of the data source they're coming from.

    Description in BNF (as used by the diagram generator at http://bottlecaps.de/rr/ui):

    ---------------------------------------------------------------------------
    UnionQuery ::= Query ( "union" Query )* ( OrderByList )?
    Query ::= FieldSelection DataSourceSelection ( FilterList )?

    FieldSelection ::= "select" AliasedFieldEquiv ( "," AliasedFieldEquiv )*
    AliasedFieldEquiv ::= FieldEquiv "as" Alias
    FieldEquiv ::= Function | Value | Field
    Function ::= Identifier "(" ( ListOfFunctionArguments )? ")"
    ListOfFunctionArguments ::= FieldEquiv ( ( "," FieldEquiv )* )?


    Field ::= DataSourceNameOrAlias "." FieldName
    DataSourceSelection ::= "from" DataSourceName ( "as" Alias )? ( JoinDefinition )*
    JoinDefinition ::= "join" DataSourceName ( "as" Alias )? "on" JoinCondition
    JoinCondition ::= SingleJoinCondition | "(" SingleJoinCondition ( "and" SingleJoinCondition )* ")"
    SingleJoinCondition ::= Field "=" Field
    DataSourceNameOrAlias ::= DataSourceName | Alias
    FilterList ::= "where" FilterAlternative
    FilterAlternative ::= FilterCombination ( "or" FilterCombination )*
    FilterCombination ::= FilterDefinition ( "and" FilterDefinition )*
    FilterDefinition ::= BinaryCondition | InCondition |
                       IsNullCondition | "(" FilterAlternative ")"
    BinaryCondition ::= FieldEquiv BinaryOperator FieldEquiv
    BinaryOperator ::= "=" | "!=" | "<" | "<=" | ">" | ">=" | "like"
    InCondition ::= FieldEquiv ( "not" )? "in" "(" Value ( "," Value )* ")"
    IsNullCondition ::= Field "is" ( "not" )? Null
    OrderByList ::= "order" "by" OrderByField ( "," OrderByField )*
    OrderByField ::= Field ( "asc" | "desc" )?
    Integer ::= ( "0" | "1" | "2" | "3" | "4" | "5" | "6" | "7" | "8" | "9" )*
    DataSourceName ::= Identifier
    FieldName ::= Identifier
    Alias ::= Identifier

    Value ::= SingleQuotedString | Number | Boolean | Null
    Number ::= Integer | Float
    Float ::= Integer "." Integer
    Boolean ::= "true" | "false"
    Null ::= "null"

    SingleQuotedString ::= "string in single quotes"
    DoubleQuotedString ::= "string in double quotes"

    Identifier ::= ( "a..z" | "A..Z" | "_" ) ( "a..z" | "A..Z" | "_" | Integer )* | DoubleQuotedString
    ---------------------------------------------------------------------------
    """

    def __init__(self):
        self.grammar = self.build_grammar()

    @classmethod
    def build_grammar(cls):
        escape_char = '\\'
        identifier = Regex(r'[a-zA-Z_][a-zA-Z0-9_]*') | QuotedString('"', escChar=escape_char)
        kw_and = CaselessKeyword('and')
        kw_or = CaselessKeyword('or')

        alias = CaselessKeyword('as') + identifier('alias')

        field = (identifier.copy()('prefix') + '.' + identifier.copy()('name') + NotAny('('))('field')
        field.setParseAction(Field.parse)

        aliased_field = field + Optional(alias)  # defaults to using name as alias
        aliased_field.setParseAction(AliasedField.parse)

        integer_value = Word(nums)
        number_value = Group(integer_value + Optional('.' + integer_value))
        number_value.setParseAction(Expression.parse_number)

        boolean_value = CaselessKeyword('true') | CaselessKeyword('false')
        boolean_value.setParseAction(Expression.parse_boolean)

        string_value = QuotedString('\'', escChar=escape_char)
        null_value = CaselessKeyword('null')
        null_value.setParseAction(Expression.parse_null)

        value = (number_value | boolean_value | string_value | null_value)('value')
        value.setParseAction(Expression.parse)

        aliased_value = (value + alias)
        aliased_value.setParseAction(AliasedExpression.parse)

        function = Forward()
        function << (identifier.copy()('name') +
                     '(' + Optional(delimitedList(field | value | function))('args') + ')')
        function.setParseAction(Function.parse)

        aliased_function = function('function') + alias
        aliased_function.setParseAction(AliasedFunction.parse)

        field_declaration = aliased_function | aliased_value | aliased_field

        field_declaration_list = Suppress(CaselessKeyword('select')) + delimitedList(field_declaration)

        data_source = identifier('name') + Optional(alias)
        data_source.setParseAction(DataSource.parse)

        join_single_condition = field('left') + '=' + field('right')
        join_single_condition.setParseAction(JoinCondition.parse)

        join_multi_condition = Literal('(') + delimitedList(join_single_condition, delim=kw_and)('conditions') + ')'
        join_multi_condition.setParseAction(JoinConditionList.parse)

        join_condition_list = join_single_condition | join_multi_condition

        join = (CaselessKeyword('join') + data_source.copy()('datasource') +
                CaselessKeyword('on') + join_condition_list.copy()('condition'))
        join.setParseAction(JoinedDataSource.parse)

        data_source_declaration = Suppress(CaselessKeyword('from')) + data_source + ZeroOrMore(join)

        condition_side = field | value | function

        operator = Literal('=') | '!=' | '<=' | '<' | '>=' | '>' | CaselessKeyword('like')
        operator.setParseAction(BinaryCondition.parse_operator)

        binary_condition = condition_side.copy()('left') + operator('operator') + condition_side.copy()('right')
        binary_condition.setParseAction(BinaryCondition.parse)

        in_condition = (condition_side.copy()('left') + Optional(CaselessKeyword('not'))('invert') +
                        CaselessKeyword('in') + '(' + Group(delimitedList(value))('in_list') + ')')
        in_condition.setParseAction(InCondition.parse)

        null_condition = (field + CaselessKeyword('is') + Optional(CaselessKeyword('not'))('invert') +
                          CaselessKeyword('null'))
        null_condition.setParseAction(IsNullCondition.parse)

        single_filter = Forward()

        filter_combination = delimitedList(single_filter, delim=kw_and)
        filter_combination.setParseAction(FilterCombination.parse)

        filter_alternative = delimitedList(filter_combination, delim=kw_or)
        filter_alternative.setParseAction(FilterAlternative.parse)

        single_filter << (null_condition | binary_condition | in_condition |
                          Suppress('(') + filter_alternative + Suppress(')'))

        filter_declaration = Suppress(CaselessKeyword('where')) + filter_alternative.copy()('conditions')

        order_by_field = field + Optional(Or([CaselessKeyword('asc'), CaselessKeyword('desc')]))('direction')
        order_by_field.setParseAction(OrderBy.parse)

        order_by_declaration = (Suppress(CaselessKeyword('order') + CaselessKeyword('by')) +
                                delimitedList(order_by_field)('fields'))
        order_by_declaration.setParseAction(Query.parse_order_by)

        query = (field_declaration_list('fields') + data_source_declaration('datasources') +
                 Optional(filter_declaration('filter')) )
        query.setParseAction(Query.parse)
<commit_msg>yes, flake, there was a space. eat it.<commit_after>from collections import Sequence

from pyparsing import nums
from pyparsing import Word, CaselessKeyword, QuotedString, Regex, Literal, Suppress
from pyparsing import Optional, ZeroOrMore, Or, Group, NotAny, Forward, StringEnd
from pyparsing import delimitedList
from enum import Enum


class OdhQLParser(object):
    """
    Parser for the OpenDataHub Query Language (OdhQL).

    Syntax borrows heavily from ANSI SQLs SELECT statement, but there are some differences.

    All keywords are case-insensitive.

    Identifiers: Everything that doesn't match r'[a-zA-Z_][a-zA-Z0-9_]*' needs to be quoted using "double quotes".
    If you need to include quote chars inside your strings, escape with a backslash ('\').

    All field references must be prefixed by the name of the data source they're coming from.

    Description in BNF (as used by the diagram generator at http://bottlecaps.de/rr/ui):

    ---------------------------------------------------------------------------
    UnionQuery ::= Query ( "union" Query )* ( OrderByList )?
    Query ::= FieldSelection DataSourceSelection ( FilterList )?

    FieldSelection ::= "select" AliasedFieldEquiv ( "," AliasedFieldEquiv )*
    AliasedFieldEquiv ::= FieldEquiv "as" Alias
    FieldEquiv ::= Function | Value | Field
    Function ::= Identifier "(" ( ListOfFunctionArguments )? ")"
    ListOfFunctionArguments ::= FieldEquiv ( ( "," FieldEquiv )* )?


    Field ::= DataSourceNameOrAlias "." FieldName
    DataSourceSelection ::= "from" DataSourceName ( "as" Alias )? ( JoinDefinition )*
    JoinDefinition ::= "join" DataSourceName ( "as" Alias )? "on" JoinCondition
    JoinCondition ::= SingleJoinCondition | "(" SingleJoinCondition ( "and" SingleJoinCondition )* ")"
    SingleJoinCondition ::= Field "=" Field
    DataSourceNameOrAlias ::= DataSourceName | Alias
    FilterList ::= "where" FilterAlternative
    FilterAlternative ::= FilterCombination ( "or" FilterCombination )*
    FilterCombination ::= FilterDefinition ( "and" FilterDefinition )*
    FilterDefinition ::= BinaryCondition | InCondition |
                       IsNullCondition | "(" FilterAlternative ")"
    BinaryCondition ::= FieldEquiv BinaryOperator FieldEquiv
    BinaryOperator ::= "=" | "!=" | "<" | "<=" | ">" | ">=" | "like"
    InCondition ::= FieldEquiv ( "not" )? "in" "(" Value ( "," Value )* ")"
    IsNullCondition ::= Field "is" ( "not" )? Null
    OrderByList ::= "order" "by" OrderByField ( "," OrderByField )*
    OrderByField ::= Field ( "asc" | "desc" )?
    Integer ::= ( "0" | "1" | "2" | "3" | "4" | "5" | "6" | "7" | "8" | "9" )*
    DataSourceName ::= Identifier
    FieldName ::= Identifier
    Alias ::= Identifier

    Value ::= SingleQuotedString | Number | Boolean | Null
    Number ::= Integer | Float
    Float ::= Integer "." Integer
    Boolean ::= "true" | "false"
    Null ::= "null"

    SingleQuotedString ::= "string in single quotes"
    DoubleQuotedString ::= "string in double quotes"

    Identifier ::= ( "a..z" | "A..Z" | "_" ) ( "a..z" | "A..Z" | "_" | Integer )* | DoubleQuotedString
    ---------------------------------------------------------------------------
    """

    def __init__(self):
        self.grammar = self.build_grammar()

    @classmethod
    def build_grammar(cls):
        escape_char = '\\'
        identifier = Regex(r'[a-zA-Z_][a-zA-Z0-9_]*') | QuotedString('"', escChar=escape_char)
        kw_and = CaselessKeyword('and')
        kw_or = CaselessKeyword('or')

        alias = CaselessKeyword('as') + identifier('alias')

        field = (identifier.copy()('prefix') + '.' + identifier.copy()('name') + NotAny('('))('field')
        field.setParseAction(Field.parse)

        aliased_field = field + Optional(alias)  # defaults to using name as alias
        aliased_field.setParseAction(AliasedField.parse)

        integer_value = Word(nums)
        number_value = Group(integer_value + Optional('.' + integer_value))
        number_value.setParseAction(Expression.parse_number)

        boolean_value = CaselessKeyword('true') | CaselessKeyword('false')
        boolean_value.setParseAction(Expression.parse_boolean)

        string_value = QuotedString('\'', escChar=escape_char)
        null_value = CaselessKeyword('null')
        null_value.setParseAction(Expression.parse_null)

        value = (number_value | boolean_value | string_value | null_value)('value')
        value.setParseAction(Expression.parse)

        aliased_value = (value + alias)
        aliased_value.setParseAction(AliasedExpression.parse)

        function = Forward()
        function << (identifier.copy()('name') +
                     '(' + Optional(delimitedList(field | value | function))('args') + ')')
        function.setParseAction(Function.parse)

        aliased_function = function('function') + alias
        aliased_function.setParseAction(AliasedFunction.parse)

        field_declaration = aliased_function | aliased_value | aliased_field

        field_declaration_list = Suppress(CaselessKeyword('select')) + delimitedList(field_declaration)

        data_source = identifier('name') + Optional(alias)
        data_source.setParseAction(DataSource.parse)

        join_single_condition = field('left') + '=' + field('right')
        join_single_condition.setParseAction(JoinCondition.parse)

        join_multi_condition = Literal('(') + delimitedList(join_single_condition, delim=kw_and)('conditions') + ')'
        join_multi_condition.setParseAction(JoinConditionList.parse)

        join_condition_list = join_single_condition | join_multi_condition

        join = (CaselessKeyword('join') + data_source.copy()('datasource') +
                CaselessKeyword('on') + join_condition_list.copy()('condition'))
        join.setParseAction(JoinedDataSource.parse)

        data_source_declaration = Suppress(CaselessKeyword('from')) + data_source + ZeroOrMore(join)

        condition_side = field | value | function

        operator = Literal('=') | '!=' | '<=' | '<' | '>=' | '>' | CaselessKeyword('like')
        operator.setParseAction(BinaryCondition.parse_operator)

        binary_condition = condition_side.copy()('left') + operator('operator') + condition_side.copy()('right')
        binary_condition.setParseAction(BinaryCondition.parse)

        in_condition = (condition_side.copy()('left') + Optional(CaselessKeyword('not'))('invert') +
                        CaselessKeyword('in') + '(' + Group(delimitedList(value))('in_list') + ')')
        in_condition.setParseAction(InCondition.parse)

        null_condition = (field + CaselessKeyword('is') + Optional(CaselessKeyword('not'))('invert') +
                          CaselessKeyword('null'))
        null_condition.setParseAction(IsNullCondition.parse)

        single_filter = Forward()

        filter_combination = delimitedList(single_filter, delim=kw_and)
        filter_combination.setParseAction(FilterCombination.parse)

        filter_alternative = delimitedList(filter_combination, delim=kw_or)
        filter_alternative.setParseAction(FilterAlternative.parse)

        single_filter << (null_condition | binary_condition | in_condition |
                          Suppress('(') + filter_alternative + Suppress(')'))

        filter_declaration = Suppress(CaselessKeyword('where')) + filter_alternative.copy()('conditions')

        order_by_field = field + Optional(Or([CaselessKeyword('asc'), CaselessKeyword('desc')]))('direction')
        order_by_field.setParseAction(OrderBy.parse)

        order_by_declaration = (Suppress(CaselessKeyword('order') + CaselessKeyword('by')) +
                                delimitedList(order_by_field)('fields'))
        order_by_declaration.setParseAction(Query.parse_order_by)

        query = (field_declaration_list('fields') + data_source_declaration('datasources') +
                 Optional(filter_declaration('filter')))
        query.setParseAction(Query.parse)

{'fraction_non_alphanumeric': 0.14830188679245282, 'fraction_numerical': 0.001761006289308176, 'mean_word_length': 3.303382949932341, 'pattern_counts': {'":': 0, '<': 19, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1669684', 'n_tokens_mistral': 4616, 'n_tokens_neox': 4459, 'n_words': 861}