<commit_before>#
# Copyright (C) 2009-2018 the sqlparse authors and contributors
# <see AUTHORS file>
#
# This module is part of python-sqlparse and is released under
# the BSD License: https://opensource.org/licenses/BSD-3-Clause

import re

from sqlparse import tokens


def is_keyword(value):
    val = value.upper()
    return (KEYWORDS_COMMON.get(val)
            or KEYWORDS_ORACLE.get(val)
            or KEYWORDS_PLPGSQL.get(val)
            or KEYWORDS_HQL.get(val)
            or KEYWORDS.get(val, tokens.Name)), value


SQL_REGEX = {
    'root': [
        (r'(--|# )\+.*?(\r\n|\r|\n|$)', tokens.Comment.Single.Hint),
        (r'/\*\+[\s\S]*?\*/', tokens.Comment.Multiline.Hint),

        (r'(--|# ).*?(\r\n|\r|\n|$)', tokens.Comment.Single),
        (r'/\*[\s\S]*?\*/', tokens.Comment.Multiline),

        (r'(\r\n|\r|\n)', tokens.Newline),
        (r'\s+?', tokens.Whitespace),

        (r':=', tokens.Assignment),
        (r'::', tokens.Punctuation),

        (r'\*', tokens.Wildcard),

        (r"`(``|[^`])*`", tokens.Name),
        (r"´(´´|[^´])*´", tokens.Name),
        (r'((?<!\S)\$(?:[_A-ZÀ-Ü]\w*)?\$)[\s\S]*?\1', tokens.Literal),

        (r'\?', tokens.Name.Placeholder),
        (r'%(\(\w+\))?s', tokens.Name.Placeholder),
        (r'(?<!\w)[$:?]\w+', tokens.Name.Placeholder),

        (r'\\\w+', tokens.Command),
        (r'(NOT\s+)?(IN)\b', tokens.Operator.Comparison),
        # FIXME(andi): VALUES shouldn't be listed here
        # see https://github.com/andialbrecht/sqlparse/pull/64
        # AS and IN are special, it may be followed by a parenthesis, but
        # are never functions, see issue183 and issue507
        (r'(CASE|IN|VALUES|USING|FROM|AS)\b', tokens.Keyword),

        (r'(@|##|#)[A-ZÀ-Ü]\w+', tokens.Name),

        # see issue #39
        # Spaces around period `schema . name` are valid identifier
        # TODO: Spaces before period not implemented
        (r'[A-ZÀ-Ü]\w*(?=\s*\.)', tokens.Name),  # 'Name'   .
        # FIXME(atronah): never match,
        # because `re.match` doesn't work with look-behind regexp feature
        (r'(?<=\.)[A-ZÀ-Ü]\w*', tokens.Name),  # .'Name'
        (r'[A-ZÀ-Ü]\w*(?=\()', tokens.Name),  # side effect: change kw to func
        (r'-?0x[\dA-F]+', tokens.Number.Hexadecimal),
        (r'-?\d*(\.\d+)?E-?\d+', tokens.Number.Float),
        (r'(?![_A-ZÀ-Ü])-?(\d+(\.\d*)|\.\d+)(?![_A-ZÀ-Ü])',
         tokens.Number.Float),
        (r'(?![_A-ZÀ-Ü])-?\d+(?![_A-ZÀ-Ü])', tokens.Number.Integer),
        (r"'(''|\\\\|\\'|[^'])*'", tokens.String.Single),
        # not a real string literal in ANSI SQL:
        (r'"(""|\\\\|\\"|[^"])*"', tokens.String.Symbol),
        (r'(""|".*?[^\\]")', tokens.String.Symbol),
        # sqlite names can be escaped with [square brackets]. left bracket
        # cannot be preceded by word character or a right bracket --
        # otherwise it's probably an array index
        (r'(?<![\w\])])(\[[^\]]+\])', tokens.Name),
        (r'((LEFT\s+|RIGHT\s+|FULL\s+)?(INNER\s+|OUTER\s+|STRAIGHT\s+)?'
         r'|(CROSS\s+|NATURAL\s+)?)?JOIN\b', tokens.Keyword),
        (r'END(\s+IF|\s+LOOP|\s+WHILE)?\b', tokens.Keyword),
        (r'NOT\s+NULL\b', tokens.Keyword),
        (r'NULLS\s+(FIRST|LAST)\b', tokens.Keyword),
        (r'UNION\s+ALL\b', tokens.Keyword),
        (r'CREATE(\s+OR\s+REPLACE)?\b', tokens.Keyword.DDL),
        (r'DOUBLE\s+PRECISION\b', tokens.Name.Builtin),
        (r'GROUP\s+BY\b', tokens.Keyword),
        (r'ORDER\s+BY\b', tokens.Keyword),
        (r'(LATERAL\s+VIEW\s+)'
         r'(EXPLODE|INLINE|PARSE_URL_TUPLE|POSEXPLODE|STACK)\b',
         tokens.Keyword),
        (r"(AT|WITH')\s+TIME\s+ZONE\s+'[^']+'", tokens.Keyword.TZCast),
        (r'(NOT\s+)?(LIKE|ILIKE)\b', tokens.Operator.Comparison),<commit_msg>Add RLIKE keyword<commit_after>#
# Copyright (C) 2009-2018 the sqlparse authors and contributors
# <see AUTHORS file>
#
# This module is part of python-sqlparse and is released under
# the BSD License: https://opensource.org/licenses/BSD-3-Clause

import re

from sqlparse import tokens


def is_keyword(value):
    val = value.upper()
    return (KEYWORDS_COMMON.get(val)
            or KEYWORDS_ORACLE.get(val)
            or KEYWORDS_PLPGSQL.get(val)
            or KEYWORDS_HQL.get(val)
            or KEYWORDS.get(val, tokens.Name)), value


SQL_REGEX = {
    'root': [
        (r'(--|# )\+.*?(\r\n|\r|\n|$)', tokens.Comment.Single.Hint),
        (r'/\*\+[\s\S]*?\*/', tokens.Comment.Multiline.Hint),

        (r'(--|# ).*?(\r\n|\r|\n|$)', tokens.Comment.Single),
        (r'/\*[\s\S]*?\*/', tokens.Comment.Multiline),

        (r'(\r\n|\r|\n)', tokens.Newline),
        (r'\s+?', tokens.Whitespace),

        (r':=', tokens.Assignment),
        (r'::', tokens.Punctuation),

        (r'\*', tokens.Wildcard),

        (r"`(``|[^`])*`", tokens.Name),
        (r"´(´´|[^´])*´", tokens.Name),
        (r'((?<!\S)\$(?:[_A-ZÀ-Ü]\w*)?\$)[\s\S]*?\1', tokens.Literal),

        (r'\?', tokens.Name.Placeholder),
        (r'%(\(\w+\))?s', tokens.Name.Placeholder),
        (r'(?<!\w)[$:?]\w+', tokens.Name.Placeholder),

        (r'\\\w+', tokens.Command),
        (r'(NOT\s+)?(IN)\b', tokens.Operator.Comparison),
        # FIXME(andi): VALUES shouldn't be listed here
        # see https://github.com/andialbrecht/sqlparse/pull/64
        # AS and IN are special, it may be followed by a parenthesis, but
        # are never functions, see issue183 and issue507
        (r'(CASE|IN|VALUES|USING|FROM|AS)\b', tokens.Keyword),

        (r'(@|##|#)[A-ZÀ-Ü]\w+', tokens.Name),

        # see issue #39
        # Spaces around period `schema . name` are valid identifier
        # TODO: Spaces before period not implemented
        (r'[A-ZÀ-Ü]\w*(?=\s*\.)', tokens.Name),  # 'Name'   .
        # FIXME(atronah): never match,
        # because `re.match` doesn't work with look-behind regexp feature
        (r'(?<=\.)[A-ZÀ-Ü]\w*', tokens.Name),  # .'Name'
        (r'[A-ZÀ-Ü]\w*(?=\()', tokens.Name),  # side effect: change kw to func
        (r'-?0x[\dA-F]+', tokens.Number.Hexadecimal),
        (r'-?\d*(\.\d+)?E-?\d+', tokens.Number.Float),
        (r'(?![_A-ZÀ-Ü])-?(\d+(\.\d*)|\.\d+)(?![_A-ZÀ-Ü])',
         tokens.Number.Float),
        (r'(?![_A-ZÀ-Ü])-?\d+(?![_A-ZÀ-Ü])', tokens.Number.Integer),
        (r"'(''|\\\\|\\'|[^'])*'", tokens.String.Single),
        # not a real string literal in ANSI SQL:
        (r'"(""|\\\\|\\"|[^"])*"', tokens.String.Symbol),
        (r'(""|".*?[^\\]")', tokens.String.Symbol),
        # sqlite names can be escaped with [square brackets]. left bracket
        # cannot be preceded by word character or a right bracket --
        # otherwise it's probably an array index
        (r'(?<![\w\])])(\[[^\]]+\])', tokens.Name),
        (r'((LEFT\s+|RIGHT\s+|FULL\s+)?(INNER\s+|OUTER\s+|STRAIGHT\s+)?'
         r'|(CROSS\s+|NATURAL\s+)?)?JOIN\b', tokens.Keyword),
        (r'END(\s+IF|\s+LOOP|\s+WHILE)?\b', tokens.Keyword),
        (r'NOT\s+NULL\b', tokens.Keyword),
        (r'NULLS\s+(FIRST|LAST)\b', tokens.Keyword),
        (r'UNION\s+ALL\b', tokens.Keyword),
        (r'CREATE(\s+OR\s+REPLACE)?\b', tokens.Keyword.DDL),
        (r'DOUBLE\s+PRECISION\b', tokens.Name.Builtin),
        (r'GROUP\s+BY\b', tokens.Keyword),
        (r'ORDER\s+BY\b', tokens.Keyword),
        (r'(LATERAL\s+VIEW\s+)'
         r'(EXPLODE|INLINE|PARSE_URL_TUPLE|POSEXPLODE|STACK)\b',
         tokens.Keyword),
        (r"(AT|WITH')\s+TIME\s+ZONE\s+'[^']+'", tokens.Keyword.TZCast),
        (r'(NOT\s+)?(LIKE|ILIKE|RLIKE)\b', tokens.Operator.Comparison),
{'fraction_non_alphanumeric': 0.25013297872340423, 'fraction_numerical': 0.005585106382978724, 'mean_word_length': 2.509566028931405, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 5, 'https://': 4, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '769483', 'n_tokens_mistral': 2993, 'n_tokens_neox': 2920, 'n_words': 475}