<commit_before>#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.

import jsonpath_rw
from jsonpath_rw import lexer
from jsonpath_rw import parser

from jsonpath_rw_ext import _filter
from jsonpath_rw_ext import _iterable

# NOTE(sileht): This block is very important otherwise py3X tests fail no joke
# ply/yacc.py order functions by line, then by module, but in py3 module are
# not sortable, so we add this block to not have methods defined at the same
# line in jsonpath_rw and jsonpath_rw_ext, yes that really sucks ...


class ExtendedJsonPathLexer(lexer.JsonPathLexer):
    """Custom LALR-lexer for JsonPath"""
    literals = lexer.JsonPathLexer.literals + ['?', '@']
    tokens = (parser.JsonPathLexer.tokens +
              ['FILTER_OP', 'FILTER_VALUE', 'SORT_DIRECTION'])

    t_FILTER_VALUE = r'\w+'
    t_FILTER_OP = r'(==?|<=|>=|!=|<|>)'

    def t_SORT_DIRECTION(self, t):
        r',?\s*(/|\\)'
        t.value = t.value[-1]
        return t

    def t_ID(self, t):
        r'@?[a-zA-Z_][a-zA-Z0-9_@\-]*'
        # NOTE(sileht): This fixes the ID expression to be
        # able to use @ for `This` like any json query
        t.type = self.reserved_words.get(t.value, 'ID')
        return t


class ExtentedJsonPathParser(parser.JsonPathParser):
    """Custom LALR-parser for JsonPath"""

    tokens = ExtendedJsonPathLexer.tokens

    def __init__(self, debug=False, lexer_class=None):
        lexer_class = lexer_class or ExtendedJsonPathLexer
        super(ExtentedJsonPathParser, self).__init__(debug, lexer_class)

    def p_jsonpath_named_operator(self, p):
        "jsonpath : NAMED_OPERATOR"
        if p[1] == 'len':
            p[0] = _iterable.Len()
        elif p[1] == 'sorted':
            p[0] = _iterable.SortedThis()
        else:
            super(ExtentedJsonPathParser, self).p_jsonpath_named_operator(p)

    def p_expression(self, p):
        """expression : jsonpath
                      | jsonpath FILTER_OP FILTER_VALUE
                      | jsonpath FILTER_OP ID
                      | jsonpath FILTER_OP NUMBER
        """
        if len(p) == 2:
            left, op, right = p[1], None, None
        else:
            __, left, op, right = p
        p[0] = _filter.Expression(left, op, right)

    def p_expressions_expression(self, p):
        "expressions : expression"
        p[0] = [p[1]]

    def p_expressions_and(self, p):
        "expressions : expressions '&' expressions"
        # TODO(sileht): implements '|'
        p[0] = p[1] + p[3]

    def p_expressions_parens(self, p):
        "expressions : '(' expressions ')'"
        p[0] = p[2]

    def p_filter(self, p):
        "filter : '?' expressions "
        p[0] = _filter.Filter(p[2])

    def p_jsonpath_filter(self, p):
        "jsonpath : jsonpath '[' filter ']'"
        p[0] = jsonpath_rw.Child(p[1], p[3])

    def p_sort(self, p):
        "sort : SORT_DIRECTION jsonpath"
        p[0] = (p[2], p[1] != "/")

    def p_sorts_sort(self, p):
        "sorts : sort"
        p[0] = [p[1]]

    def p_sorts_comma(self, p):
        "sorts : sorts sorts"
        p[0] = p[1] + p[2]

    def p_jsonpath_sort(self, p):
        "jsonpath : jsonpath '[' sorts ']'"
        sort = _iterable.SortedThis(p[3])
        p[0] = jsonpath_rw.Child(p[1], sort)

    def p_jsonpath_this(self, p):
        "jsonpath : '@'"
        p[0] = jsonpath_rw.This()

    precedence = ([
    ] + parser.JsonPathParser.precedence)


def parse(path, debug=False):
    return ExtentedJsonPathParser(debug=debug).parse(path)<commit_msg>Remove useless code<commit_after>#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.

import jsonpath_rw
from jsonpath_rw import lexer
from jsonpath_rw import parser

from jsonpath_rw_ext import _filter
from jsonpath_rw_ext import _iterable

# NOTE(sileht): This block is very important otherwise py3X tests fail no joke
# ply/yacc.py order functions by line, then by module, but in py3 module are
# not sortable, so we add this block to not have methods defined at the same
# line in jsonpath_rw and jsonpath_rw_ext, yes that really sucks ...


class ExtendedJsonPathLexer(lexer.JsonPathLexer):
    """Custom LALR-lexer for JsonPath"""
    literals = lexer.JsonPathLexer.literals + ['?', '@']
    tokens = (parser.JsonPathLexer.tokens +
              ['FILTER_OP', 'FILTER_VALUE', 'SORT_DIRECTION'])

    t_FILTER_VALUE = r'\w+'
    t_FILTER_OP = r'(==?|<=|>=|!=|<|>)'

    def t_SORT_DIRECTION(self, t):
        r',?\s*(/|\\)'
        t.value = t.value[-1]
        return t

    def t_ID(self, t):
        r'@?[a-zA-Z_][a-zA-Z0-9_@\-]*'
        # NOTE(sileht): This fixes the ID expression to be
        # able to use @ for `This` like any json query
        t.type = self.reserved_words.get(t.value, 'ID')
        return t


class ExtentedJsonPathParser(parser.JsonPathParser):
    """Custom LALR-parser for JsonPath"""

    tokens = ExtendedJsonPathLexer.tokens

    def __init__(self, debug=False, lexer_class=None):
        lexer_class = lexer_class or ExtendedJsonPathLexer
        super(ExtentedJsonPathParser, self).__init__(debug, lexer_class)

    def p_jsonpath_named_operator(self, p):
        "jsonpath : NAMED_OPERATOR"
        if p[1] == 'len':
            p[0] = _iterable.Len()
        elif p[1] == 'sorted':
            p[0] = _iterable.SortedThis()
        else:
            super(ExtentedJsonPathParser, self).p_jsonpath_named_operator(p)

    def p_expression(self, p):
        """expression : jsonpath
                      | jsonpath FILTER_OP FILTER_VALUE
                      | jsonpath FILTER_OP ID
                      | jsonpath FILTER_OP NUMBER
        """
        if len(p) == 2:
            left, op, right = p[1], None, None
        else:
            __, left, op, right = p
        p[0] = _filter.Expression(left, op, right)

    def p_expressions_expression(self, p):
        "expressions : expression"
        p[0] = [p[1]]

    def p_expressions_and(self, p):
        "expressions : expressions '&' expressions"
        # TODO(sileht): implements '|'
        p[0] = p[1] + p[3]

    def p_expressions_parens(self, p):
        "expressions : '(' expressions ')'"
        p[0] = p[2]

    def p_filter(self, p):
        "filter : '?' expressions "
        p[0] = _filter.Filter(p[2])

    def p_jsonpath_filter(self, p):
        "jsonpath : jsonpath '[' filter ']'"
        p[0] = jsonpath_rw.Child(p[1], p[3])

    def p_sort(self, p):
        "sort : SORT_DIRECTION jsonpath"
        p[0] = (p[2], p[1] != "/")

    def p_sorts_sort(self, p):
        "sorts : sort"
        p[0] = [p[1]]

    def p_sorts_comma(self, p):
        "sorts : sorts sorts"
        p[0] = p[1] + p[2]

    def p_jsonpath_sort(self, p):
        "jsonpath : jsonpath '[' sorts ']'"
        sort = _iterable.SortedThis(p[3])
        p[0] = jsonpath_rw.Child(p[1], sort)

    def p_jsonpath_this(self, p):
        "jsonpath : '@'"
        p[0] = jsonpath_rw.This()


def parse(path, debug=False):
    return ExtentedJsonPathParser(debug=debug).parse(path)
{'fraction_non_alphanumeric': 0.12479639142964541, 'fraction_numerical': 0.010023806540533768, 'mean_word_length': 2.6480804387568555, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4860156', 'n_tokens_mistral': 2733, 'n_tokens_neox': 2628, 'n_words': 711}