<commit_before>"""
Search query parser

Influenced by http://pyparsing.wikispaces.com/file/view/searchparser.py
"""
import unicodedata

from pyparsing import (
    Forward,
    Group,
    Keyword,
    Literal,
    oneOf,
    OneOrMore,
    Suppress,
    Word,
)


def is_alphanumeric(c):
    return unicodedata.category(c) in ['Lu', 'Ll', 'Nd']


all_unicode = u''.join(unichr(c) for c in xrange(65536))
unicode_alnum = ''.join(
    c for c in all_unicode
    if is_alphanumeric(c)
)
unicode_non_alnum = ''.join(
    c for c in all_unicode
    if not is_alphanumeric(c) and
    c not in ['-', '(', ')']
)


class SearchQueryParser(object):
    def __init__(self, wildcard=':*'):
        self.wildcard = wildcard
        self._methods = {
            'and': self.eval_and,
            'not': self.eval_not,
            'or': self.eval_or,<commit_msg>Made search parser py3 compatible<commit_after>"""
Search query parser

Influenced by http://pyparsing.wikispaces.com/file/view/searchparser.py
"""
import unicodedata
import six
from pyparsing import (
    Forward,
    Group,
    Keyword,
    Literal,
    oneOf,
    OneOrMore,
    Suppress,
    Word,
)


def is_alphanumeric(c):
    return unicodedata.category(c) in ['Lu', 'Ll', 'Nd']


all_unicode = u''.join(unichr(c) for c in six.moves.range(65536))
unicode_alnum = ''.join(
    c for c in all_unicode
    if is_alphanumeric(c)
)
unicode_non_alnum = ''.join(
    c for c in all_unicode
    if not is_alphanumeric(c) and
    c not in ['-', '(', ')']
)


class SearchQueryParser(object):
    def __init__(self, wildcard=':*'):
        self.wildcard = wildcard
        self._methods = {
            'and': self.eval_and,
            'not': self.eval_not,
            'or': self.eval_or,
{'fraction_non_alphanumeric': 0.1335636154289004, 'fraction_numerical': 0.0063327576280944155, 'mean_word_length': 2.8451327433628317, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5193580', 'n_tokens_mistral': 659, 'n_tokens_neox': 633, 'n_words': 147}