<commit_before>import pdb
import re
import ply.lex as lex
from core.search.compiler.invfilter import *


class InvLexer(object):

    tokens = (
        'TEXT', 'DIRECTIVE', 'RE',
        'AND', 'NOT', 'OR',
        'LPAREN', 'RPAREN',
    )

    # Tokens
    t_LPAREN = r'\('
    t_RPAREN = r'\)'

    def t_NOT(self, t):
        r'\s*-'
        t.value = 'NOT'
        return t

    def t_AND(self, t):
        r'AND'
        return t

    def t_OR(self, t):
        r'OR'
        return t

    def t_DIRECTIVE(self, t):
        r'([a-zA-Z_]+)=:([a-zA-Z0-9_\./]+)'
        r = r'([a-zA-Z_]+)=:([a-zA-Z0-9_\./]+)'
        match = re.compile(r).match(t.value)
        directive = match.groups(1)[0]
        dvalue = match.groups(1)[1]
        t.value = DirectiveFilter(t.value, directive, dvalue)
        return t

    def t_RE(self, t):
        r'/\S+'
        r = r'/(\S+)'
        match = re.compile(r).match(t.value)
        rvalue = match.groups(1)[0]
        t.value = REFilter(rvalue)
        return t

    def t_TEXT(self, t):
        r'[a-zA-Z0-9_\.-]+'
        t.value = TextFilter(t.value)
        return t

    # Ignored characters
    t_ignore = " \t"

    def t_newline(self, t):
        r'\n+'
        t.lexer.lineno += t.value.count("\n")

    def t_error(self, t):
        raise SyntaxError("Illegal character '%s'" % t.value[0])

    # Build the lexer
    def build_lexer(self, **kwargs):
        self.lexer = lex.lex(module=self, **kwargs)<commit_msg>small bug preventing searches for ipv6 addresses<commit_after>import pdb
import re
import ply.lex as lex
from core.search.compiler.invfilter import *


class InvLexer(object):

    tokens = (
        'TEXT', 'DIRECTIVE', 'RE',
        'AND', 'NOT', 'OR',
        'LPAREN', 'RPAREN',
    )

    # Tokens
    t_LPAREN = r'\('
    t_RPAREN = r'\)'

    def t_NOT(self, t):
        r'\s*-'
        t.value = 'NOT'
        return t

    def t_AND(self, t):
        r'AND'
        return t

    def t_OR(self, t):
        r'OR'
        return t

    def t_DIRECTIVE(self, t):
        r'([a-zA-Z_]+)=:([a-zA-Z0-9_\./]+)'
        r = r'([a-zA-Z_]+)=:([a-zA-Z0-9_\./:]+)'
        match = re.compile(r).match(t.value)
        directive = match.groups(1)[0]
        dvalue = match.groups(1)[1]
        t.value = DirectiveFilter(t.value, directive, dvalue)
        return t

    def t_RE(self, t):
        r'/\S+'
        r = r'/(\S+)'
        match = re.compile(r).match(t.value)
        rvalue = match.groups(1)[0]
        t.value = REFilter(rvalue)
        return t

    def t_TEXT(self, t):
        r'[a-zA-Z0-9_\.-:]+'
        t.value = TextFilter(t.value)
        return t

    # Ignored characters
    t_ignore = " \t"

    def t_newline(self, t):
        r'\n+'
        t.lexer.lineno += t.value.count("\n")

    def t_error(self, t):
        raise SyntaxError("Illegal character '%s'" % t.value[0])

    # Build the lexer
    def build_lexer(self, **kwargs):
        self.lexer = lex.lex(module=self, **kwargs)
{'fraction_non_alphanumeric': 0.17534430634867315, 'fraction_numerical': 0.009069533087000335, 'mean_word_length': 1.8497607655502393, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5806415', 'n_tokens_mistral': 1176, 'n_tokens_neox': 1130, 'n_words': 233}