<commit_before>import operator
import ipaddr
import pdb

from django.core.exceptions import ObjectDoesNotExist, MultipleObjectsReturned
from django.db.models import Q

from mozdns.address_record.models import AddressRecord
from mozdns.cname.models import CNAME
from mozdns.domain.models import Domain
from mozdns.mx.models import MX
from mozdns.nameserver.models import Nameserver
from mozdns.ptr.models import PTR
from mozdns.srv.models import SRV
from mozdns.soa.models import SOA
from mozdns.sshfp.models import SSHFP
from mozdns.txt.models import TXT
from mozdns.view.models import View

from core.interface.static_intr.models import StaticInterface
from core.site.models import Site
from core.utils import two_to_four, IPFilter
from core.utils import start_end_filter, one_to_two, two_to_one

from core.vlan.models import Vlan

from systems.models import System


searchables = (
        ('A', AddressRecord),
        ('CNAME', CNAME),
        ('DOMAIN', Domain),
        ('MX', MX),
        ('NS', Nameserver),
        ('PTR', PTR),
        ('SOA', SOA),
        ('SRV', SRV),
        ('SSHFP', SSHFP),
        ('INTR', StaticInterface),
        ('SYSTEM', System),
        ('TXT', TXT),
)

def get_managers():
    managers = []
    for name, Klass in searchables:
        if name == 'System':
            managers.append(Klass.objectsselect_related('server_model',
                            'system_rack__location'))
        else:
            managers.append(Klass.objects)
    return managers

class _Filter(object):
    """The Base class of different filters. Implement these methods
    """
    ntype = "FILTER"
    def __str__(self):
        return self.value

    def __repr__(self):
        return "<{1}>".format(self.__class__, self)

    def compile_Q(self, ntype):
        pass


def build_filter(filter_, fields, filter_type):
    # rtucker++
    final_filter = Q()
    for t in fields:
        final_filter = final_filter | Q(**{"{0}__{1}".format(t,
            filter_type): filter_})

    return final_filter


class TextFilter(_Filter):
    def __init__(self, rvalue):
        self.value = rvalue
        self.Q = self.compile_Q(rvalue)

    def compile_Q(self, value):
        # Value is the search term
        result = []
        for name, Klass in searchables:
            result.append(build_filter(value, Klass.search_fields, 'icontains'))
        return result


class REFilter(TextFilter):
    def compile_Q(self, value):
        result = []
        for name, Klass in searchables:
            result.append(build_filter(value, Klass.search_fields, 'regex'))
        return result


class DirectiveFilter(_Filter):
    def __init__(self, rvalue, directive, dvalue):
        self.value = rvalue
        self.directive = directive
        self.dvalue = dvalue
        self.Q = self.compile_Q(directive, dvalue)

    def compile_Q(self, directive, dvalue):
        if directive == 'view':
            return build_view_qsets(dvalue)
        elif directive == 'network':
            return build_network_qsets(dvalue)
        elif directive == 'vlan':
            return build_vlan_qsets(dvalue)
        elif directive == 'zone':
            return build_zone_qsets(dvalue)
        elif directive == 'range':
            return build_range_qsets(dvalue)
        elif directive == 'type':
            return build_rdtype_qsets(dvalue)
        elif directive == 'site':
            return build_site_qsets(dvalue)
        else:
            raise BadDirective("Unknown Directive '{0}'".format(directive))
<commit_msg>smart regex filter for matching numbers https://bugzilla.mozilla.org/show_bug.cgi?id=805637<commit_after>import operator
import ipaddr
import pdb

from django.core.exceptions import ObjectDoesNotExist, MultipleObjectsReturned
from django.db.models import Q

from mozdns.address_record.models import AddressRecord
from mozdns.cname.models import CNAME
from mozdns.domain.models import Domain
from mozdns.mx.models import MX
from mozdns.nameserver.models import Nameserver
from mozdns.ptr.models import PTR
from mozdns.srv.models import SRV
from mozdns.soa.models import SOA
from mozdns.sshfp.models import SSHFP
from mozdns.txt.models import TXT
from mozdns.view.models import View

from core.interface.static_intr.models import StaticInterface
from core.site.models import Site
from core.utils import two_to_four, IPFilter
from core.utils import start_end_filter, one_to_two, two_to_one

from core.vlan.models import Vlan

from systems.models import System
import re


searchables = (
        ('A', AddressRecord),
        ('CNAME', CNAME),
        ('DOMAIN', Domain),
        ('MX', MX),
        ('NS', Nameserver),
        ('PTR', PTR),
        ('SOA', SOA),
        ('SRV', SRV),
        ('SSHFP', SSHFP),
        ('INTR', StaticInterface),
        ('SYSTEM', System),
        ('TXT', TXT),
)

def get_managers():
    managers = []
    for name, Klass in searchables:
        if name == 'System':
            managers.append(Klass.objectsselect_related('server_model',
                            'system_rack__location'))
        else:
            managers.append(Klass.objects)
    return managers

class _Filter(object):
    """The Base class of different filters. Implement these methods
    """
    ntype = "FILTER"
    def __str__(self):
        return self.value

    def __repr__(self):
        return "<{1}>".format(self.__class__, self)

    def compile_Q(self, ntype):
        pass


def build_filter(filter_, fields, filter_type):
    # rtucker++
    final_filter = Q()
    for t in fields:
        final_filter = final_filter | Q(**{"{0}__{1}".format(t,
            filter_type): filter_})

    return final_filter


class TextFilter(_Filter):
    def __init__(self, rvalue):
        self.value = rvalue
        self.Q = self.compile_Q(rvalue)

    def compile_Q(self, value):
        # Value is the search term
        result = []
        for name, Klass in searchables:
            result.append(build_filter(value, Klass.search_fields, 'icontains'))
        return result


class REFilter(TextFilter):
    num_match = re.compile("\[(\d+)-(\d+)\]")

    def _expand_number_regex(self, value):
        """We want to turn something like /hp-node[31-40].phx1 into
            '/hp-node(31|32|33|34|35|36|37|38|39|40).phx1'"""
        matches = self.num_match.findall(value)
        for low, high in matches:
            if int(low) >= int(high):
                continue
            new_value = ""
            for i in xrange(int(low), int(high) + 1):
                new_value += "{0}|".format(i)
            new_value = '(' + new_value.strip('|') + ')'
            value = value.replace('[{0}-{1}]'.format(low, high), new_value)
        return value

    def compile_Q(self, value):
        result = []
        value = self._expand_number_regex(value)
        for name, Klass in searchables:
            result.append(build_filter(value, Klass.search_fields, 'regex'))
        return result


class DirectiveFilter(_Filter):
    def __init__(self, rvalue, directive, dvalue):
        self.value = rvalue
        self.directive = directive
        self.dvalue = dvalue
        self.Q = self.compile_Q(directive, dvalue)

    def compile_Q(self, directive, dvalue):
        if directive == 'view':
            return build_view_qsets(dvalue)
        elif directive == 'network':
            return build_network_qsets(dvalue)
        elif directive == 'vlan':
            return build_vlan_qsets(dvalue)
        elif directive == 'zone':
            return build_zone_qsets(dvalue)
        elif directive == 'range':
            return build_range_qsets(dvalue)
        elif directive == 'type':
            return build_rdtype_qsets(dvalue)
        elif directive == 'site':
            return build_site_qsets(dvalue)
        else:
            raise BadDirective("Unknown Directive '{0}'".format(directive))

{'fraction_non_alphanumeric': 0.10742435799644037, 'fraction_numerical': 0.0055936943808797355, 'mean_word_length': 2.648886827458256, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 6, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2866127', 'n_tokens_mistral': 2428, 'n_tokens_neox': 2338, 'n_words': 604}