<commit_before>import re
from itertools import chain

from django.core.exceptions import ObjectDoesNotExist
from django.db.models.query_utils import Q

LOOKUP_SEP = '__'

QUERY_TERMS = set([
    'exact', 'iexact', 'contains', 'icontains', 'gt', 'gte', 'lt', 'lte', 'in',
    'startswith', 'istartswith', 'endswith', 'iendswith', 'range', 'year',
    'month', 'day', 'week_day', 'isnull', 'search', 'regex', 'iregex',
])


def eval_wrapper(children):
    """
    generator to yield child nodes, or to wrap filter expressions
    """
    for child in children:
        if isinstance(child, P):
            yield child
        elif isinstance(child, tuple) and len(child) == 2:
            yield LookupExpression(child)


class P(Q):
    """
    A Django 'predicate' construct

    This is a variation on Q objects, but instead of being used to generate
    SQL, they are used to test a model instance against a set of conditions.
    """

    # allow the use of the 'in' operator for membership testing
    def __contains__(self, obj):
        # TODO: This overrides Q's __contains__ method. It should only have
        # the custom behavior for non-Node objects.
        return self.eval(obj)

    def eval(self, instance):
        """
        Returns true if the model instance matches this predicate
        """
        evaluators = {"AND": all, "OR": any}
        evaluator = evaluators[self.connector]
        ret = evaluator(c.eval(instance) for c in eval_wrapper(self.children))
        if self.negated:
            return not ret
        else:
            return ret

    def to_identifier(self):
        s = ""
        for c in sorted(self.children):
            if isinstance(c, type(self)):
                s += c.to_identifier()
            else:
                s += ''.join([str(val) for val in c])
        return s.replace('_', '')


class LookupExpression(object):
    """
    A thin wrapper around a filter expression tuple of (lookup-type, value) to
    provide an eval method
    """

    def __init__(self, expr):
        self.lookup, self.value = expr
        self.field = None

    def get_field_from_objs(self, lookup_name, objs):
        values = []
        for obj in objs:
            if obj is None:
                values.append(None)
                continue
            field = obj._meta.get_field(lookup_name)
            direct = not field.auto_created or field.concrete
            accessor = lookup_name if direct else field.get_accessor_name()
            try:
                result = getattr(obj, accessor)
            except ObjectDoesNotExist:
                values.append(None)
            else:
                if hasattr(result, 'all'):
                    values.extend(result.all())
                else:
                    values.append(result)
        return values

    def get_field(self, instance):
        parts = self.lookup.split(LOOKUP_SEP)

        lookup_type = 'exact'  # Default lookup type
        if parts[-1] in QUERY_TERMS:
            lookup_type = parts.pop()
        values = [instance]
        for part in parts:
            values = self.get_field_from_objs(part, values)
        return instance, values, lookup_type
<commit_msg>Fix django 1.7 compatibility<commit_after>import re

import django
from django.core.exceptions import ObjectDoesNotExist
from django.db.models.query_utils import Q

LOOKUP_SEP = '__'

QUERY_TERMS = set([
    'exact', 'iexact', 'contains', 'icontains', 'gt', 'gte', 'lt', 'lte', 'in',
    'startswith', 'istartswith', 'endswith', 'iendswith', 'range', 'year',
    'month', 'day', 'week_day', 'isnull', 'search', 'regex', 'iregex',
])


def eval_wrapper(children):
    """
    generator to yield child nodes, or to wrap filter expressions
    """
    for child in children:
        if isinstance(child, P):
            yield child
        elif isinstance(child, tuple) and len(child) == 2:
            yield LookupExpression(child)


class P(Q):
    """
    A Django 'predicate' construct

    This is a variation on Q objects, but instead of being used to generate
    SQL, they are used to test a model instance against a set of conditions.
    """

    # allow the use of the 'in' operator for membership testing
    def __contains__(self, obj):
        # TODO: This overrides Q's __contains__ method. It should only have
        # the custom behavior for non-Node objects.
        return self.eval(obj)

    def eval(self, instance):
        """
        Returns true if the model instance matches this predicate
        """
        evaluators = {"AND": all, "OR": any}
        evaluator = evaluators[self.connector]
        ret = evaluator(c.eval(instance) for c in eval_wrapper(self.children))
        if self.negated:
            return not ret
        else:
            return ret

    def to_identifier(self):
        s = ""
        for c in sorted(self.children):
            if isinstance(c, type(self)):
                s += c.to_identifier()
            else:
                s += ''.join([str(val) for val in c])
        return s.replace('_', '')


class LookupExpression(object):
    """
    A thin wrapper around a filter expression tuple of (lookup-type, value) to
    provide an eval method
    """

    def __init__(self, expr):
        self.lookup, self.value = expr
        self.field = None

    def get_field_from_objs(self, lookup_name, objs):
        values = []
        for obj in objs:
            if obj is None:
                values.append(None)
                continue
            if django.VERSION < (1, 8):
                field, model, direct,  m2m = obj._meta.get_field_by_name(
                    lookup_name)
            else:
                field = obj._meta.get_field(lookup_name)
                direct = not field.auto_created or field.concrete
            accessor = lookup_name if direct else field.get_accessor_name()
            try:
                result = getattr(obj, accessor)
            except ObjectDoesNotExist:
                values.append(None)
            else:
                if hasattr(result, 'all'):
                    values.extend(result.all())
                else:
                    values.append(result)
        return values

    def get_field(self, instance):
        parts = self.lookup.split(LOOKUP_SEP)

        lookup_type = 'exact'  # Default lookup type
        if parts[-1] in QUERY_TERMS:
            lookup_type = parts.pop()
        values = [instance]
        for part in parts:
            values = self.get_field_from_objs(part, values)
        return instance, values, lookup_type

{'fraction_non_alphanumeric': 0.10024375380865326, 'fraction_numerical': 0.0013711151736745886, 'mean_word_length': 2.0421686746987953, 'pattern_counts': {'":': 4, '<': 4, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1167269', 'n_tokens_mistral': 1882, 'n_tokens_neox': 1826, 'n_words': 584}