<commit_before>from itertools import  product
from django.db.models.sql.where import Constraint
from commands import parse_constraint, OPERATORS_MAP
from django.db.models.sql.datastructures import EmptyResultSet
from djangae.db.backends.appengine.dbapi import NotSupportedError

from google.appengine.api import datastore

try:
    from django.db.models import Lookup # Django 1.7+ uses lookup
except ImportError:
    # Fake lookup, so we can test using isinstance(node, Lookup) and this
    # will always return False on < 1.7
    class Lookup(object):
        pass


IMPOSSIBLE_FILTER = ('__key__', '<', datastore.Key.from_path('', 1))

class QueryContainsOR(Exception):
    pass

def check_for_inequalities(node, key=None, other=None):
    """
        Parses a query WHERE tree and logs which fields have inequalities.

        If the WHERE tree includes an OR connector, we throw QueryContainsOR for efficiency
        (because for our purposes we need to know quickly if the query contains an OR connector)
    """

    other = other or set()

    if hasattr(node, "children"):
        literals = [ x for x in node.children if not hasattr(x, "children") ]
        branches = [ x for x in node.children if hasattr(x, "children") ]

        if node.connector == 'OR':
            raise QueryContainsOR

        for literal in literals:
            # Django 1.7+ have a lhs attribute which stores column info, django 1.6 doesn't
            field = literal.lhs.output_field if hasattr(literal, "lhs") else literal[0].field
            field_name = field.column

            # Again, Django 1.7+ has lookup_name, 1.6 doesn't
            lookup = literal.lookup_name if hasattr(literal, "lookup_name") else literal[1]

            if node.negated and lookup == "exact" and (field and field.primary_key):
                key = field_name
            elif ((node.negated and lookup == "exact") or (lookup in ("gt", "gte", "lt", "lte"))) and not (field and field.primary_key):
                other.add(field_name)

        for branch in branches:
            k, o = check_for_inequalities(branch, key, other)
            if k:
                key = k
            other.update(o)

    return key, other

def should_exclude_pks_in_memory(query, ordering):
    """
        We try to let the datastore do as much of the work as possible, however, there are times
        when we can actually support two inequalities in a query, if one of them is on the key. Also,
        we can support an inequality on a key, and an ordering by something else. This code literally
        returns True in those cases, in which case the dnf parsing will keep track of the excluded PKs
        and remove them from the tree, otherwise, it will add them to the tree as a > && < combo as
        normal.
    """
    try:
        key_inequality, other_inequalities = check_for_inequalities(query)
    except QueryContainsOR:
        return False

    if len(other_inequalities) == 1 or (
        key_inequality and not other_inequalities and ordering and ordering[0].lstrip("-") != key_inequality):
        return True
    else:
        return False

def process_literal(node, is_pk_filter, excluded_pks, filtered_columns=None, negated=False):
    column, op, value = node[1]
    if filtered_columns is not None:
        assert isinstance(filtered_columns, set)
        filtered_columns.add(column)
    if op == 'in':  # Explode INs into OR
        if not isinstance(value, (list, tuple, set)):
            raise ValueError("IN queries must be supplied a list of values")
        if negated:
            if len(value) == 0:
                return None, filtered_columns

            lits = []
            for x in value:
                lits.append(('LIT', (column, '>', x)))
                lits.append(('LIT', (column, '<', x)))
            return ('OR', lits), filtered_columns
        else:
            if not value:
                # Add an impossible filter when someone queries on an empty list, which should never return anything for
                # this branch. We can't just raise an EmptyResultSet because another branch might return something
                return ('AND', [('LIT', IMPOSSIBLE_FILTER)]), filtered_columns
            return ('OR', [('LIT', (column, '=', x)) for x in value]), filtered_columns
    elif op == "isnull":
        if negated:
            value = not value
            negated = not negated

        if not value:
            lits = []
            lits.append(('LIT', (column, '>', None)))
            lits.append(('LIT', (column, '<', None)))
            return ('OR', lits), filtered_columns
        else:
            op = "exact"
            value = None
    elif op == "range":
        lits = []
        if not negated:
            lits.append(('LIT', (column, '>=', value[0])))
            lits.append(('LIT', (column, '<=', value[1])))
            return ('AND', lits), filtered_columns
        else:
            lits.append(('LIT', (column, '<', value[0])))
            lits.append(('LIT', (column, '>', value[1])))
            return ('OR', lits), filtered_columns

    if not OPERATORS_MAP.get(op):
        raise NotSupportedError("Unsupported operator %s" % op)
    _op = OPERATORS_MAP[op]

    if negated and _op == '=':  # Explode
        if is_pk_filter and excluded_pks is not None: #Excluded pks is a set() if we should be using it
            excluded_pks.add(value)
            return None, filtered_columns

        return ('OR', [('LIT', (column, '>', value)), ('LIT', (column, '<', value))]), filtered_columns
    return ('LIT', (column, _op, value)), filtered_columns


def process_node(node, connection, negated=False):
    if isinstance(node, Lookup):
        field = node.lhs.output_field
        is_pk = field and field.primary_key
        return ('LIT', parse_constraint(node, connection, negated)), negated, is_pk
    elif isinstance(node, tuple) and isinstance(node[0], Constraint):
        field = node[0].field
        is_pk = field and field.primary_key
        return ('LIT', parse_constraint(node, connection, negated)), negated, is_pk
    if isinstance(node, tuple):
        return node, negated, False

    if node.negated:
        negated = True


        # This is a little crazy, but bear with me...
        # If you run a query like this:  filter(thing=1).exclude(field1="test") where field1 is
        # null-able you'll end up with a negated branch in the where tree which is:

        #           AND (negated)
        #          /            \
        #   field1="test"   field1__isnull=False

        # This is because on SQL, field1 != "test" won't give back rows where field1 is null, so
        # django has to include the negated isnull=False as well in order to get back the null rows
        # as well.  On App Engine though None is just a value, not the lack of a value, so it's
        # enough to just have the first branch in the negated node and in fact, if you try to use
        # the above tree, it will result in looking for:
        #  field1 < "test" and field1 > "test" and field1__isnull=True
        # which returns the wrong result (it'll return when field1 == None only)

        if node.connector == 'AND':
            # If the connector is 'AND'
            field_equalities = {}

            # Look and see if we have an exact and isnull on the same field
            for child in node.children:
                if child[1] in ('exact', 'isnull'):
                    field_equalities.setdefault(child[0].col, []).append(child[1])

            # If so, remove the isnull
            for field, equalities in field_equalities.iteritems():
                if sorted(equalities) != [ 'exact', 'isnull' ]:
                    continue

                # If we have more than one equality and one of them is isnull, then remove it
                node.children = [ x for x in node.children if x[0].col != field or x[1] != 'isnull' ]


    return (node.connector, [child for child in node.children]), negated, False


def parse_dnf(node, connection, ordering=None):
    should_in_memory_exclude = should_exclude_pks_in_memory(node, ordering)

    tree, filtered_columns, excluded_pks = parse_tree(
        node, connection,
        excluded_pks = set() if should_in_memory_exclude else None
    )

    if not should_exclude_pks_in_memory:
        assert excluded_pks is None

    if tree:
        tree = tripled(tree)<commit_msg>Fix the exclude nullable field workaround for 1.7+<commit_after>from itertools import  product
from django.db.models.sql.where import Constraint
from commands import parse_constraint, OPERATORS_MAP
from django.db.models.sql.datastructures import EmptyResultSet
from djangae.db.backends.appengine.dbapi import NotSupportedError

from google.appengine.api import datastore

try:
    from django.db.models import Lookup # Django 1.7+ uses lookup
except ImportError:
    # Fake lookup, so we can test using isinstance(node, Lookup) and this
    # will always return False on < 1.7
    class Lookup(object):
        pass


IMPOSSIBLE_FILTER = ('__key__', '<', datastore.Key.from_path('', 1))

class QueryContainsOR(Exception):
    pass

def check_for_inequalities(node, key=None, other=None):
    """
        Parses a query WHERE tree and logs which fields have inequalities.

        If the WHERE tree includes an OR connector, we throw QueryContainsOR for efficiency
        (because for our purposes we need to know quickly if the query contains an OR connector)
    """

    other = other or set()

    if hasattr(node, "children"):
        literals = [ x for x in node.children if not hasattr(x, "children") ]
        branches = [ x for x in node.children if hasattr(x, "children") ]

        if node.connector == 'OR':
            raise QueryContainsOR

        for literal in literals:
            # Django 1.7+ have a lhs attribute which stores column info, django 1.6 doesn't
            field = literal.lhs.output_field if hasattr(literal, "lhs") else literal[0].field
            field_name = field.column

            # Again, Django 1.7+ has lookup_name, 1.6 doesn't
            lookup = literal.lookup_name if hasattr(literal, "lookup_name") else literal[1]

            if node.negated and lookup == "exact" and (field and field.primary_key):
                key = field_name
            elif ((node.negated and lookup == "exact") or (lookup in ("gt", "gte", "lt", "lte"))) and not (field and field.primary_key):
                other.add(field_name)

        for branch in branches:
            k, o = check_for_inequalities(branch, key, other)
            if k:
                key = k
            other.update(o)

    return key, other

def should_exclude_pks_in_memory(query, ordering):
    """
        We try to let the datastore do as much of the work as possible, however, there are times
        when we can actually support two inequalities in a query, if one of them is on the key. Also,
        we can support an inequality on a key, and an ordering by something else. This code literally
        returns True in those cases, in which case the dnf parsing will keep track of the excluded PKs
        and remove them from the tree, otherwise, it will add them to the tree as a > && < combo as
        normal.
    """
    try:
        key_inequality, other_inequalities = check_for_inequalities(query)
    except QueryContainsOR:
        return False

    if len(other_inequalities) == 1 or (
        key_inequality and not other_inequalities and ordering and ordering[0].lstrip("-") != key_inequality):
        return True
    else:
        return False

def process_literal(node, is_pk_filter, excluded_pks, filtered_columns=None, negated=False):
    column, op, value = node[1]
    if filtered_columns is not None:
        assert isinstance(filtered_columns, set)
        filtered_columns.add(column)
    if op == 'in':  # Explode INs into OR
        if not isinstance(value, (list, tuple, set)):
            raise ValueError("IN queries must be supplied a list of values")
        if negated:
            if len(value) == 0:
                return None, filtered_columns

            lits = []
            for x in value:
                lits.append(('LIT', (column, '>', x)))
                lits.append(('LIT', (column, '<', x)))
            return ('OR', lits), filtered_columns
        else:
            if not value:
                # Add an impossible filter when someone queries on an empty list, which should never return anything for
                # this branch. We can't just raise an EmptyResultSet because another branch might return something
                return ('AND', [('LIT', IMPOSSIBLE_FILTER)]), filtered_columns
            return ('OR', [('LIT', (column, '=', x)) for x in value]), filtered_columns
    elif op == "isnull":
        if negated:
            value = not value
            negated = not negated

        if not value:
            lits = []
            lits.append(('LIT', (column, '>', None)))
            lits.append(('LIT', (column, '<', None)))
            return ('OR', lits), filtered_columns
        else:
            op = "exact"
            value = None
    elif op == "range":
        lits = []
        if not negated:
            lits.append(('LIT', (column, '>=', value[0])))
            lits.append(('LIT', (column, '<=', value[1])))
            return ('AND', lits), filtered_columns
        else:
            lits.append(('LIT', (column, '<', value[0])))
            lits.append(('LIT', (column, '>', value[1])))
            return ('OR', lits), filtered_columns

    if not OPERATORS_MAP.get(op):
        raise NotSupportedError("Unsupported operator %s" % op)
    _op = OPERATORS_MAP[op]

    if negated and _op == '=':  # Explode
        if is_pk_filter and excluded_pks is not None: #Excluded pks is a set() if we should be using it
            excluded_pks.add(value)
            return None, filtered_columns

        return ('OR', [('LIT', (column, '>', value)), ('LIT', (column, '<', value))]), filtered_columns
    return ('LIT', (column, _op, value)), filtered_columns


def process_node(node, connection, negated=False):
    if isinstance(node, Lookup):
        field = node.lhs.output_field
        is_pk = field and field.primary_key
        return ('LIT', parse_constraint(node, connection, negated)), negated, is_pk
    elif isinstance(node, tuple) and isinstance(node[0], Constraint):
        field = node[0].field
        is_pk = field and field.primary_key
        return ('LIT', parse_constraint(node, connection, negated)), negated, is_pk
    if isinstance(node, tuple):
        return node, negated, False

    if node.negated:
        negated = True


        # This is a little crazy, but bear with me...
        # If you run a query like this:  filter(thing=1).exclude(field1="test") where field1 is
        # null-able you'll end up with a negated branch in the where tree which is:

        #           AND (negated)
        #          /            \
        #   field1="test"   field1__isnull=False

        # This is because on SQL, field1 != "test" won't give back rows where field1 is null, so
        # django has to include the negated isnull=False as well in order to get back the null rows
        # as well.  On App Engine though None is just a value, not the lack of a value, so it's
        # enough to just have the first branch in the negated node and in fact, if you try to use
        # the above tree, it will result in looking for:
        #  field1 < "test" and field1 > "test" and field1__isnull=True
        # which returns the wrong result (it'll return when field1 == None only)

        if node.connector == 'AND':
            # If the connector is 'AND'
            field_equalities = {}

            def get_op(constraint_or_lookup):
                # <= 1.6 child is a tuple, else it's a lookup
                return constraint_or_lookup[1] if isinstance(constraint_or_lookup, tuple) else constraint_or_lookup.lookup_name

            def get_lhs_col(constraint_or_lookup):
                # <= 1.6 child is a tuple, else it's a lookup
                return constraint_or_lookup[0] if isinstance(constraint_or_lookup, tuple) else constraint_or_lookup.lhs.target.column

            # Look and see if we have an exact and isnull on the same field
            for child in node.children:
                op = get_op(child)
                column = get_lhs_col(child)
                if op in ('exact', 'isnull'):
                    field_equalities.setdefault(column, []).append(op)

            # If so, remove the isnull
            for field, equalities in field_equalities.iteritems():
                if sorted(equalities) != [ 'exact', 'isnull' ]:
                    continue

                # If we have more than one equality and one of them is isnull, then remove it
                node.children = [ x for x in node.children if get_lhs_col(x) != field or get_op(x) != 'isnull' ]


    return (node.connector, [child for child in node.children]), negated, False


def parse_dnf(node, connection, ordering=None):
    should_in_memory_exclude = should_exclude_pks_in_memory(node, ordering)

    tree, filtered_columns, excluded_pks = parse_tree(
        node, connection,
        excluded_pks = set() if should_in_memory_exclude else None
    )

    if not should_exclude_pks_in_memory:
        assert excluded_pks is None

    if tree:
        tree = tripled(tree)
{'fraction_non_alphanumeric': 0.0963703193200092, 'fraction_numerical': 0.004881690787962325, 'mean_word_length': 2.370040642539191, 'pattern_counts': {'":': 4, '<': 23, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7231690', 'n_tokens_mistral': 5030, 'n_tokens_neox': 4850, 'n_words': 1788}