<commit_before>from itertools import  product
from django.db.models.sql.where import Constraint
from commands import parse_constraint, OPERATORS_MAP
from django.db.models.sql.datastructures import EmptyResultSet
from djangae.db.backends.appengine.dbapi import NotSupportedError

from google.appengine.api import datastore


IMPOSSIBLE_FILTER = ('__key__', '<', datastore.Key.from_path('', 1))

def process_literal(node, filtered_columns=[], negated=False):
    column, op, value = node[1]
    if filtered_columns is not None:
        assert isinstance(filtered_columns, set)
        filtered_columns.add(column)
    if op == 'in':  # Explode INs into OR
        if not isinstance(value, (list, tuple, set)):
            raise ValueError("IN queries must be supplied a list of values")
        if negated:
            lits = []
            for x in value:
                lits.append(('LIT', (column, '>', x)))
                lits.append(('LIT', (column, '<', x)))
            return ('OR', lits), filtered_columns
        else:
            if not value:
                # Add an impossible filter when someone queries on an empty list, which should never return anything for
                # this branch. We can't just raise an EmptyResultSet because another branch might return something
                return ('AND', [('LIT', IMPOSSIBLE_FILTER)]), filtered_columns
            return ('OR', [('LIT', (column, '=', x)) for x in value]), filtered_columns
    elif op == "isnull":
        op = "exact"
        value = None

    if not OPERATORS_MAP.get(op):
        raise NotSupportedError("Unsupported operator %s" % op)
    _op = OPERATORS_MAP[op]

    if negated and _op == '=':  # Explode
        return ('OR', [('LIT', (column, '>', value)), ('LIT', (column, '<', value))]), filtered_columns
    return ('LIT', (column, _op, value)), filtered_columns


def process_node(node, connection, negated=False):
    if isinstance(node, tuple) and isinstance(node[0], Constraint):
        return ('LIT', parse_constraint(node, connection, negated)), negated
    if isinstance(node, tuple):
        return node, negated
    if node.connector == 'AND' or 'OR':
        if node.negated:
            negated = True
        return (node.connector, [child for child in node.children]), negated


def parse_dnf(node, connection):
    tree, filtered_columns = parse_tree(node, connection)
    if tree:
        tree = tripled(tree)


    if tree and tree[0] != 'OR':
        tree = ('OR', [tree])

    if tree:
        final = []
        for and_branch in tree[-1]:
            if and_branch[0] == 'LIT' and and_branch[-1] == IMPOSSIBLE_FILTER:
                continue
            elif and_branch[0] == 'AND' and IMPOSSIBLE_FILTER in [x[-1] for x in and_branch[-1] ]:
                continue

            final.append(and_branch)
        if not final:
            raise EmptyResultSet()
        else:
            tree = (tree[0], final)<commit_msg>Add a comment<commit_after>from itertools import  product
from django.db.models.sql.where import Constraint
from commands import parse_constraint, OPERATORS_MAP
from django.db.models.sql.datastructures import EmptyResultSet
from djangae.db.backends.appengine.dbapi import NotSupportedError

from google.appengine.api import datastore


IMPOSSIBLE_FILTER = ('__key__', '<', datastore.Key.from_path('', 1))

def process_literal(node, filtered_columns=[], negated=False):
    column, op, value = node[1]
    if filtered_columns is not None:
        assert isinstance(filtered_columns, set)
        filtered_columns.add(column)
    if op == 'in':  # Explode INs into OR
        if not isinstance(value, (list, tuple, set)):
            raise ValueError("IN queries must be supplied a list of values")
        if negated:
            lits = []
            for x in value:
                lits.append(('LIT', (column, '>', x)))
                lits.append(('LIT', (column, '<', x)))
            return ('OR', lits), filtered_columns
        else:
            if not value:
                # Add an impossible filter when someone queries on an empty list, which should never return anything for
                # this branch. We can't just raise an EmptyResultSet because another branch might return something
                return ('AND', [('LIT', IMPOSSIBLE_FILTER)]), filtered_columns
            return ('OR', [('LIT', (column, '=', x)) for x in value]), filtered_columns
    elif op == "isnull":
        op = "exact"
        value = None

    if not OPERATORS_MAP.get(op):
        raise NotSupportedError("Unsupported operator %s" % op)
    _op = OPERATORS_MAP[op]

    if negated and _op == '=':  # Explode
        return ('OR', [('LIT', (column, '>', value)), ('LIT', (column, '<', value))]), filtered_columns
    return ('LIT', (column, _op, value)), filtered_columns


def process_node(node, connection, negated=False):
    if isinstance(node, tuple) and isinstance(node[0], Constraint):
        return ('LIT', parse_constraint(node, connection, negated)), negated
    if isinstance(node, tuple):
        return node, negated
    if node.connector == 'AND' or 'OR':
        if node.negated:
            negated = True
        return (node.connector, [child for child in node.children]), negated


def parse_dnf(node, connection):
    tree, filtered_columns = parse_tree(node, connection)
    if tree:
        tree = tripled(tree)


    if tree and tree[0] != 'OR':
        tree = ('OR', [tree])


    # Filter out impossible branches of the where, if that then results in an empty tree then
    # raise an EmptyResultSet, otherwise replace the tree with the now simpler query
    if tree:
        final = []
        for and_branch in tree[-1]:
            if and_branch[0] == 'LIT' and and_branch[-1] == IMPOSSIBLE_FILTER:
                continue
            elif and_branch[0] == 'AND' and IMPOSSIBLE_FILTER in [x[-1] for x in and_branch[-1] ]:
                continue

            final.append(and_branch)
        if not final:
            raise EmptyResultSet()
        else:
            tree = (tree[0], final)
{'fraction_non_alphanumeric': 0.11687025420931, 'fraction_numerical': 0.0036315615714757345, 'mean_word_length': 2.6194743130227, 'pattern_counts': {'":': 2, '<': 9, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5307659', 'n_tokens_mistral': 1789, 'n_tokens_neox': 1729, 'n_words': 563}