<commit_before># coding: utf-8
import django

from django.db.models.sql import compiler
from django.db.models.sql.query import get_order_dir, ORDER_DIR

from django.db.models.sql.where import WhereNode, ExtraWhere, AND
from django.db.models.sql.where import EmptyShortCircuit, EmptyResultSet
from django.db.models.sql.expressions import SQLEvaluator
import re
from django.utils.datastructures import SortedDict

DJANGO15 = (1, 5, 0, 'alpha', 0)
DJANGO16 = (1, 6, 0, 'alpha', 0)


class SphinxExtraWhere(ExtraWhere):

    def as_sql(self, qn=None, connection=None):
        sqls = ["%s" % sql for sql in self.sqls]
        return " AND ".join(sqls), tuple(self.params or ())


class SphinxWhereNode(WhereNode):
    def sql_for_columns(self, data, qn, connection, field_internal_type=None):
        table_alias, name, db_type = data
        if django.get_version() < "1.6":
            return connection.ops.field_cast_sql(db_type) % name
        return connection.ops.field_cast_sql(db_type, field_internal_type) % name

    def make_atom(self, child, qn, connection):
        """
        Transform search, the keyword should not be quoted.
        """
        lvalue, lookup_type, value_annot, params_or_value = child
        sql, params = super(SphinxWhereNode, self).make_atom(child, qn, connection)
        if lookup_type == 'search':
            if hasattr(lvalue, 'process'):
                try:
                    lvalue, params = lvalue.process(lookup_type, params_or_value, connection)
                except EmptyShortCircuit:
                    raise EmptyResultSet
            if isinstance(lvalue, tuple):
                # A direct database column lookup.
                field_sql = self.sql_for_columns(lvalue, qn, connection)
            else:
                # A smart object with an as_sql() method.
                field_sql = lvalue.as_sql(qn, connection)
            # TODO: There are a couple problems here.
            # 1. The user _might_ want to search only a specific field.
            # 2. However, since Django requires a field name to use the __search operator
            #    There is no way to do a search in _all_ fields.
            # 3. Because, using multiple __search operators is not supported.
            # So, we need to merge multiped __search operators into a single MATCH(), we
            # can't do that here, we have to do that one level up...
            # Ignore the field name, search all fields:
            params = ('@* %s' % params[0], )
            # _OR_ respect the field name, and search on it:
            #params = ('@%s %s' % (field_sql, params[0]), )
        if self._real_negated:
            col = lvalue.col
            if lookup_type == 'exact':
                sql = '%s <> %%s' % col
            if lookup_type == 'in':
                params_placeholder = '(%s)' % (', '.join(['%s'] * len(params)))
                sql = '%s NOT IN %s' % (col, params_placeholder)
        return sql, params

    def as_sql(self, qn, connection):
        self._real_negated = self.negated
        # don't allow Django to add unsupported NOT (...) before all lookups
        self.negated = False
        sql_string, result_params = super(SphinxWhereNode, self).as_sql(qn, connection)
        self.negated = self._real_negated
        return sql_string, result_params

class SphinxQLCompiler(compiler.SQLCompiler):
    def get_columns(self, *args, **kwargs):
        result = columns = super(SphinxQLCompiler, self).get_columns(*args, **kwargs)
        if django.VERSION < DJANGO16:
            columns = result
        else:
            columns = result[0]
        db_table = self.query.model._meta.db_table
        for i, column in enumerate(columns):
            if column.startswith(db_table + '.'):
                column = column.partition('.')[2]
            # fix not accepted expression (bool(value)) AS v
            columns[i] = re.sub(r"^\((.*)\) AS ([\w\d\_]+)$", '\\1 AS \\2',
                                column)
        return result
<commit_msg>add exception<commit_after># coding: utf-8
import django

from django.db.models.sql import compiler
from django.db.models.sql.query import get_order_dir, ORDER_DIR

from django.db.models.sql.where import WhereNode, ExtraWhere, AND
from django.db.models.sql.where import EmptyShortCircuit, EmptyResultSet
from django.db.models.sql.expressions import SQLEvaluator
import re
from django.utils.datastructures import SortedDict

DJANGO15 = (1, 5, 0, 'alpha', 0)
DJANGO16 = (1, 6, 0, 'alpha', 0)


class SphinxExtraWhere(ExtraWhere):

    def as_sql(self, qn=None, connection=None):
        sqls = ["%s" % sql for sql in self.sqls]
        return " AND ".join(sqls), tuple(self.params or ())


class SphinxWhereNode(WhereNode):
    def sql_for_columns(self, data, qn, connection, field_internal_type=None):
        table_alias, name, db_type = data
        if django.get_version() < "1.6":
            return connection.ops.field_cast_sql(db_type) % name
        return connection.ops.field_cast_sql(db_type, field_internal_type) % name

    def make_atom(self, child, qn, connection):
        """
        Transform search, the keyword should not be quoted.
        """
        lvalue, lookup_type, value_annot, params_or_value = child
        sql, params = super(SphinxWhereNode, self).make_atom(child, qn, connection)
        if lookup_type == 'search':
            if hasattr(lvalue, 'process'):
                try:
                    lvalue, params = lvalue.process(lookup_type, params_or_value, connection)
                except EmptyShortCircuit:
                    raise EmptyResultSet
            if isinstance(lvalue, tuple):
                # A direct database column lookup.
                field_sql = self.sql_for_columns(lvalue, qn, connection)
            else:
                # A smart object with an as_sql() method.
                field_sql = lvalue.as_sql(qn, connection)
            # TODO: There are a couple problems here.
            # 1. The user _might_ want to search only a specific field.
            # 2. However, since Django requires a field name to use the __search operator
            #    There is no way to do a search in _all_ fields.
            # 3. Because, using multiple __search operators is not supported.
            # So, we need to merge multiped __search operators into a single MATCH(), we
            # can't do that here, we have to do that one level up...
            # Ignore the field name, search all fields:
            params = ('@* %s' % params[0], )
            # _OR_ respect the field name, and search on it:
            #params = ('@%s %s' % (field_sql, params[0]), )
        if self._real_negated:
            col = lvalue.col
            if lookup_type == 'exact':
                sql = '%s <> %%s' % col
            elif lookup_type == 'in':
                params_placeholder = '(%s)' % (', '.join(['%s'] * len(params)))
                sql = '%s NOT IN %s' % (col, params_placeholder)
            else:
                raise ValueError("Negative '%s' lookup not supported" % lookup_type)
        return sql, params

    def as_sql(self, qn, connection):
        self._real_negated = self.negated
        # don't allow Django to add unsupported NOT (...) before all lookups
        self.negated = False
        sql_string, result_params = super(SphinxWhereNode, self).as_sql(qn, connection)
        self.negated = self._real_negated
        return sql_string, result_params

class SphinxQLCompiler(compiler.SQLCompiler):
    def get_columns(self, *args, **kwargs):
        result = columns = super(SphinxQLCompiler, self).get_columns(*args, **kwargs)
        if django.VERSION < DJANGO16:
            columns = result
        else:
            columns = result[0]
        db_table = self.query.model._meta.db_table
        for i, column in enumerate(columns):
            if column.startswith(db_table + '.'):
                column = column.partition('.')[2]
            # fix not accepted expression (bool(value)) AS v
            columns[i] = re.sub(r"^\((.*)\) AS ([\w\d\_]+)$", '\\1 AS \\2',
                                column)
        return result

{'fraction_non_alphanumeric': 0.10521792510742788, 'fraction_numerical': 0.006384284837323511, 'mean_word_length': 2.2286959968291717, 'pattern_counts': {'":': 2, '<': 9, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1876313', 'n_tokens_mistral': 2436, 'n_tokens_neox': 2326, 'n_words': 715}