<commit_before># -*- encoding: utf-8 -*-

import re
import copy
import operator
from collections import OrderedDict
try:
    from functools import reduce
except ImportError:
    pass

from django.db import models
from django.db.models import Count
from django.db.models.fields import FieldDoesNotExist
from django.template.loader import render_to_string
try:
    from django.utils.encoding import force_text
except ImportError:
    from django.utils.encoding import force_unicode as force_text
try:
    from django.utils.encoding import python_2_unicode_compatible
except ImportError:
    from .compat import python_2_unicode_compatible

import six


from .exceptions import ColumnError, SkipRecord
from .columns import (Column, TextColumn, DateColumn, DateTimeColumn, BooleanColumn, IntegerColumn,
                      FloatColumn, DisplayColumn, CompoundColumn, get_column_for_modelfield)
from .utils import (OPTION_NAME_MAP, MINIMUM_PAGE_LENGTH, contains_plural_field, split_terms,
                    resolve_orm_path)

def pretty_name(name):
    if not name:
        return ''
    return name[0].capitalize() + name[1:]


# Borrowed from the Django forms implementation 
def columns_for_model(model, fields=None, exclude=None, labels=None, processors=None,
                      unsortable=None, hidden=None):
    field_list = []
    opts = model._meta
    for f in sorted(opts.fields):
        if fields is not None and f.name not in fields:
            continue
        if exclude and f.name in exclude:
            continue

        column_class = get_column_for_modelfield(f)
        if column_class is None:
            raise ColumnError("Unhandled model field %r." % (f,))
        if labels and f.name in labels:
            label = labels[f.name]
        else:
            label = f.verbose_name
        if processors and f.name in processors:
            processor = processors[f.name]
        else:
            processor = None
        if unsortable and f.name in unsortable:
            sortable = False
        else:
            sortable = True
        if hidden and f.name in hidden:
            visible = False
        else:
            visible = True
        label = (labels or {}).get(f.name, pretty_name(f.verbose_name))
        column = column_class(sources=[f.name], label=label, processor=processor, sortable=sortable,
                              visible=visible)
        column.name = f.name
        field_list.append((f.name, column))

    field_dict = OrderedDict(field_list)
    if fields:
        field_dict = OrderedDict(
            [(f, field_dict.get(f)) for f in fields
                if (not exclude) or (exclude and f not in exclude)]
        )
    return field_dict

# Borrowed from the Django forms implementation 
def get_declared_columns(bases, attrs, with_base_columns=True):
    """
    Create a list of form field instances from the passed in 'attrs', plus any
    similar fields on the base classes (in 'bases'). This is used by both the
    Form and ModelForm metclasses.

    If 'with_base_columns' is True, all fields from the bases are used.
    Otherwise, only fields in the 'declared_fields' attribute on the bases are
    used. The distinction is useful in ModelForm subclassing.
    Also integrates any additional media definitions
    """
    local_columns = [
        (column_name, attrs.pop(column_name)) \
                for column_name, obj in list(six.iteritems(attrs)) \
                if isinstance(obj, Column)
    ]
    local_columns.sort(key=lambda x: x[1].creation_counter)

    # If this class is subclassing another Form, add that Form's columns.
    # Note that we loop over the bases in *reverse*. This is necessary in
    # order to preserve the correct order of columns.
    if with_base_columns:
        for base in bases[::-1]:
            if hasattr(base, 'base_columns'):
                local_columns = list(six.iteritems(base.base_columns)) + local_columns
    else:
        for base in bases[::-1]:
            if hasattr(base, 'declared_columns'):
                local_columns = list(six.iteritems(base.declared_columns)) + local_columns

    return OrderedDict(local_columns)

class DatatableOptions(object):
    def __init__(self, options=None):
        self.model = getattr(options, 'model', None)
        self.columns = getattr(options, 'columns', None)  # table headers
        self.exclude = getattr(options, 'exclude', None)
        self.ordering = getattr(options, 'ordering', None)  # override to Model._meta.ordering
        # self.start_offset = getattr(options, 'start_offset', None)  # results to skip ahead
        self.page_length = getattr(options, 'page_length', 25)  # length of a single result page
        # self.search = getattr(options, 'search', None)  # client search string
        self.search_fields = getattr(options, 'search_fields', None)  # extra searchable ORM fields
        self.unsortable_columns = getattr(options, 'unsortable_columns', None)
        self.hidden_columns = getattr(options, 'hidden_columns', None)  # generated, but hidden

        self.structure_template = getattr(options, 'structure_template', "datatableview/default_structure.html")
        self.footer = getattr(options, 'footer', False)
        self.result_counter_id = getattr(options, 'result_counter_id', 'id_count')

        # Dictionaries of column names to values
        self.labels = getattr(options, 'labels', None)
        self.processors = getattr(options, 'processors', None)


default_options = DatatableOptions()

class DatatableMetaclass(type):
    """
    Each declared Datatable object inspects its declared "fields" in order to facilitate an
    inheritance system resembling the django.forms system.  Except for our custom Meta options that
    offer field options ('labels', 'processors', etc), this code is essentially a clone of the
    django.forms strategy.
    """

    def __new__(cls, name, bases, attrs):
        declared_columns = get_declared_columns(bases, attrs, with_base_columns=False)
        new_class = super(DatatableMetaclass, cls).__new__(cls, name, bases, attrs)

        opts = new_class._meta = new_class.options_class(getattr(new_class, 'Meta', None))
        if opts.model:
            columns = columns_for_model(opts.model, opts.columns, opts.exclude, opts.labels,
                                        opts.processors, opts.unsortable_columns, opts.hidden_columns)
            none_model_columns = [k for k, v in six.iteritems(columns) if not v]
            missing_columns = set(none_model_columns) - set(declared_columns.keys())

            for name, column in declared_columns.items():
                column.name = name
                # if not column.sources:
                #     column.sources = [name]
                if not column.label:
                    try:
                        field = resolve_orm_path(opts.model, name)
                    except FieldDoesNotExist:
                        label = name
                    else:
                        label = field.verbose_name
                    column.label = pretty_name(label)

            columns.update(declared_columns)
        else:
            columns = declared_columns
            missing_columns = []

        new_class.declared_columns = declared_columns
        new_class.base_columns = columns
        new_class.missing_columns = missing_columns
        return new_class


@python_2_unicode_compatible
class Datatable(six.with_metaclass(DatatableMetaclass)):
    """
    Declaration container for a clientside datatable, containing an optional Meta inner class,
    class-level field declarations, and callbacks for filtering and post-processing values requested
    by the client.

    Client options sent over AJAX GET parameters will override the settings given in the inner Meta
    class.

    This object can be sent to a template context and rendered there in order to generate an
    annotated HTML frame for the javascript to initialize.
    """

    options_class = DatatableOptions

    def __init__(self, object_list, url, view=None, callback_target=None, model=None,
                 query_config=None, force_distinct=True, **kwargs):
        self.object_list = object_list
        self.url = url
        self.view = view
        self.forward_callback_target = callback_target
        self.model = model or self._meta.model
        if self.model is None and hasattr(object_list, 'model'):
            self.model = object_list.model
        if query_config is None:
            query_config = {}
        self.query_config = query_config

        self.columns = copy.deepcopy(self.base_columns)

        self._force_distinct = force_distinct
        self.total_initial_record_count = None
        self.unpaged_record_count = None

    def configure(self):
        """
        Combines (in order) the declared/inherited inner Meta, any view options, and finally any
        valid AJAX GET parameters from client modifications to the data they see.
        """

        self.resolve_virtual_columns(*tuple(self.missing_columns))

        self.config = self.normalize_config(self._meta.__dict__, self.query_config)

        self.config['column_searches'] = {}
        for i, name in enumerate(self.columns.keys()):
            column_search = self.query_config.get(OPTION_NAME_MAP['search_column'] % i, None)
            if column_search:
                self.config['column_searches'][name] = column_search

        column_order = list(self.columns.keys())
        if self.config['ordering']:
            for i, name in enumerate(self.config['ordering']):
                column_name = name.lstrip('-+')
                try:
                    index = column_order.index(column_name)
                except ValueError:
                    # It is important to ignore a bad ordering name, since the model.Meta may
                    # specify a field name that is not present on the datatable columns list.
                    continue
                self.columns[column_name].sort_priority = i
                self.columns[column_name].sort_direction = 'desc' if name[0] == '-' else 'asc'
                self.columns[column_name].index = index

    # Client request configuration mergers
    def normalize_config(self, config, query_config):
        """
        Merge the declared configuration with whatever valid query parameters are found from the
        client's AJAX request.
        """

        # Core options, not modifiable by client updates
        if config['hidden_columns'] is None:
            config['hidden_columns'] = []
        if config['search_fields'] is None:
            config['search_fields'] = []
        if config['unsortable_columns'] is None:
            config['unsortable_columns'] = []

        config['search'] = self.normalize_config_search(config, query_config)
        config['start_offset'] = self.normalize_config_start_offset(config, query_config)
        config['page_length'] = self.normalize_config_page_length(config, query_config)
        config['ordering'] = self.normalize_config_ordering(config, query_config)

        return config

    def normalize_config_search(self, config, query_config):
        return query_config.get(OPTION_NAME_MAP['search'], '').strip()

    def normalize_config_start_offset(self, config, query_config):
        try:
            start_offset = query_config.get(OPTION_NAME_MAP['start_offset'], 0)
            start_offset = int(start_offset)
        except ValueError:
            start_offset = 0
        else:
            if start_offset < 0:
                start_offset = 0
        return start_offset

    def normalize_config_page_length(self, config, query_config):
        try:
            page_length = query_config.get(OPTION_NAME_MAP['page_length'], config['page_length'])
            page_length = int(page_length)
        except ValueError:
            page_length = config['page_length']
        else:
            if page_length == -1:  # dataTables' way of asking for all items, no pagination
                pass
            elif page_length < MINIMUM_PAGE_LENGTH:
                page_length = MINIMUM_PAGE_LENGTH
        return page_length

    def normalize_config_ordering(self, config, query_config):
        # For "n" columns (iSortingCols), the queried values iSortCol_0..iSortCol_n are used as
        # column indices to check the values of sSortDir_X and bSortable_X

        ordering = []
        columns_list = list(self.columns.values())

        for sort_queue_i in range(len(columns_list)):
            try:
                column_index = int(query_config.get(OPTION_NAME_MAP['sort_column'] % sort_queue_i, ''))
            except ValueError:
                continue

            column = columns_list[column_index]

            # Reject requests for unsortable columns
            if column.name in config['unsortable_columns']:
                continue

            sort_direction = query_config.get(OPTION_NAME_MAP['sort_column_direction'] % sort_queue_i, None)

            # sort_modifier = None
            if sort_direction == 'asc':
                sort_modifier = ''
            elif sort_direction == 'desc':
                sort_modifier = '-'
            else:
                # Aggressively skip invalid specification
                continue

            ordering.append('%s%s' % (sort_modifier, column.name))

        if not ordering and config['model']:
            return config['model']._meta.ordering
        return ordering

    def resolve_virtual_columns(self, *names):
        """
        Called with ``*args`` from the Meta.columns declaration that don't match the model's known
        fields.  This method can inspect these names and decide what to do with them in special
        scenarios, but by default, they are simply raised in an exception to notify the developer of
        an apparent configuration error.
        """
        if names:
            raise ColumnError("Unknown column name(s): %r" % (names,))

    # Reflection methods for wrapped columns
    def get_ordering_splits(self):<commit_msg>Remove commented line<commit_after># -*- encoding: utf-8 -*-

import re
import copy
import operator
from collections import OrderedDict
try:
    from functools import reduce
except ImportError:
    pass

from django.db import models
from django.db.models import Count
from django.db.models.fields import FieldDoesNotExist
from django.template.loader import render_to_string
try:
    from django.utils.encoding import force_text
except ImportError:
    from django.utils.encoding import force_unicode as force_text
try:
    from django.utils.encoding import python_2_unicode_compatible
except ImportError:
    from .compat import python_2_unicode_compatible

import six


from .exceptions import ColumnError, SkipRecord
from .columns import (Column, TextColumn, DateColumn, DateTimeColumn, BooleanColumn, IntegerColumn,
                      FloatColumn, DisplayColumn, CompoundColumn, get_column_for_modelfield)
from .utils import (OPTION_NAME_MAP, MINIMUM_PAGE_LENGTH, contains_plural_field, split_terms,
                    resolve_orm_path)

def pretty_name(name):
    if not name:
        return ''
    return name[0].capitalize() + name[1:]


# Borrowed from the Django forms implementation 
def columns_for_model(model, fields=None, exclude=None, labels=None, processors=None,
                      unsortable=None, hidden=None):
    field_list = []
    opts = model._meta
    for f in sorted(opts.fields):
        if fields is not None and f.name not in fields:
            continue
        if exclude and f.name in exclude:
            continue

        column_class = get_column_for_modelfield(f)
        if column_class is None:
            raise ColumnError("Unhandled model field %r." % (f,))
        if labels and f.name in labels:
            label = labels[f.name]
        else:
            label = f.verbose_name
        if processors and f.name in processors:
            processor = processors[f.name]
        else:
            processor = None
        if unsortable and f.name in unsortable:
            sortable = False
        else:
            sortable = True
        if hidden and f.name in hidden:
            visible = False
        else:
            visible = True
        label = (labels or {}).get(f.name, pretty_name(f.verbose_name))
        column = column_class(sources=[f.name], label=label, processor=processor, sortable=sortable,
                              visible=visible)
        column.name = f.name
        field_list.append((f.name, column))

    field_dict = OrderedDict(field_list)
    if fields:
        field_dict = OrderedDict(
            [(f, field_dict.get(f)) for f in fields
                if (not exclude) or (exclude and f not in exclude)]
        )
    return field_dict

# Borrowed from the Django forms implementation 
def get_declared_columns(bases, attrs, with_base_columns=True):
    """
    Create a list of form field instances from the passed in 'attrs', plus any
    similar fields on the base classes (in 'bases'). This is used by both the
    Form and ModelForm metclasses.

    If 'with_base_columns' is True, all fields from the bases are used.
    Otherwise, only fields in the 'declared_fields' attribute on the bases are
    used. The distinction is useful in ModelForm subclassing.
    Also integrates any additional media definitions
    """
    local_columns = [
        (column_name, attrs.pop(column_name)) \
                for column_name, obj in list(six.iteritems(attrs)) \
                if isinstance(obj, Column)
    ]
    local_columns.sort(key=lambda x: x[1].creation_counter)

    # If this class is subclassing another Form, add that Form's columns.
    # Note that we loop over the bases in *reverse*. This is necessary in
    # order to preserve the correct order of columns.
    if with_base_columns:
        for base in bases[::-1]:
            if hasattr(base, 'base_columns'):
                local_columns = list(six.iteritems(base.base_columns)) + local_columns
    else:
        for base in bases[::-1]:
            if hasattr(base, 'declared_columns'):
                local_columns = list(six.iteritems(base.declared_columns)) + local_columns

    return OrderedDict(local_columns)

class DatatableOptions(object):
    def __init__(self, options=None):
        self.model = getattr(options, 'model', None)
        self.columns = getattr(options, 'columns', None)  # table headers
        self.exclude = getattr(options, 'exclude', None)
        self.ordering = getattr(options, 'ordering', None)  # override to Model._meta.ordering
        # self.start_offset = getattr(options, 'start_offset', None)  # results to skip ahead
        self.page_length = getattr(options, 'page_length', 25)  # length of a single result page
        # self.search = getattr(options, 'search', None)  # client search string
        self.search_fields = getattr(options, 'search_fields', None)  # extra searchable ORM fields
        self.unsortable_columns = getattr(options, 'unsortable_columns', None)
        self.hidden_columns = getattr(options, 'hidden_columns', None)  # generated, but hidden

        self.structure_template = getattr(options, 'structure_template', "datatableview/default_structure.html")
        self.footer = getattr(options, 'footer', False)
        self.result_counter_id = getattr(options, 'result_counter_id', 'id_count')

        # Dictionaries of column names to values
        self.labels = getattr(options, 'labels', None)
        self.processors = getattr(options, 'processors', None)


default_options = DatatableOptions()

class DatatableMetaclass(type):
    """
    Each declared Datatable object inspects its declared "fields" in order to facilitate an
    inheritance system resembling the django.forms system.  Except for our custom Meta options that
    offer field options ('labels', 'processors', etc), this code is essentially a clone of the
    django.forms strategy.
    """

    def __new__(cls, name, bases, attrs):
        declared_columns = get_declared_columns(bases, attrs, with_base_columns=False)
        new_class = super(DatatableMetaclass, cls).__new__(cls, name, bases, attrs)

        opts = new_class._meta = new_class.options_class(getattr(new_class, 'Meta', None))
        if opts.model:
            columns = columns_for_model(opts.model, opts.columns, opts.exclude, opts.labels,
                                        opts.processors, opts.unsortable_columns, opts.hidden_columns)
            none_model_columns = [k for k, v in six.iteritems(columns) if not v]
            missing_columns = set(none_model_columns) - set(declared_columns.keys())

            for name, column in declared_columns.items():
                column.name = name
                # if not column.sources:
                #     column.sources = [name]
                if not column.label:
                    try:
                        field = resolve_orm_path(opts.model, name)
                    except FieldDoesNotExist:
                        label = name
                    else:
                        label = field.verbose_name
                    column.label = pretty_name(label)

            columns.update(declared_columns)
        else:
            columns = declared_columns
            missing_columns = []

        new_class.declared_columns = declared_columns
        new_class.base_columns = columns
        new_class.missing_columns = missing_columns
        return new_class


@python_2_unicode_compatible
class Datatable(six.with_metaclass(DatatableMetaclass)):
    """
    Declaration container for a clientside datatable, containing an optional Meta inner class,
    class-level field declarations, and callbacks for filtering and post-processing values requested
    by the client.

    Client options sent over AJAX GET parameters will override the settings given in the inner Meta
    class.

    This object can be sent to a template context and rendered there in order to generate an
    annotated HTML frame for the javascript to initialize.
    """

    options_class = DatatableOptions

    def __init__(self, object_list, url, view=None, callback_target=None, model=None,
                 query_config=None, force_distinct=True, **kwargs):
        self.object_list = object_list
        self.url = url
        self.view = view
        self.forward_callback_target = callback_target
        self.model = model or self._meta.model
        if self.model is None and hasattr(object_list, 'model'):
            self.model = object_list.model
        if query_config is None:
            query_config = {}
        self.query_config = query_config

        self.columns = copy.deepcopy(self.base_columns)

        self._force_distinct = force_distinct
        self.total_initial_record_count = None
        self.unpaged_record_count = None

    def configure(self):
        """
        Combines (in order) the declared/inherited inner Meta, any view options, and finally any
        valid AJAX GET parameters from client modifications to the data they see.
        """

        self.resolve_virtual_columns(*tuple(self.missing_columns))

        self.config = self.normalize_config(self._meta.__dict__, self.query_config)

        self.config['column_searches'] = {}
        for i, name in enumerate(self.columns.keys()):
            column_search = self.query_config.get(OPTION_NAME_MAP['search_column'] % i, None)
            if column_search:
                self.config['column_searches'][name] = column_search

        column_order = list(self.columns.keys())
        if self.config['ordering']:
            for i, name in enumerate(self.config['ordering']):
                column_name = name.lstrip('-+')
                try:
                    index = column_order.index(column_name)
                except ValueError:
                    # It is important to ignore a bad ordering name, since the model.Meta may
                    # specify a field name that is not present on the datatable columns list.
                    continue
                self.columns[column_name].sort_priority = i
                self.columns[column_name].sort_direction = 'desc' if name[0] == '-' else 'asc'
                self.columns[column_name].index = index

    # Client request configuration mergers
    def normalize_config(self, config, query_config):
        """
        Merge the declared configuration with whatever valid query parameters are found from the
        client's AJAX request.
        """

        # Core options, not modifiable by client updates
        if config['hidden_columns'] is None:
            config['hidden_columns'] = []
        if config['search_fields'] is None:
            config['search_fields'] = []
        if config['unsortable_columns'] is None:
            config['unsortable_columns'] = []

        config['search'] = self.normalize_config_search(config, query_config)
        config['start_offset'] = self.normalize_config_start_offset(config, query_config)
        config['page_length'] = self.normalize_config_page_length(config, query_config)
        config['ordering'] = self.normalize_config_ordering(config, query_config)

        return config

    def normalize_config_search(self, config, query_config):
        return query_config.get(OPTION_NAME_MAP['search'], '').strip()

    def normalize_config_start_offset(self, config, query_config):
        try:
            start_offset = query_config.get(OPTION_NAME_MAP['start_offset'], 0)
            start_offset = int(start_offset)
        except ValueError:
            start_offset = 0
        else:
            if start_offset < 0:
                start_offset = 0
        return start_offset

    def normalize_config_page_length(self, config, query_config):
        try:
            page_length = query_config.get(OPTION_NAME_MAP['page_length'], config['page_length'])
            page_length = int(page_length)
        except ValueError:
            page_length = config['page_length']
        else:
            if page_length == -1:  # dataTables' way of asking for all items, no pagination
                pass
            elif page_length < MINIMUM_PAGE_LENGTH:
                page_length = MINIMUM_PAGE_LENGTH
        return page_length

    def normalize_config_ordering(self, config, query_config):
        # For "n" columns (iSortingCols), the queried values iSortCol_0..iSortCol_n are used as
        # column indices to check the values of sSortDir_X and bSortable_X

        ordering = []
        columns_list = list(self.columns.values())

        for sort_queue_i in range(len(columns_list)):
            try:
                column_index = int(query_config.get(OPTION_NAME_MAP['sort_column'] % sort_queue_i, ''))
            except ValueError:
                continue

            column = columns_list[column_index]

            # Reject requests for unsortable columns
            if column.name in config['unsortable_columns']:
                continue

            sort_direction = query_config.get(OPTION_NAME_MAP['sort_column_direction'] % sort_queue_i, None)

            if sort_direction == 'asc':
                sort_modifier = ''
            elif sort_direction == 'desc':
                sort_modifier = '-'
            else:
                # Aggressively skip invalid specification
                continue

            ordering.append('%s%s' % (sort_modifier, column.name))

        if not ordering and config['model']:
            return config['model']._meta.ordering
        return ordering

    def resolve_virtual_columns(self, *names):
        """
        Called with ``*args`` from the Meta.columns declaration that don't match the model's known
        fields.  This method can inspect these names and decide what to do with them in special
        scenarios, but by default, they are simply raised in an exception to notify the developer of
        an apparent configuration error.
        """
        if names:
            raise ColumnError("Unknown column name(s): %r" % (names,))

    # Reflection methods for wrapped columns
    def get_ordering_splits(self):
{'fraction_non_alphanumeric': 0.08014494298198871, 'fraction_numerical': 0.0012789086646062028, 'mean_word_length': 2.4964600670724133, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2331585', 'n_tokens_mistral': 7610, 'n_tokens_neox': 7349, 'n_words': 2195}