<commit_before>from __future__ import unicode_literals
from operator import attrgetter

from django import VERSION
from django.contrib.contenttypes.generic import GenericRelation
from django.contrib.contenttypes.models import ContentType
from django.db import models, router
from django.db.models.fields import Field
from django.db.models.fields.related import ManyToManyRel, RelatedField, add_lazy_relation
from django.db.models.related import RelatedObject
from django.utils.text import capfirst
from django.utils.translation import ugettext_lazy as _
from django.utils import six

try:
    from django.db.models.related import PathInfo
except ImportError:
    pass  # PathInfo is not used on Django < 1.6

from taggit.forms import TagField
from taggit.models import TaggedItem, GenericTaggedItemBase
from taggit.utils import require_instance_manager


def _model_name(model):
    if VERSION < (1, 7):
        return model._meta.module_name
    else:
        return model._meta.model_name


class TaggableRel(ManyToManyRel):
    def __init__(self, field, related_name, through):
        self.related_name = related_name
        self.limit_choices_to = {}
        self.symmetrical = True
        self.multiple = True
        self.through = None if VERSION < (1, 7) else through
        self.field = field

    def get_joining_columns(self):
        return self.field.get_reverse_joining_columns()

    def get_extra_restriction(self, where_class, alias, related_alias):
        return self.field.get_extra_restriction(where_class, related_alias, alias)


class ExtraJoinRestriction(object):
    """
    An extra restriction used for contenttype restriction in joins.
    """
    def __init__(self, alias, col, content_types):
        self.alias = alias
        self.col = col
        self.content_types = content_types

    def as_sql(self, qn, connection):
        if len(self.content_types) == 1:
            extra_where = "%s.%s = %%s" % (qn(self.alias), qn(self.col))
        else:
            extra_where = "%s.%s IN (%s)" % (qn(self.alias), qn(self.col),
                                             ','.join(['%s'] * len(self.content_types)))
        return extra_where, self.content_types

    def relabel_aliases(self, change_map):
        self.alias = change_map.get(self.alias, self.alias)

    def clone(self):
        return self.__class__(self.alias, self.col, self.content_types[:])


class TaggableManager(RelatedField, Field):
    _related_name_counter = 0

    def __init__(self, verbose_name=_("Tags"), help_text=_("A comma-separated list of tags."),
            through=None, blank=False, related_name=None):
        Field.__init__(self, verbose_name=verbose_name, help_text=help_text, blank=blank, null=True, serialize=False)
        self.through = through or TaggedItem
        self.rel = TaggableRel(self, related_name, self.through)
        self.swappable = False

    def __get__(self, instance, model):
        if instance is not None and instance.pk is None:
            raise ValueError("%s objects need to have a primary key value "
                "before you can access their tags." % model.__name__)
        manager = _TaggableManager(
            through=self.through,
            model=model,
            instance=instance,
            prefetch_cache_name = self.name
        )
        return manager<commit_msg>Allow pluggable manager for TaggableManager.<commit_after>from __future__ import unicode_literals
from operator import attrgetter

from django import VERSION
from django.contrib.contenttypes.generic import GenericRelation
from django.contrib.contenttypes.models import ContentType
from django.db import models, router
from django.db.models.fields import Field
from django.db.models.fields.related import ManyToManyRel, RelatedField, add_lazy_relation
from django.db.models.related import RelatedObject
from django.utils.text import capfirst
from django.utils.translation import ugettext_lazy as _
from django.utils import six

try:
    from django.db.models.related import PathInfo
except ImportError:
    pass  # PathInfo is not used on Django < 1.6

from taggit.forms import TagField
from taggit.models import TaggedItem, GenericTaggedItemBase
from taggit.utils import require_instance_manager


def _model_name(model):
    if VERSION < (1, 7):
        return model._meta.module_name
    else:
        return model._meta.model_name


class TaggableRel(ManyToManyRel):
    def __init__(self, field, related_name, through):
        self.related_name = related_name
        self.limit_choices_to = {}
        self.symmetrical = True
        self.multiple = True
        self.through = None if VERSION < (1, 7) else through
        self.field = field

    def get_joining_columns(self):
        return self.field.get_reverse_joining_columns()

    def get_extra_restriction(self, where_class, alias, related_alias):
        return self.field.get_extra_restriction(where_class, related_alias, alias)


class ExtraJoinRestriction(object):
    """
    An extra restriction used for contenttype restriction in joins.
    """
    def __init__(self, alias, col, content_types):
        self.alias = alias
        self.col = col
        self.content_types = content_types

    def as_sql(self, qn, connection):
        if len(self.content_types) == 1:
            extra_where = "%s.%s = %%s" % (qn(self.alias), qn(self.col))
        else:
            extra_where = "%s.%s IN (%s)" % (qn(self.alias), qn(self.col),
                                             ','.join(['%s'] * len(self.content_types)))
        return extra_where, self.content_types

    def relabel_aliases(self, change_map):
        self.alias = change_map.get(self.alias, self.alias)

    def clone(self):
        return self.__class__(self.alias, self.col, self.content_types[:])


class TaggableManager(RelatedField, Field):
    _related_name_counter = 0

    def __init__(self, verbose_name=_("Tags"), help_text=_("A comma-separated list of tags."),
            through=None, blank=False, related_name=None, manager=None):
        Field.__init__(self, verbose_name=verbose_name, help_text=help_text, blank=blank, null=True, serialize=False)
        self.through = through or TaggedItem
        self.rel = TaggableRel(self, related_name, self.through)
        self.swappable = False
        self.manager = manager or _TaggableManager

    def __get__(self, instance, model):
        if instance is not None and instance.pk is None:
            raise ValueError("%s objects need to have a primary key value "
                "before you can access their tags." % model.__name__)
        manager = self.manager(
            through=self.through,
            model=model,
            instance=instance,
            prefetch_cache_name = self.name
        )
        return manager
{'fraction_non_alphanumeric': 0.08812260536398467, 'fraction_numerical': 0.002357795461243737, 'mean_word_length': 3.08855421686747, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2440680', 'n_tokens_mistral': 1975, 'n_tokens_neox': 1920, 'n_words': 522}