<commit_before>from __future__ import unicode_literals

from operator import attrgetter

from django import VERSION
from django.contrib.contenttypes.models import ContentType
from django.db import models, router
from django.db.models.fields import Field
from django.db.models.fields.related import (add_lazy_relation, ManyToManyRel,
                                             RelatedField)

try:
    from django.db.models.related import RelatedObject
except ImportError:  # Django 1.8 +
    RelatedObject = None

from django.utils import six
from django.utils.text import capfirst
from django.utils.translation import ugettext_lazy as _

from taggit.forms import TagField
from taggit.models import GenericTaggedItemBase, TaggedItem
from taggit.utils import require_instance_manager

try:
    from django.contrib.contenttypes.fields import GenericRelation
except ImportError:  # django < 1.7
    from django.contrib.contenttypes.generic import GenericRelation

try:
    from django.db.models.related import PathInfo
except ImportError:
    pass  # PathInfo is not used on Django < 1.6


def _model_name(model):
    if VERSION < (1, 7):
        return model._meta.module_name
    else:
        return model._meta.model_name


class TaggableRel(ManyToManyRel):
    def __init__(self, field, related_name, through, to=None):
        self.to = to
        self.related_name = related_name
        self.limit_choices_to = {}
        self.symmetrical = True
        self.multiple = True
        self.through = None if VERSION < (1, 7) else through
        self.field = field

    def get_joining_columns(self):
        return self.field.get_reverse_joining_columns()

    def get_extra_restriction(self, where_class, alias, related_alias):
        return self.field.get_extra_restriction(where_class, related_alias, alias)


class ExtraJoinRestriction(object):
    """
    An extra restriction used for contenttype restriction in joins.
    """
    def __init__(self, alias, col, content_types):
        self.alias = alias
        self.col = col
        self.content_types = content_types

    def as_sql(self, qn, connection):
        if len(self.content_types) == 1:
            extra_where = "%s.%s = %%s" % (qn(self.alias), qn(self.col))
        else:
            extra_where = "%s.%s IN (%s)" % (qn(self.alias), qn(self.col),
                                             ','.join(['%s'] * len(self.content_types)))
        return extra_where, self.content_types

    def relabel_aliases(self, change_map):
        self.alias = change_map.get(self.alias, self.alias)

    def clone(self):
        return self.__class__(self.alias, self.col, self.content_types[:])


class _TaggableManager(models.Manager):
    def __init__(self, through, model, instance, prefetch_cache_name):
        self.through = through
        self.model = model
        self.instance = instance
        self.prefetch_cache_name = prefetch_cache_name
        self._db = None

    def is_cached(self, instance):
        return self.prefetch_cache_name in instance._prefetched_objects_cache

    def get_queryset(self):
        try:
            return self.instance._prefetched_objects_cache[self.prefetch_cache_name]
        except (AttributeError, KeyError):
            return self.through.tags_for(self.model, self.instance)

    def get_prefetch_queryset(self, instances, queryset=None):
        if queryset is not None:
            raise ValueError("Custom queryset can't be used for this lookup.")

        instance = instances[0]
        from django.db import connections
        db = self._db or router.db_for_read(instance.__class__, instance=instance)

        fieldname = ('object_id' if issubclass(self.through, GenericTaggedItemBase)
                     else 'content_object')
        fk = self.through._meta.get_field(fieldname)
        query = {
            '%s__%s__in' % (self.through.tag_relname(), fk.name):
                set(obj._get_pk_val() for obj in instances)
        }
        join_table = self.through._meta.db_table
        source_col = fk.column
        connection = connections[db]
        qn = connection.ops.quote_name
        qs = self.get_queryset().using(db)._next_is_sticky().filter(**query).extra(
            select={
                '_prefetch_related_val': '%s.%s' % (qn(join_table), qn(source_col))
            }
        )
        return (qs,
                attrgetter('_prefetch_related_val'),
                lambda obj: obj._get_pk_val(),
                False,
                self.prefetch_cache_name)

    # Django < 1.6 uses the previous name of query_set
    get_query_set = get_queryset
    get_prefetch_query_set = get_prefetch_queryset

    def _lookup_kwargs(self):
        return self.through.lookup_kwargs(self.instance)

    @require_instance_manager
    def add(self, *tags):
        str_tags = set()
        tag_objs = set()
        for t in tags:
            if isinstance(t, self.through.tag_model()):
                tag_objs.add(t)
            elif isinstance(t, six.string_types):
                str_tags.add(t)
            else:
                raise ValueError("Cannot add {0} ({1}). Expected {2} or str.".format(
                    t, type(t), type(self.through.tag_model())))

        # If str_tags has 0 elements Django actually optimizes that to not do a
        # query.  Malcolm is very smart.
        existing = self.through.tag_model().objects.filter(
            name__in=str_tags
        )
        tag_objs.update(existing)

        for new_tag in str_tags - set(t.name for t in existing):
            tag_objs.add(self.through.tag_model().objects.create(name=new_tag))

        for tag in tag_objs:
            self.through.objects.get_or_create(tag=tag, **self._lookup_kwargs())

    @require_instance_manager
    def names(self):
        return self.get_queryset().values_list('name', flat=True)

    @require_instance_manager
    def slugs(self):
        return self.get_queryset().values_list('slug', flat=True)

    @require_instance_manager
    def set(self, *tags):
        self.clear()
        self.add(*tags)

    @require_instance_manager
    def remove(self, *tags):
        self.through.objects.filter(**self._lookup_kwargs()).filter(
            tag__name__in=tags).delete()

    @require_instance_manager
    def clear(self):
        self.through.objects.filter(**self._lookup_kwargs()).delete()

    def most_common(self):
        return self.get_queryset().annotate(
            num_times=models.Count(self.through.tag_relname())
        ).order_by('-num_times')

    @require_instance_manager
    def similar_objects(self):
        lookup_kwargs = self._lookup_kwargs()
        lookup_keys = sorted(lookup_kwargs)
        qs = self.through.objects.values(*six.iterkeys(lookup_kwargs))
        qs = qs.annotate(n=models.Count('pk'))
        qs = qs.exclude(**lookup_kwargs)
        qs = qs.filter(tag__in=self.all())
        qs = qs.order_by('-n')

        # TODO: This all feels like a bit of a hack.
        items = {}
        if len(lookup_keys) == 1:
            # Can we do this without a second query by using a select_related()
            # somehow?
            f = self.through._meta.get_field_by_name(lookup_keys[0])[0]
            objs = f.rel.to._default_manager.filter(**{
                "%s__in" % f.rel.field_name: [r["content_object"] for r in qs]
            })
            for obj in objs:
                items[(getattr(obj, f.rel.field_name),)] = obj
        else:
            preload = {}
            for result in qs:
                preload.setdefault(result['content_type'], set())
                preload[result["content_type"]].add(result["object_id"])

            for ct, obj_ids in preload.items():
                ct = ContentType.objects.get_for_id(ct)
                for obj in ct.model_class()._default_manager.filter(pk__in=obj_ids):
                    items[(ct.pk, obj.pk)] = obj

        results = []
        for result in qs:
            obj = items[
                tuple(result[k] for k in lookup_keys)
            ]
            obj.similar_tags = result["n"]
            results.append(obj)
        return results


class TaggableManager(RelatedField, Field):
    # Field flags
    many_to_many = True
    many_to_one = False
    one_to_many = False
    one_to_one = False

    _related_name_counter = 0

    def __init__(self, verbose_name=_("Tags"),
                 help_text=_("A comma-separated list of tags."),
                 through=None, blank=False, related_name=None, to=None,
                 manager=_TaggableManager):
        Field.__init__(self, verbose_name=verbose_name, help_text=help_text,
                       blank=blank, null=True, serialize=False)
        self.through = through or TaggedItem
        self.rel = TaggableRel(self, related_name, self.through, to=to)
        self.swappable = False
        self.manager = manager
        # NOTE: `to` is ignored, only used via `deconstruct`.<commit_msg>Pass rel into Field constructor<commit_after>from __future__ import unicode_literals

from operator import attrgetter

from django import VERSION
from django.contrib.contenttypes.models import ContentType
from django.db import models, router
from django.db.models.fields import Field
from django.db.models.fields.related import (add_lazy_relation, ManyToManyRel,
                                             RelatedField)

try:
    from django.db.models.related import RelatedObject
except ImportError:  # Django 1.8 +
    RelatedObject = None

from django.utils import six
from django.utils.text import capfirst
from django.utils.translation import ugettext_lazy as _

from taggit.forms import TagField
from taggit.models import GenericTaggedItemBase, TaggedItem
from taggit.utils import require_instance_manager

try:
    from django.contrib.contenttypes.fields import GenericRelation
except ImportError:  # django < 1.7
    from django.contrib.contenttypes.generic import GenericRelation

try:
    from django.db.models.related import PathInfo
except ImportError:
    pass  # PathInfo is not used on Django < 1.6


def _model_name(model):
    if VERSION < (1, 7):
        return model._meta.module_name
    else:
        return model._meta.model_name


class TaggableRel(ManyToManyRel):
    def __init__(self, field, related_name, through, to=None):
        self.to = to
        self.related_name = related_name
        self.limit_choices_to = {}
        self.symmetrical = True
        self.multiple = True
        self.through = None if VERSION < (1, 7) else through
        self.field = field

    def get_joining_columns(self):
        return self.field.get_reverse_joining_columns()

    def get_extra_restriction(self, where_class, alias, related_alias):
        return self.field.get_extra_restriction(where_class, related_alias, alias)


class ExtraJoinRestriction(object):
    """
    An extra restriction used for contenttype restriction in joins.
    """
    def __init__(self, alias, col, content_types):
        self.alias = alias
        self.col = col
        self.content_types = content_types

    def as_sql(self, qn, connection):
        if len(self.content_types) == 1:
            extra_where = "%s.%s = %%s" % (qn(self.alias), qn(self.col))
        else:
            extra_where = "%s.%s IN (%s)" % (qn(self.alias), qn(self.col),
                                             ','.join(['%s'] * len(self.content_types)))
        return extra_where, self.content_types

    def relabel_aliases(self, change_map):
        self.alias = change_map.get(self.alias, self.alias)

    def clone(self):
        return self.__class__(self.alias, self.col, self.content_types[:])


class _TaggableManager(models.Manager):
    def __init__(self, through, model, instance, prefetch_cache_name):
        self.through = through
        self.model = model
        self.instance = instance
        self.prefetch_cache_name = prefetch_cache_name
        self._db = None

    def is_cached(self, instance):
        return self.prefetch_cache_name in instance._prefetched_objects_cache

    def get_queryset(self):
        try:
            return self.instance._prefetched_objects_cache[self.prefetch_cache_name]
        except (AttributeError, KeyError):
            return self.through.tags_for(self.model, self.instance)

    def get_prefetch_queryset(self, instances, queryset=None):
        if queryset is not None:
            raise ValueError("Custom queryset can't be used for this lookup.")

        instance = instances[0]
        from django.db import connections
        db = self._db or router.db_for_read(instance.__class__, instance=instance)

        fieldname = ('object_id' if issubclass(self.through, GenericTaggedItemBase)
                     else 'content_object')
        fk = self.through._meta.get_field(fieldname)
        query = {
            '%s__%s__in' % (self.through.tag_relname(), fk.name):
                set(obj._get_pk_val() for obj in instances)
        }
        join_table = self.through._meta.db_table
        source_col = fk.column
        connection = connections[db]
        qn = connection.ops.quote_name
        qs = self.get_queryset().using(db)._next_is_sticky().filter(**query).extra(
            select={
                '_prefetch_related_val': '%s.%s' % (qn(join_table), qn(source_col))
            }
        )
        return (qs,
                attrgetter('_prefetch_related_val'),
                lambda obj: obj._get_pk_val(),
                False,
                self.prefetch_cache_name)

    # Django < 1.6 uses the previous name of query_set
    get_query_set = get_queryset
    get_prefetch_query_set = get_prefetch_queryset

    def _lookup_kwargs(self):
        return self.through.lookup_kwargs(self.instance)

    @require_instance_manager
    def add(self, *tags):
        str_tags = set()
        tag_objs = set()
        for t in tags:
            if isinstance(t, self.through.tag_model()):
                tag_objs.add(t)
            elif isinstance(t, six.string_types):
                str_tags.add(t)
            else:
                raise ValueError("Cannot add {0} ({1}). Expected {2} or str.".format(
                    t, type(t), type(self.through.tag_model())))

        # If str_tags has 0 elements Django actually optimizes that to not do a
        # query.  Malcolm is very smart.
        existing = self.through.tag_model().objects.filter(
            name__in=str_tags
        )
        tag_objs.update(existing)

        for new_tag in str_tags - set(t.name for t in existing):
            tag_objs.add(self.through.tag_model().objects.create(name=new_tag))

        for tag in tag_objs:
            self.through.objects.get_or_create(tag=tag, **self._lookup_kwargs())

    @require_instance_manager
    def names(self):
        return self.get_queryset().values_list('name', flat=True)

    @require_instance_manager
    def slugs(self):
        return self.get_queryset().values_list('slug', flat=True)

    @require_instance_manager
    def set(self, *tags):
        self.clear()
        self.add(*tags)

    @require_instance_manager
    def remove(self, *tags):
        self.through.objects.filter(**self._lookup_kwargs()).filter(
            tag__name__in=tags).delete()

    @require_instance_manager
    def clear(self):
        self.through.objects.filter(**self._lookup_kwargs()).delete()

    def most_common(self):
        return self.get_queryset().annotate(
            num_times=models.Count(self.through.tag_relname())
        ).order_by('-num_times')

    @require_instance_manager
    def similar_objects(self):
        lookup_kwargs = self._lookup_kwargs()
        lookup_keys = sorted(lookup_kwargs)
        qs = self.through.objects.values(*six.iterkeys(lookup_kwargs))
        qs = qs.annotate(n=models.Count('pk'))
        qs = qs.exclude(**lookup_kwargs)
        qs = qs.filter(tag__in=self.all())
        qs = qs.order_by('-n')

        # TODO: This all feels like a bit of a hack.
        items = {}
        if len(lookup_keys) == 1:
            # Can we do this without a second query by using a select_related()
            # somehow?
            f = self.through._meta.get_field_by_name(lookup_keys[0])[0]
            objs = f.rel.to._default_manager.filter(**{
                "%s__in" % f.rel.field_name: [r["content_object"] for r in qs]
            })
            for obj in objs:
                items[(getattr(obj, f.rel.field_name),)] = obj
        else:
            preload = {}
            for result in qs:
                preload.setdefault(result['content_type'], set())
                preload[result["content_type"]].add(result["object_id"])

            for ct, obj_ids in preload.items():
                ct = ContentType.objects.get_for_id(ct)
                for obj in ct.model_class()._default_manager.filter(pk__in=obj_ids):
                    items[(ct.pk, obj.pk)] = obj

        results = []
        for result in qs:
            obj = items[
                tuple(result[k] for k in lookup_keys)
            ]
            obj.similar_tags = result["n"]
            results.append(obj)
        return results


class TaggableManager(RelatedField, Field):
    # Field flags
    many_to_many = True
    many_to_one = False
    one_to_many = False
    one_to_one = False

    _related_name_counter = 0

    def __init__(self, verbose_name=_("Tags"),
                 help_text=_("A comma-separated list of tags."),
                 through=None, blank=False, related_name=None, to=None,
                 manager=_TaggableManager):

        self.through = through or TaggedItem
        self.swappable = False
        self.manager = manager

        rel = TaggableRel(self, related_name, self.through, to=to)

        Field.__init__(
            self,
            verbose_name=verbose_name,
            help_text=help_text,
            blank=blank,
            null=True,
            serialize=False,
            rel=rel,
        )
        # NOTE: `to` is ignored, only used via `deconstruct`.
{'fraction_non_alphanumeric': 0.09982749986088699, 'fraction_numerical': 0.002448389071281509, 'mean_word_length': 2.3775606089081, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 8, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5587477', 'n_tokens_mistral': 5410, 'n_tokens_neox': 5224, 'n_words': 1183}