<commit_before>from __future__ import unicode_literals

from collections import OrderedDict

from django.db import models
from django.conf import settings

from django.contrib.auth.models import User, Group
from django.contrib.contenttypes.models import ContentType
from django.contrib.contenttypes.fields import GenericForeignKey
from django_redis import get_redis_connection

from event_store.models import Organization


_ORGANIZATIONS = {}  # by slug

class ReviewGroup(models.Model):
    """
    many-to-many link of what groups can review for what organizations
    (TODO) Eventually, we may want a workflow that allows someone
    from one organization to vet events from another for the purposes
    of copying them over.  In that case, the org would allow sharing (see event_exim app)
    and wouldn't be added to the other org here, but a user might also be a member
    of more than one org, and both orgs might be ok with 'sharing' the vetting
    for the same event (assuming they have the same criteria).
    """
    organization = models.ForeignKey(Organization, db_index=True)
    group = models.ForeignKey(Group, db_index=True)

    class Meta:
        unique_together = ('organization', 'group')

    @classmethod
    def user_allowed(cls, user, organization):
        allowed = ReviewGroup.org_groups(organization)
        allowed_groups = set([x.group_id for x in allowed])
        group_ids = set(user.groups.values_list('id', flat=True))
        return (group_ids.intersection(allowed_groups)
                or user.is_superuser)

    @classmethod
    def org_groups(cls, organization_slug):
        """
        memo-ize orgs--it'll be a long time before we're too big for memory
        """
        if organization_slug not in _ORGANIZATIONS:
            _ORGANIZATIONS[organization_slug] = list(ReviewGroup.objects.filter(
                organization__slug=organization_slug))
        return _ORGANIZATIONS.get(organization_slug)

    @classmethod
    def user_review_groups(cls, user):
        return ReviewGroup.objects.filter(group__user=user)

    def __str__(self):
        return '{} for {}'.format(self.group, self.organization)

class Review(models.Model):
    content_type = models.ForeignKey(ContentType,
                                     editable=False,
                                     blank=True)
    object_id = models.IntegerField()
    content_object = GenericForeignKey('content_type', 'object_id')

    created_at = models.DateTimeField(auto_now_add=True)
    modified_at = models.DateTimeField(auto_now=True)

    organization = models.ForeignKey(Organization)
    reviewer = models.ForeignKey(User)
    key = models.CharField(max_length=128,
                           help_text=("the data key for what is being reviewed"
                                      " -- this allows multiple keys per-object"),
                           default="review")
    decision = models.CharField(max_length=128)
    is_current = models.BooleanField(default=True)

    @classmethod
    def bulk_add_tag(cls, content_queryset, organization, reviewer, key, decision):
        """
        Adds one tag at a time to selected object instances
        TODO: this may require more work when supporting multi-select
        """
        content_type = ContentType.objects.get_for_model(content_queryset.model)
        obj_ids = [x.id for x in content_queryset]
        result = Review.objects.bulk_create([
            Review(organization=organization,
                   reviewer=reviewer,
                   key=key,
                   decision=decision,
                   object_id=obj_id,
                   content_type=content_type)
            for obj_id in obj_ids
        ])
        cls.bulk_clear_review_cache(obj_ids, content_type.id, organization)
        return result

    @classmethod
    def bulk_delete_tag(cls, content_queryset, organization, reviewer, key):
        """
        Deletes one tag at a time from selected object instances
        TODO: this may require more work when supporting multi-select
        """
        content_type = ContentType.objects.get_for_model(content_queryset.model)
        obj_ids = [x.id for x in content_queryset]
        result = Review.objects.filter(
            organization=organization,
            reviewer=reviewer,
            key=key,
            content_type=content_type,
            object_id__in=obj_ids
        ).update(is_current=False)
        cls.bulk_clear_review_cache(obj_ids, content_type.id, organization)
        return result

    @classmethod
    def bulk_clear_review_cache(cls, obj_ids, content_type_id, organization):
        """
        Clears cached reviews so we can update the display accurately
        after bulk delete/add
        """
        REDIS_CACHE_KEY = getattr(settings, 'REVIEWER_CACHE_KEY', 'default')
        redis = get_redis_connection(REDIS_CACHE_KEY)
        reviewskey = '{}_reviews'.format(organization.slug)
        obj_keys = ['{}_{}'.format(content_type_id, x) for x in obj_ids]
        if obj_keys:<commit_msg>missing filter for updated review model might be causing bulk tag deletion issues<commit_after>from __future__ import unicode_literals

from collections import OrderedDict

from django.db import models
from django.conf import settings

from django.contrib.auth.models import User, Group
from django.contrib.contenttypes.models import ContentType
from django.contrib.contenttypes.fields import GenericForeignKey
from django_redis import get_redis_connection

from event_store.models import Organization


_ORGANIZATIONS = {}  # by slug

class ReviewGroup(models.Model):
    """
    many-to-many link of what groups can review for what organizations
    (TODO) Eventually, we may want a workflow that allows someone
    from one organization to vet events from another for the purposes
    of copying them over.  In that case, the org would allow sharing (see event_exim app)
    and wouldn't be added to the other org here, but a user might also be a member
    of more than one org, and both orgs might be ok with 'sharing' the vetting
    for the same event (assuming they have the same criteria).
    """
    organization = models.ForeignKey(Organization, db_index=True)
    group = models.ForeignKey(Group, db_index=True)

    class Meta:
        unique_together = ('organization', 'group')

    @classmethod
    def user_allowed(cls, user, organization):
        allowed = ReviewGroup.org_groups(organization)
        allowed_groups = set([x.group_id for x in allowed])
        group_ids = set(user.groups.values_list('id', flat=True))
        return (group_ids.intersection(allowed_groups)
                or user.is_superuser)

    @classmethod
    def org_groups(cls, organization_slug):
        """
        memo-ize orgs--it'll be a long time before we're too big for memory
        """
        if organization_slug not in _ORGANIZATIONS:
            _ORGANIZATIONS[organization_slug] = list(ReviewGroup.objects.filter(
                organization__slug=organization_slug))
        return _ORGANIZATIONS.get(organization_slug)

    @classmethod
    def user_review_groups(cls, user):
        return ReviewGroup.objects.filter(group__user=user)

    def __str__(self):
        return '{} for {}'.format(self.group, self.organization)

class Review(models.Model):
    content_type = models.ForeignKey(ContentType,
                                     editable=False,
                                     blank=True)
    object_id = models.IntegerField()
    content_object = GenericForeignKey('content_type', 'object_id')

    created_at = models.DateTimeField(auto_now_add=True)
    modified_at = models.DateTimeField(auto_now=True)

    organization = models.ForeignKey(Organization)
    reviewer = models.ForeignKey(User)
    key = models.CharField(max_length=128,
                           help_text=("the data key for what is being reviewed"
                                      " -- this allows multiple keys per-object"),
                           default="review")
    decision = models.CharField(max_length=128)
    is_current = models.BooleanField(default=True)

    @classmethod
    def bulk_add_tag(cls, content_queryset, organization, reviewer, key, decision):
        """
        Adds one tag at a time to selected object instances
        TODO: this may require more work when supporting multi-select
        """
        content_type = ContentType.objects.get_for_model(content_queryset.model)
        obj_ids = [x.id for x in content_queryset]
        result = Review.objects.bulk_create([
            Review(organization=organization,
                   reviewer=reviewer,
                   key=key,
                   decision=decision,
                   object_id=obj_id,
                   content_type=content_type)
            for obj_id in obj_ids
        ])
        cls.bulk_clear_review_cache(obj_ids, content_type.id, organization)
        return result

    @classmethod
    def bulk_delete_tag(cls, content_queryset, organization, reviewer, key):
        """
        Deletes one tag at a time from selected object instances
        TODO: this may require more work when supporting multi-select
        """
        content_type = ContentType.objects.get_for_model(content_queryset.model)
        obj_ids = [x.id for x in content_queryset]
        result = Review.objects.filter(
            organization=organization,
            reviewer=reviewer,
            key=key,
            content_type=content_type,
            object_id__in=obj_ids,
            is_current=True
        ).update(is_current=False)
        cls.bulk_clear_review_cache(obj_ids, content_type.id, organization)
        return result

    @classmethod
    def bulk_clear_review_cache(cls, obj_ids, content_type_id, organization):
        """
        Clears cached reviews so we can update the display accurately
        after bulk delete/add
        """
        REDIS_CACHE_KEY = getattr(settings, 'REVIEWER_CACHE_KEY', 'default')
        redis = get_redis_connection(REDIS_CACHE_KEY)
        reviewskey = '{}_reviews'.format(organization.slug)
        obj_keys = ['{}_{}'.format(content_type_id, x) for x in obj_ids]
        if obj_keys:
{'fraction_non_alphanumeric': 0.07471320717717424, 'fraction_numerical': 0.0011765859397980193, 'mean_word_length': 2.7010159651669086, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 8, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2596677', 'n_tokens_mistral': 2812, 'n_tokens_neox': 2714, 'n_words': 758}