<commit_before>import itertools

from django.conf import settings
from django.core.cache import cache
from django.db import models
from django.utils import translation

from tower import ugettext_lazy as _

import amo.models
from translations.fields import TranslatedField
from translations.models import Translation
from users.models import UserProfile
from versions.models import Version


class ReviewManager(amo.models.ManagerBase):

    def get_query_set(self):
        qs = super(ReviewManager, self).get_query_set()
        return qs.transform(Review.transformer)

    def valid(self):
        """Get all reviews with rating > 0 that aren't replies."""
        return self.filter(reply_to=None, rating__gt=0)

    def latest(self):
        """Get all the latest valid reviews."""
        return self.valid().filter(is_latest=True)


class Review(amo.models.ModelBase):
    addon = models.ForeignKey('addons.Addon', related_name='_reviews')
    version = models.ForeignKey('versions.Version', related_name='reviews',
                                null=True)
    user = models.ForeignKey('users.UserProfile', related_name='_reviews_all')
    reply_to = models.ForeignKey('self', null=True, unique=True,
                                 related_name='replies', db_column='reply_to')

    rating = models.PositiveSmallIntegerField(null=True)
    title = TranslatedField()
    body = TranslatedField()
    ip_address = models.CharField(max_length=255, default='0.0.0.0')

    editorreview = models.BooleanField(default=False)
    flag = models.BooleanField(default=False)
    sandbox = models.BooleanField(default=False)

    # Denormalized fields for easy lookup queries.
    # TODO: index on addon, user, latest
    is_latest = models.BooleanField(default=True, editable=False,
        help_text="Is this the user's latest review for the add-on?")
    previous_count = models.PositiveIntegerField(default=0, editable=False,
        help_text="How many previous reviews by the user for this add-on?")

    objects = ReviewManager()

    class Meta:
        db_table = 'reviews'
        ordering = ('-created',)

    def flush_urls(self):
        urls = ['*/addon/%d/' % self.addon_id,
                '*/addon/%d/reviews/' % self.addon_id,
                '*/addon/%d/reviews/format:rss' % self.addon_id,
                '*/addon/%d/reviews/%d/' % (self.addon_id, self.id),
                '*/user/%d/' % self.user.id, ]
        return urls

    @classmethod
    def fetch_translations(cls, ids, lang):
        if not ids:
            return []

        rv = {}
        ts = Translation.objects.filter(id__in=ids)

        # If a translation exists for the current language, use it.  Otherwise,
        # make do with whatever is available.  (Reviewers only write reviews in
        # their language).
        for id, translations in itertools.groupby(ts, lambda t: t.id):
            locales = dict((t.locale, t) for t in translations)
            if lang in locales:
                rv[id] = locales[lang]
            else:
                rv[id] = locales.itervalues().next()

        return rv.values()

    @staticmethod
    def post_save(sender, instance, created, **kwargs):
        if created:
            Review.post_delete(sender, instance)

    @staticmethod
    def post_delete(sender, instance, **kwargs):
        from . import tasks
        pair = instance.addon_id, instance.user_id
        # Do this immediately so is_latest is correct.
        tasks.update_denorm(pair)
        # Use default so we don't hit slave lag.
        tasks.addon_review_aggregates.delay(instance.addon_id, using='default')

    @staticmethod
    def transformer(reviews):
        if not reviews:
            return

        # Attach users.
        user_ids = [r.user_id for r in reviews]
        users = UserProfile.objects.filter(id__in=user_ids)
        user_dict = dict((u.id, u) for u in users)
        for review in reviews:
            review.user = user_dict[review.user_id]

        # Attach translations. Some of these will be picked up by the
        # Translation transformer, but reviews have special requirements
        # (see fetch_translations).
        names = dict((f.attname, f.name)
                     for f in Review._meta.translated_fields)
        ids, trans = {}, {}
        for review in reviews:
            for attname, name in names.items():
                trans_id = getattr(review, attname)
                if getattr(review, name) is None and trans_id is not None:
                    ids[trans_id] = attname
                    trans[trans_id] = review
        translations = Review.fetch_translations(trans.keys(),
                                                 translation.get_language())<commit_msg>more slave lag avoidance (bug 583367)<commit_after>import itertools

from django.conf import settings
from django.core.cache import cache
from django.db import models
from django.utils import translation

from tower import ugettext_lazy as _

import amo.models
from translations.fields import TranslatedField
from translations.models import Translation
from users.models import UserProfile
from versions.models import Version


class ReviewManager(amo.models.ManagerBase):

    def get_query_set(self):
        qs = super(ReviewManager, self).get_query_set()
        return qs.transform(Review.transformer)

    def valid(self):
        """Get all reviews with rating > 0 that aren't replies."""
        return self.filter(reply_to=None, rating__gt=0)

    def latest(self):
        """Get all the latest valid reviews."""
        return self.valid().filter(is_latest=True)


class Review(amo.models.ModelBase):
    addon = models.ForeignKey('addons.Addon', related_name='_reviews')
    version = models.ForeignKey('versions.Version', related_name='reviews',
                                null=True)
    user = models.ForeignKey('users.UserProfile', related_name='_reviews_all')
    reply_to = models.ForeignKey('self', null=True, unique=True,
                                 related_name='replies', db_column='reply_to')

    rating = models.PositiveSmallIntegerField(null=True)
    title = TranslatedField()
    body = TranslatedField()
    ip_address = models.CharField(max_length=255, default='0.0.0.0')

    editorreview = models.BooleanField(default=False)
    flag = models.BooleanField(default=False)
    sandbox = models.BooleanField(default=False)

    # Denormalized fields for easy lookup queries.
    # TODO: index on addon, user, latest
    is_latest = models.BooleanField(default=True, editable=False,
        help_text="Is this the user's latest review for the add-on?")
    previous_count = models.PositiveIntegerField(default=0, editable=False,
        help_text="How many previous reviews by the user for this add-on?")

    objects = ReviewManager()

    class Meta:
        db_table = 'reviews'
        ordering = ('-created',)

    def flush_urls(self):
        urls = ['*/addon/%d/' % self.addon_id,
                '*/addon/%d/reviews/' % self.addon_id,
                '*/addon/%d/reviews/format:rss' % self.addon_id,
                '*/addon/%d/reviews/%d/' % (self.addon_id, self.id),
                '*/user/%d/' % self.user.id, ]
        return urls

    @classmethod
    def fetch_translations(cls, ids, lang):
        if not ids:
            return []

        rv = {}
        ts = Translation.objects.filter(id__in=ids)

        # If a translation exists for the current language, use it.  Otherwise,
        # make do with whatever is available.  (Reviewers only write reviews in
        # their language).
        for id, translations in itertools.groupby(ts, lambda t: t.id):
            locales = dict((t.locale, t) for t in translations)
            if lang in locales:
                rv[id] = locales[lang]
            else:
                rv[id] = locales.itervalues().next()

        return rv.values()

    @staticmethod
    def post_save(sender, instance, created, **kwargs):
        if created:
            Review.post_delete(sender, instance)

    @staticmethod
    def post_delete(sender, instance, **kwargs):
        from . import tasks
        pair = instance.addon_id, instance.user_id
        # Do this immediately so is_latest is correct. Use default to avoid
        # slave lag.
        tasks.update_denorm(pair, using='default')
        tasks.addon_review_aggregates.delay(instance.addon_id, using='default')

    @staticmethod
    def transformer(reviews):
        if not reviews:
            return

        # Attach users.
        user_ids = [r.user_id for r in reviews]
        users = UserProfile.objects.filter(id__in=user_ids)
        user_dict = dict((u.id, u) for u in users)
        for review in reviews:
            review.user = user_dict[review.user_id]

        # Attach translations. Some of these will be picked up by the
        # Translation transformer, but reviews have special requirements
        # (see fetch_translations).
        names = dict((f.attname, f.name)
                     for f in Review._meta.translated_fields)
        ids, trans = {}, {}
        for review in reviews:
            for attname, name in names.items():
                trans_id = getattr(review, attname)
                if getattr(review, name) is None and trans_id is not None:
                    ids[trans_id] = attname
                    trans[trans_id] = review
        translations = Review.fetch_translations(trans.keys(),
                                                 translation.get_language())
{'fraction_non_alphanumeric': 0.09917963819941102, 'fraction_numerical': 0.0027345393352965924, 'mean_word_length': 2.4316131360519666, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 2, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 6, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3356439', 'n_tokens_mistral': 2731, 'n_tokens_neox': 2619, 'n_words': 717}