<commit_before>from django.core.management.base import NoArgsCommand

from crashmanager.management.common import mgmt_lock_required
from crashmanager.models import CrashEntry, Bucket


class Command(NoArgsCommand):
    help = "Iterates over all unbucketed crash entries that have never been triaged before to assign them into the existing buckets."
    @mgmt_lock_required
    def handle_noargs(self, **options):
        entries = CrashEntry.objects.filter(triagedOnce=False, bucket=None)
        buckets = Bucket.objects.all()

        for bucket in buckets:
            signature = bucket.getSignature()
            needTest = signature.matchRequiresTest()

            for entry in entries:
                if signature.matches(entry.getCrashInfo(attachTestcase=needTest)):
                    entry.bucket = bucket

                entry.triagedOnce = True
                entry.save()

        # This query ensures that all issues that have been bucketed manually before
        # the server had a chance to triage them will have their triageOnce flag set,
        # so the hourglass in the UI isn't displayed anymore.
        CrashEntry.objects.exclude(bucket=None).update(triagedOnce=True)<commit_msg>[CrashManager] Optimize triage_new_crashes command (#230)<commit_after>from django.core.management.base import NoArgsCommand

from crashmanager.management.common import mgmt_lock_required
from crashmanager.models import CrashEntry, Bucket


class Command(NoArgsCommand):
    help = "Iterates over all unbucketed crash entries that have never been triaged before to assign them into the existing buckets."
    @mgmt_lock_required
    def handle_noargs(self, **options):
        entries = CrashEntry.objects.filter(triagedOnce=False, bucket=None)
        buckets = Bucket.objects.all()

        signatureCache = {}

        for entry in entries:
            # Benchmarks have shown that always attaching the testcase
            # is faster than recreating the crash info in every iteration
            # when signatures can be cached.
            crashInfo = entry.getCrashInfo(attachTestcase=True)

            for bucket in buckets:
                if not bucket.pk in signatureCache:
                    signatureCache[bucket.pk] = bucket.getSignature()

                signature = signatureCache[bucket.pk]
                matched = False

                if signature.matches(crashInfo):
                    entry.bucket = bucket
                    matched = True

                entry.triagedOnce = True
                entry.save()

                if matched:
                    break

        # This query ensures that all issues that have been bucketed manually before
        # the server had a chance to triage them will have their triageOnce flag set,
        # so the hourglass in the UI isn't displayed anymore.
        CrashEntry.objects.exclude(bucket=None).update(triagedOnce=True)
{'fraction_non_alphanumeric': 0.06013745704467354, 'fraction_numerical': 0.0010309278350515464, 'mean_word_length': 2.498798076923077, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6035019', 'n_tokens_mistral': 736, 'n_tokens_neox': 691, 'n_words': 239}