<commit_before>from django.conf import settings  # noqa
from django.core.files.base import ContentFile

from celeryconf import app  # noqa

import hashlib
import json


@app.task
def check_revision_update(pk):
    from covmanager.models import Collection, Repository  # noqa
    collection = Collection.objects.get(pk=pk)

    # Get the SourceCodeProvider associated with this collection
    provider = collection.repository.getInstance()

    # Check if the provider knows the specified revision
    if not provider.testRevision(collection.revision):
        # If not, update the repository
        provider.update()

    # TODO: We could double-check here that the revision is now known
    # and raise an error if not. This error would have to be propagated
    # to the user and since we already saved the collection, we can't
    # reject the client anymore.

    return


@app.task
def aggregate_coverage_data(pk, pks):
    from covmanager.models import Collection, CollectionFile  # noqa
    from FTB import CoverageHelper # noqa

    # Fetch our existing, but incomplete destination collection
    mergedCollection = Collection.objects.get(pk=pk)

    # Fetch all source collections
    collections = Collection.objects.filter(pk__in=pks)

    # Merge the coverage of all other collections into the first one
    first_collection = collections[0]
    first_collection.loadCoverage()
    newCoverage = first_collection.content
    for collection in collections[1:]:
        # Load coverage, perform the merge, then release reference to the JSON blob again
        collection.loadCoverage()
        CoverageHelper.merge_coverage_data(newCoverage, collection.content)
        collection.content = None

    # Save the new coverage blob to disk and database
    newCoverage = json.dumps(newCoverage, separators=(',', ':'))
    h = hashlib.new('sha1')
    h.update(newCoverage.encode('utf-8'))
    dbobj = CollectionFile()
    dbobj.file.save("%s.coverage" % h.hexdigest(), ContentFile(newCoverage))
    dbobj.save()

    # Save the collection<commit_msg>[CovManager] Add debug stats to aggregated collection description<commit_after>from django.conf import settings  # noqa
from django.core.files.base import ContentFile

from celeryconf import app  # noqa

import hashlib
import json


@app.task
def check_revision_update(pk):
    from covmanager.models import Collection, Repository  # noqa
    collection = Collection.objects.get(pk=pk)

    # Get the SourceCodeProvider associated with this collection
    provider = collection.repository.getInstance()

    # Check if the provider knows the specified revision
    if not provider.testRevision(collection.revision):
        # If not, update the repository
        provider.update()

    # TODO: We could double-check here that the revision is now known
    # and raise an error if not. This error would have to be propagated
    # to the user and since we already saved the collection, we can't
    # reject the client anymore.

    return


@app.task
def aggregate_coverage_data(pk, pks):
    from covmanager.models import Collection, CollectionFile  # noqa
    from FTB import CoverageHelper # noqa

    # Fetch our existing, but incomplete destination collection
    mergedCollection = Collection.objects.get(pk=pk)

    # Fetch all source collections
    collections = Collection.objects.filter(pk__in=pks)

    # Merge the coverage of all other collections into the first one
    first_collection = collections[0]
    first_collection.loadCoverage()
    newCoverage = first_collection.content
    total_stats = None

    for collection in collections[1:]:
        # Load coverage, perform the merge, then release reference to the JSON blob again
        collection.loadCoverage()
        stats = CoverageHelper.merge_coverage_data(newCoverage, collection.content)
        collection.content = None

        # Merge stats appropriately
        if total_stats is None:
            total_stats = stats
        else:
            for x in total_stats:
                total_stats[x] += stats[x]

    # Save the new coverage blob to disk and database
    newCoverage = json.dumps(newCoverage, separators=(',', ':'))
    h = hashlib.new('sha1')
    h.update(newCoverage.encode('utf-8'))
    dbobj = CollectionFile()
    dbobj.file.save("%s.coverage" % h.hexdigest(), ContentFile(newCoverage))
    dbobj.save()

    mergedCollection.description += " (NC %s, LM %s, CM %s)" % (stats['null_coverable_count'],
                                                                stats['length_mismatch_count'],
                                                                stats['coverable_mismatch_count'])

    # Save the collection
{'fraction_non_alphanumeric': 0.07516549220585095, 'fraction_numerical': 0.001708306641042067, 'mean_word_length': 3.1341571050308916, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4197131', 'n_tokens_mistral': 1264, 'n_tokens_neox': 1205, 'n_words': 411}