<commit_before># -*- coding: utf-8 -*-
from __future__ import absolute_import, print_function, unicode_literals

from django.db.models import Count
from django.utils.translation import ugettext_lazy as _

from filer.models import File
from filer.models import Image

from .base import SubcommandsCommand


class ResolveDuplicatesCommand(SubcommandsCommand):
    help_string = _('resolve duplicates (remove detected duplicates, assign original instead)')
    command_name = 'resolve_duplicates'

    def handle(self, *args, **options):
        self.resolve_for_class(File)
        self.resolve_for_class(Image)

    def resolve_for_class(self, cls):
        # exit("untested code! exiting!")
        # file dups
        temp = cls.objects.values('sha1').annotate(Count('id')).values('sha1').order_by().filter(id__count__gt=1)
        # keep the oldes, that is probably longest in search indexes...
        duplicates = cls.objects.filter(sha1__in=temp).order_by('uploaded_at')
        duplicates_count = duplicates.distinct().count()
        print("resolving {} duplicates in {}!".format(duplicates_count, cls.__name__))
        done = {}
        for file in duplicates:
            # do we have an original!?
            if file.sha1 in done:
                # we have an original. go through all related objects for this file.
                try:
                    file._meta._related_objects_cache
                except AttributeError:
                    file._meta._fill_related_objects_cache()
                for related in file._meta._related_objects_cache:
                    # get all of the same model that use this file
                    kwargs = {
                        related.field.name: file.id,
                    }
                    affected = related.model.objects.filter(**kwargs)
                    # update them to use the original
                    kwargs = {
                        related.field.name: done[file.sha1],
                    }
                    affected.update(**kwargs)
                # DELETE
                file.delete()
            else:
                # first time, keep this file
                done[file.sha1] = file.id
<commit_msg>wip resolve duplicates<commit_after># -*- coding: utf-8 -*-
from __future__ import absolute_import, print_function, unicode_literals

from django.db.models import Count
from django.utils.translation import ugettext_lazy as _

from filer.models import File
from filer.models import Image

from .base import SubcommandsCommand


class ResolveDuplicatesCommand(SubcommandsCommand):
    help_string = _('resolve duplicates (remove detected duplicates, assign original instead)')
    command_name = 'resolve_duplicates'

    def handle(self, *args, **options):
        self.resolve_for_class(File)
        self.resolve_for_class(Image)

    def resolve_for_class(self, cls):
        # exit("untested code! exiting!")
        # file dups
        temp = cls.objects.values('sha1').annotate(Count('id')).values('sha1').order_by().filter(id__count__gt=1)
        # keep the oldes, that is probably longest in search indexes...
        duplicates = cls.objects.filter(sha1__in=temp).order_by('uploaded_at')
        duplicates_count = duplicates.distinct().count()
        print("resolving {} duplicates in {}!".format(duplicates_count, cls.__name__))
        done = {}
        for file in duplicates:
            # do we have an original!?
            if file.sha1 in done:
                # we have an original. go through all related objects for this file.
                for related in file._meta._related_objects:
                    # get all of the same model that use this file
                    kwargs = {
                        related.field.name: file.id,
                    }
                    affected = related.model.objects.filter(**kwargs)
                    # update them to use the original
                    kwargs = {
                        related.field.name: done[file.sha1],
                    }
                    affected.update(**kwargs)
                # DELETE
                file.delete()
            else:
                # first time, keep this file
                done[file.sha1] = file.id

{'fraction_non_alphanumeric': 0.08901515151515152, 'fraction_numerical': 0.003787878787878788, 'mean_word_length': 1.966994382022472, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3405892', 'n_tokens_mistral': 1139, 'n_tokens_neox': 1072, 'n_words': 293}