<commit_before>from datetime import datetime

from django.core.management.base import BaseCommand

from mygpo import settings
from mygpo.directory.models import Category
from mygpo.data.models import DirectoryEntry


class Command(BaseCommand):

    def handle(self, *args, **options):

        start_time = datetime.utcnow()

        excluded_tags = getattr(settings, 'DIRECTORY_EXCLUDED_TAGS', [])

        top_tags = DirectoryEntry.objects.top_tags(None)
        for tag in top_tags:
            #print tag.tag

            category = Category.for_tag(tag.tag)
            if category:

                # delete if it has been excluded after it has been created
                if tag.tag in excluded_tags:
                    category.delete()
                    continue

                if category.updated < start_time:
                    category.weight = tag.entries
                else:
                    category.weight += tag.entries

            else:

                if tag.tag in excluded_tags:
                    continue

                label = tag.tag.strip()
                if not label:
                    continue
                category = Category()
                category.label = label
                category.spellings = []
                category.weight = tag.entries


            category.updated = start_time
            category.save()

<commit_msg>avoid duplicate categories<commit_after>from datetime import datetime

from django.core.management.base import BaseCommand

from mygpo import settings
from mygpo.directory.models import Category
from mygpo.data.models import DirectoryEntry


class Command(BaseCommand):

    def handle(self, *args, **options):

        start_time = datetime.utcnow()

        excluded_tags = getattr(settings, 'DIRECTORY_EXCLUDED_TAGS', [])

        top_tags = DirectoryEntry.objects.top_tags(None)
        for tag in top_tags:
            label = tag.tag.strip()

            category = Category.for_tag(label)
            if category:

                # delete if it has been excluded after it has been created
                if label in excluded_tags:
                    category.delete()
                    continue

                if category.updated < start_time:
                    category.weight = tag.entries
                else:
                    category.weight += tag.entries

            else:

                if label in excluded_tags:
                    continue

                if not label:
                    continue
                category = Category()
                category.label = label
                category.spellings = []
                category.weight = tag.entries


            category.updated = start_time
            category.save()


{'fraction_non_alphanumeric': 0.06408399710354816, 'fraction_numerical': 0.0, 'mean_word_length': 1.5279048490393412, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1083939', 'n_tokens_mistral': 686, 'n_tokens_neox': 624, 'n_words': 180}