<commit_before>import gettext
import json

from django.core.management import call_command
from shutil import copyfile

import os

from collections import defaultdict
from django.conf import settings
from django.core.management.base import BaseCommand

from proso_flashcards.models import Context, Term, Flashcard
from geography.views import get_place_types


class Command(BaseCommand):
    help = "Generate concepts to JSON file"
    default_lang = "cs"

    def handle(self, *args, **options):
        def _get_lang(obj, lang):
            if lang in obj:
                return obj[lang]
            return obj[self.default_lang]

        domains = settings.LANGUAGE_DOMAINS
        langs = domains.keys()
        self.prepare_mo_files(langs)
        data = {
            "concepts": [],
            "action_names": {
                "practice": {},
                "view": {},
            },
            "tags": {
                "context": {
                    "names": {lang: "Oblast" for lang in langs},
                    "values": defaultdict(lambda: {})
                },
                "type": {
                    "names": {lang: "Typ místa" for lang in langs},
                    "values": defaultdict(lambda: {})
                },
            }
        }
        # TODO - add translations of tag names

        contexts = defaultdict(lambda: {})
        for context in Context.objects.all().values("identifier", "name", "lang"):
            contexts[context["identifier"]][context["lang"]] = context["name"]
            data["tags"]["context"]["values"][context["identifier"]][context["lang"]] = context["name"]
        types = Term.objects.all().values_list("type", flat=True).distinct()

        for lang in langs:
            translation = gettext.translation('djangojs', os.path.join(settings.BASE_DIR, "conf", "locale"), [lang])
            data["action_names"]["practice"][lang] = translation.gettext("Procvičovat")
            data["action_names"]["view"][lang] = translation.gettext("Přehled map")

        for context, lang_map in contexts.items():
            for type in types:
                languages = []
                for lang in langs:
                    if len(Flashcard.objects.filtered_ids([], [context], [type], [], lang)[0]) > 0:
                        languages.append(lang)
                if len(languages) == 0:
                    continue
                concept = {
                    "query": 'contexts=["{}"]&categories=[]&types=["{}"]'.format(context, type),
                    "tags": ["type:{}".format(type), "context:{}".format(context)],
                    "names": {},
                    "actions": {
                        "practice": {},
                        "view": {},
                    }
                }
                for lang in languages:
                    translation = gettext.translation('djangojs', os.path.join(settings.BASE_DIR, "conf", "locale"), [lang])
                    name = translation.gettext(get_place_types()[type])
                    concept["names"][lang] = "{} - {}".format(_get_lang(lang_map, lang), name)
                    data["tags"]["type"]["values"][type][lang] = name
                    concept["actions"]["practice"][lang] = "http://{}/practice/{}/{}".format(domains[lang], context, type)
                    concept["actions"]["view"][lang] = "http://{}/view/{}/{}".format(domains[lang], context, type)

                data["concepts"].append(concept)

        json.dump(data, open(os.path.join(settings.BASE_DIR, "data", "concepts.json"), "w"), indent=4, ensure_ascii=False)
        self.clean_up(langs)

    def prepare_mo_files(self, languages):
        for lang in languages:
            copyfile(os.path.join(settings.BASE_DIR, "geography", "static", "po", "{}.po".format(lang)),
                     os.path.join(settings.BASE_DIR, "conf", "locale", lang, "LC_MESSAGES", "djangojs.po"))
        call_command('compilemessages')

    def clean_up(self, languages):
        for lang in languages:
            os.remove(os.path.join(settings.BASE_DIR, "conf", "locale", lang, "LC_MESSAGES", "djangojs.po"))
            os.remove(os.path.join(settings.BASE_DIR, "conf", "locale", lang, "LC_MESSAGES", "djangojs.mo"))
<commit_msg>concepts - reflect new proso_models in generating script<commit_after>from collections import defaultdict
from django.conf import settings
from django.core.management import call_command
from django.core.management.base import BaseCommand
from proso_flashcards.models import Context, Category
from proso_models.models import Item
from shutil import copyfile
import gettext
import json
import os


class Command(BaseCommand):
    help = "Generate concepts to JSON file"
    default_lang = "cs"

    def handle(self, *args, **options):
        def _get_lang(obj, lang):
            if lang in obj:
                return obj[lang]
            return obj[self.default_lang]

        domains = settings.LANGUAGE_DOMAINS
        langs = domains.keys()
        self.prepare_mo_files(langs)
        data = {
            "concepts": [],
            "action_names": {
                "practice": {},
                "view": {},
            },
            "tags": {
                "context": {
                    "names": {lang: "Oblast" for lang in langs},
                    "values": defaultdict(lambda: {})
                },
                "type": {
                    "names": {lang: "Typ místa" for lang in langs},
                    "values": defaultdict(lambda: {})
                },
            }
        }
        # TODO - add translations of tag names

        contexts = defaultdict(lambda: {})
        for context in Context.objects.all().values("identifier", "name", "lang"):
            contexts[context["identifier"]][context["lang"]] = context["name"]
            data["tags"]["context"]["values"][context["identifier"]][context["lang"]] = context["name"]
        types = Category.objects.filter(type='flashcard_type').values_list("identifier", flat=True).distinct()

        for lang in langs:
            translation = gettext.translation('djangojs', os.path.join(settings.BASE_DIR, "conf", "locale"), [lang])
            data["action_names"]["practice"][lang] = translation.gettext("Procvičovat")
            data["action_names"]["view"][lang] = translation.gettext("Přehled map")

        for context, lang_map in contexts.items():
            for type in types:
                languages = []
                for lang in langs:
                    try:
                        if len(Item.objects.filter_all_reachable_leaves(
                                [["context/"+context, "category/"+type]], lang)) > 0:
                            languages.append(lang)
                    except Exception:
                        pass
                if len(languages) == 0:
                    continue
                concept = {
                    "query": '[["context/{}", "category/{}"]]'.format(context, type),
                    "tags": ["type:{}".format(type), "context:{}".format(context)],
                    "names": {},
                    "actions": {
                        "practice": {},
                        "view": {},
                    }
                }
                for lang in languages:
                    translation = gettext.translation('djangojs', os.path.join(settings.BASE_DIR, "conf", "locale"), [lang])
                    name = translation.gettext(get_place_types()[type])
                    concept["names"][lang] = "{} - {}".format(_get_lang(lang_map, lang), name)
                    data["tags"]["type"]["values"][type][lang] = name
                    concept["actions"]["practice"][lang] = "http://{}/practice/{}/{}".format(domains[lang], context, type)
                    concept["actions"]["view"][lang] = "http://{}/view/{}/{}".format(domains[lang], context, type)

                data["concepts"].append(concept)

        json.dump(data, open(os.path.join(settings.BASE_DIR, "data", "concepts.json"), "w"), indent=4, ensure_ascii=False)
        self.clean_up(langs)

    def prepare_mo_files(self, languages):
        for lang in languages:
            copyfile(os.path.join(settings.BASE_DIR, "geography", "static", "po", "{}.po".format(lang)),
                     os.path.join(settings.BASE_DIR, "conf", "locale", lang, "LC_MESSAGES", "djangojs.po"))
        call_command('compilemessages')

    def clean_up(self, languages):
        for lang in languages:
            os.remove(os.path.join(settings.BASE_DIR, "conf", "locale", lang, "LC_MESSAGES", "djangojs.po"))
            os.remove(os.path.join(settings.BASE_DIR, "conf", "locale", lang, "LC_MESSAGES", "djangojs.mo"))

{'fraction_non_alphanumeric': 0.15037334865020102, 'fraction_numerical': 0.0008041355542791499, 'mean_word_length': 2.0124567474048445, 'pattern_counts': {'":': 34, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7385400', 'n_tokens_mistral': 2435, 'n_tokens_neox': 2328, 'n_words': 486}