<commit_before>from optparse import make_option
from xbrowse_server import xbrowse_controls
from django.core.management.base import BaseCommand


class Command(BaseCommand):

    def handle(self, *args, **options):
        from xbrowse_server import mall
        mall.get_custom_population_store().load()
<commit_msg>Load specific populations<commit_after>from optparse import make_option
from xbrowse_server import xbrowse_controls
from django.core.management.base import BaseCommand
from django.conf import settings
from xbrowse_server.base.models import ReferencePopulation

class Command(BaseCommand):
    option_list = BaseCommand.option_list + (
        make_option('--some-option'),
    )


    def handle(self, *args, **options):
        from xbrowse_server import mall
        if len(args) == 0:
            print("Global: " + str([slug for slug in settings.ANNOTATOR_REFERENCE_POPULATION_SLUGS]))
            print("Private: " + str([p.slug for p in ReferencePopulation.objects.all()]))
        else:
            pop_store = mall.get_custom_population_store()
            pop_store._ensure_indices()
            print("Loading population: " + args[0])

            requested_slug = args[0]
            projects = [p for p in settings.ANNOTATOR_REFERENCE_POPULATIONS if p["slug"] == requested_slug] + \
                [p.to_dict() for p in ReferencePopulation.objects.all() if p.slug == requested_slug]

            assert len(projects) == 1
            pop_store.load_population(projects[0])


        #[{'slug': s['slug'], 'name': s['name']} for s in settings.ANNOTATOR_REFERENCE_POPULATIONS] +
        #[{'slug': s.slug, 'name': s.name} for s in self.private_reference_populations.all()]


{'fraction_non_alphanumeric': 0.10517038777908343, 'fraction_numerical': 0.002937720329024677, 'mean_word_length': 3.3891752577319587, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6346318', 'n_tokens_mistral': 516, 'n_tokens_neox': 498, 'n_words': 119}