<commit_before>from django.core.management.base import BaseCommand
from django.core.management import call_command
import os
from facilities.models import Variable, KeyRename
from utils.csv_reader import CsvReader


class Command(BaseCommand):
    help = "Load the LGAs from fixtures."

    def handle(self, *args, **kwargs):
        call_command('syncdb', interactive=False)
        self.load_lgas()
        csvs = [
            (Variable, os.path.join('facilities', 'fixtures', 'variables.csv')),
            (KeyRename, os.path.join('facilities', 'fixtures', 'key_renames.csv')),
            ]
        for model, path in csvs:
            self.create_objects_from_csv(model, path)
        self.load_surveys()
        self.create_admin_user()

    def load_lgas(self):
        for file_name in ['zone.json', 'state.json', 'lga.json']:
            call_command('loaddata', file_name)

    def create_objects_from_csv(self, model, path):
        csv_reader = CsvReader(path)
        for d in csv_reader.iter_dicts():
            model.objects.get_or_create(**d)

    def load_surveys(self):
        if not os.path.exists('xform_manager_dataset.json'):
            raise Exception("Download and unpack xform_manager_dataset.json into project dir.")
        call_command('loaddata', 'xform_manager_dataset.json')

    def create_admin_user(self):
        from django.contrib.auth.models import User
        admin, created = User.objects.get_or_create(
            username="admin",
            email="<EMAIL>",
            is_staff=True,
            is_superuser=True
            )
        admin.set_password("<PASSWORD>")
        admin.save()
<commit_msg>Added stats to load script.<commit_after>from django.core.management.base import BaseCommand
from django.core.management import call_command
import os
import json
from facilities.models import Facility, Variable, KeyRename, DataRecord
from utils.csv_reader import CsvReader


class Command(BaseCommand):
    help = "Load the LGAs from fixtures."

    def handle(self, *args, **kwargs):
        call_command('syncdb', interactive=False)
        self.print_stats()
        self.load_lgas()
        csvs = [
            (Variable, os.path.join('facilities', 'fixtures', 'variables.csv')),
            (KeyRename, os.path.join('facilities', 'fixtures', 'key_renames.csv')),
            ]
        for model, path in csvs:
            self.create_objects_from_csv(model, path)
        self.load_surveys()
        self.create_admin_user()
        self.print_stats()

    def print_stats(self):
        info = {
            'number of facilities': Facility.objects.count(),
            'facilities without lgas': Facility.objects.filter(lga=None).count(),
            'number of data records': DataRecord.objects.count(),
            }
        print json.dumps(info, indent=4)

    def load_lgas(self):
        for file_name in ['zone.json', 'state.json', 'lga.json']:
            call_command('loaddata', file_name)

    def create_objects_from_csv(self, model, path):
        csv_reader = CsvReader(path)
        for d in csv_reader.iter_dicts():
            model.objects.get_or_create(**d)

    def load_surveys(self):
        if not os.path.exists('xform_manager_dataset.json'):
            raise Exception("Download and unpack xform_manager_dataset.json into project dir.")
        call_command('loaddata', 'xform_manager_dataset.json')

    def create_admin_user(self):
        from django.contrib.auth.models import User
        admin, created = User.objects.get_or_create(
            username="admin",
            email="<EMAIL>",
            is_staff=True,
            is_superuser=True
            )
        admin.set_password("<PASSWORD>")
        admin.save()

{'fraction_non_alphanumeric': 0.11182795698924732, 'fraction_numerical': 0.00026881720430107527, 'mean_word_length': 2.6732477788746296, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6640878', 'n_tokens_mistral': 1110, 'n_tokens_neox': 1077, 'n_words': 235}