<commit_before>from datetime import datetime, timedelta
import time

import requests
import requests_cache

requests_cache.install_cache('demo_cache')
CACHE = requests_cache.get_cache()

from django.core.management.base import BaseCommand
from django.utils.text import slugify

from organisations.models import (
    Organisation,
    OrganisationDivision,
    OrganisationDivisionSet
)
from organisations.constants import PARENT_TO_CHILD_AREAS


class Command(BaseCommand):
    skip_gss = [
        "E12000007",
        "W08000001"  # See import_welsh_areas
    ]
    BASE = "http://mapit.mysociety.org"

    def add_arguments(self, parser):
        parser.add_argument(
            '--always_pick_option', action='store', type=int, default=0)

    def handle(self, **options):
        self.always_pick_option = int(options['always_pick_option'])
        self.load_mapit_generations()

        self.import_scottish_areas()
        self.import_gla_areas()
        self.import_parl_areas()
        self.import_ni_areas()
        self.import_welsh_areas()

        qs = Organisation.objects.exclude(gss='')
        qs = qs.exclude(gss=None)
        qs = qs.exclude(gss__startswith='N')
        qs = qs.exclude(gss__in=self.skip_gss)
        self.process_qs(qs)

    def process_qs(self, qs, default_child_types=None):
        for organisation in qs:
            initial_url = "{}/area/{}".format(self.BASE, organisation.gss)
            req = requests.get(initial_url)
            url = req.url

            if not default_child_types:
                parent_type = req.json()['type']
                child_types = PARENT_TO_CHILD_AREAS.get(parent_type)
            child_types = ",".join(child_types)
            req = requests.get("{}/children?type={}".format(url, child_types))
            self.import_divisions(organisation, req.json())

    def load_mapit_generations(self):
        url = "{}/generations".format(self.BASE)
        self.mapit_generations = requests.get(url).json()
        for generation_id, generation in self.mapit_generations.items():
            generation['uri'] = "{}/{}".format(url, generation_id)

    def create_single_division(self, division_set, organisation, mapit_dict):
        all_codes = [
            ('gss', mapit_dict['codes'].get('gss')),
            ('ons', mapit_dict['codes'].get('ons')),
            ('unit_id', mapit_dict['codes'].get('unit_id')),
            ('osni_oid', mapit_dict['codes'].get('osni_oid')),
        ]
        all_codes = [x for x in all_codes if x[1]]

        geography_curie = ":".join(all_codes[0])

        OrganisationDivision.objects.update_or_create(
            official_identifier=geography_curie,
            organisation=organisation,
            divisionset=division_set,
            defaults={
                'geography_curie': geography_curie,
                'name': mapit_dict['name'],
                'slug': slugify(mapit_dict['name']),
                'division_type': mapit_dict['type'],
                'division_subtype': mapit_dict['type_name'],
            }<commit_msg>Add mapit_generation when importing divisions<commit_after>from datetime import datetime, timedelta
import time

import requests
import requests_cache

requests_cache.install_cache('demo_cache')
CACHE = requests_cache.get_cache()

from django.core.management.base import BaseCommand
from django.utils.text import slugify

from organisations.models import (
    Organisation,
    OrganisationDivision,
    OrganisationDivisionSet
)
from organisations.constants import PARENT_TO_CHILD_AREAS


class Command(BaseCommand):
    skip_gss = [
        "E12000007",
        "W08000001"  # See import_welsh_areas
    ]
    BASE = "http://mapit.mysociety.org"

    def add_arguments(self, parser):
        parser.add_argument(
            '--always_pick_option', action='store', type=int, default=0)

    def handle(self, **options):
        self.always_pick_option = int(options['always_pick_option'])
        self.load_mapit_generations()

        self.import_scottish_areas()
        self.import_gla_areas()
        self.import_parl_areas()
        self.import_ni_areas()
        self.import_welsh_areas()

        qs = Organisation.objects.exclude(gss='')
        qs = qs.exclude(gss=None)
        qs = qs.exclude(gss__startswith='N')
        qs = qs.exclude(gss__in=self.skip_gss)
        self.process_qs(qs)

    def process_qs(self, qs, default_child_types=None):
        for organisation in qs:
            initial_url = "{}/area/{}".format(self.BASE, organisation.gss)
            req = requests.get(initial_url)
            url = req.url

            if not default_child_types:
                parent_type = req.json()['type']
                child_types = PARENT_TO_CHILD_AREAS.get(parent_type)
            child_types = ",".join(child_types)
            req = requests.get("{}/children?type={}".format(url, child_types))
            self.import_divisions(organisation, req.json())

    def load_mapit_generations(self):
        url = "{}/generations".format(self.BASE)
        self.mapit_generations = requests.get(url).json()
        for generation_id, generation in self.mapit_generations.items():
            generation['uri'] = "{}/{}".format(url, generation_id)

    def create_single_division(self, division_set, organisation, mapit_dict):
        all_codes = [
            ('gss', mapit_dict['codes'].get('gss')),
            ('ons', mapit_dict['codes'].get('ons')),
            ('unit_id', mapit_dict['codes'].get('unit_id')),
            ('osni_oid', mapit_dict['codes'].get('osni_oid')),
        ]
        all_codes = [x for x in all_codes if x[1]]

        geography_curie = ":".join(all_codes[0])

        OrganisationDivision.objects.update_or_create(
            official_identifier=geography_curie,
            organisation=organisation,
            divisionset=division_set,
            defaults={
                'geography_curie': geography_curie,
                'name': mapit_dict['name'],
                'slug': slugify(mapit_dict['name']),
                'division_type': mapit_dict['type'],
                'division_subtype': mapit_dict['type_name'],
                'mapit_generation_low': int(mapit_dict['generation_low']),
                'mapit_generation_high': int(mapit_dict['generation_high']),
            }
{'fraction_non_alphanumeric': 0.12119289340101523, 'fraction_numerical': 0.006027918781725888, 'mean_word_length': 2.741839762611276, 'pattern_counts': {'":': 2, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4234409', 'n_tokens_mistral': 2014, 'n_tokens_neox': 1916, 'n_words': 310}