<commit_before>import re
from optparse import make_option
from django.db import transaction
from django.core.management.base import BaseCommand
from django.conf import settings
from ...models import DivisionGeometry
from opencivicdata.divisions import Division
from boundaries.models import BoundarySet


def load_mapping(boundary_set_id, key, prefix, boundary_key='external_id', ignore=None, quiet=False):
    if ignore:
        ignore = re.compile(ignore)
    ignored = 0
    geoid_mapping = {}

    division_geometries = []

    for div in Division.get('ocd-division/country:' + settings.IMAGO_COUNTRY).children(levels=100):
        if div.attrs[key]:
            geoid_mapping[div.attrs[key]] = div.id

    print('processing', boundary_set_id)

    boundary_set = BoundarySet.objects.get(pk=boundary_set_id)
    if callable(boundary_key):
        fields = []
    else:
        fields = [boundary_key]
    for boundary in boundary_set.boundaries.values('id', 'name', *fields):
        if callable(boundary_key):
            boundary_property = boundary_key(boundary)
        else:
            boundary_property = boundary[boundary_key]
        ocd_id = geoid_mapping.get(prefix + boundary_property)<commit_msg>Making sure that we always find division IDs when mapping to boundaries<commit_after>import re
from optparse import make_option
from django.db import transaction
from django.core.management.base import BaseCommand
from django.conf import settings
from ...models import DivisionGeometry
from opencivicdata.divisions import Division
from boundaries.models import BoundarySet


def load_mapping(boundary_set_id, key, prefix, boundary_key='external_id', ignore=None, quiet=False):
    if ignore:
        ignore = re.compile(ignore)
    ignored = 0
    geoid_mapping = {}

    division_geometries = []

    for div in Division.get('ocd-division/country:' + settings.IMAGO_COUNTRY).children(levels=100):
        if div.attrs[key]:
            geoid_mapping[div.attrs[key]] = div.id
        else:
            geoid_mapping[div.id] = div.id
    
    print('processing', boundary_set_id)

    boundary_set = BoundarySet.objects.get(pk=boundary_set_id)
    if callable(boundary_key):
        fields = []
    else:
        fields = [boundary_key]
    for boundary in boundary_set.boundaries.values('id', 'name', *fields):
        if callable(boundary_key):
            boundary_property = boundary_key(boundary)
        else:
            boundary_property = boundary[boundary_key]
        ocd_id = geoid_mapping.get(prefix + boundary_property)
{'fraction_non_alphanumeric': 0.09037328094302555, 'fraction_numerical': 0.003143418467583497, 'mean_word_length': 3.579136690647482, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1071359', 'n_tokens_mistral': 758, 'n_tokens_neox': 719, 'n_words': 173}