<commit_before>import logging
import os.path
import re
from collections import OrderedDict

import requests
from django.conf import settings
from django.core.management.base import BaseCommand
from opencivicdata.divisions import Division

log = logging.getLogger(__name__)

numeric_re = re.compile('^[\d.]+$')


class Command(BaseCommand):
    args = '<module>'
    help = 'Generate mappings between boundary sets and divisions'

    def handle(self, *args, **options):
        mappings = {}

        divisions = list(Division.all('ca'))  # cache all divisions
        for obj in requests.get('https://represent.opennorth.ca/boundary-sets/?limit=0').json()['objects']:
            slug = obj['url'].split('/')[2]
            if obj['url'] in ('/boundary-sets/census-divisions/', '/boundary-sets/census-subdivisions/'):
                continue
            if obj['url'] in ('/boundary-sets/federal-electoral-districts/', '/boundary-sets/federal-electoral-districts-next-election/'):
                prefix = 'ocd-division/country:ca/ed:'
                boundary_key = 'external_id'
            else:
                url = 'http://represent.opennorth.ca{}'.format(obj['url'])
                boundary_set = requests.get(url).json()

                if boundary_set['extra'].get('ocd_division'):<commit_msg>Use HTTPS for Represent<commit_after>import logging
import os.path
import re
from collections import OrderedDict

import requests
from django.conf import settings
from django.core.management.base import BaseCommand
from opencivicdata.divisions import Division

log = logging.getLogger(__name__)

numeric_re = re.compile('^[\d.]+$')


class Command(BaseCommand):
    args = '<module>'
    help = 'Generate mappings between boundary sets and divisions'

    def handle(self, *args, **options):
        mappings = {}

        divisions = list(Division.all('ca'))  # cache all divisions
        for obj in requests.get('https://represent.opennorth.ca/boundary-sets/?limit=0').json()['objects']:
            slug = obj['url'].split('/')[2]
            if obj['url'] in ('/boundary-sets/census-divisions/', '/boundary-sets/census-subdivisions/'):
                continue
            if obj['url'] in ('/boundary-sets/federal-electoral-districts/', '/boundary-sets/federal-electoral-districts-next-election/'):
                prefix = 'ocd-division/country:ca/ed:'
                boundary_key = 'external_id'
            else:
                url = 'https://represent.opennorth.ca{}'.format(obj['url'])
                boundary_set = requests.get(url).json()

                if boundary_set['extra'].get('ocd_division'):
{'fraction_non_alphanumeric': 0.14258482653450247, 'fraction_numerical': 0.0015249714067861228, 'mean_word_length': 2.8816568047337277, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 3, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6257222', 'n_tokens_mistral': 783, 'n_tokens_neox': 728, 'n_words': 158}