<commit_before># -*- coding: utf-8 -*-
from __future__ import unicode_literals

import json
from urllib2 import urlopen

from django.core.management.base import CommandError, BaseCommand

from catalog.addresses.models import Region, Country


COUNTRIES_API_URL = \
    'http://api.geonames.org/countryInfoJSON?username=dinoperovic'


class Command(BaseCommand):
    help = 'Create countries and regions automatically from api.geonames.org.'
    args = '<code code code code...>'
    data = None

    def __init__(self, *args, **kwargs):
        super(Command, self).__init__(*args, **kwargs)
        print 'Querying database at %s.' % COUNTRIES_API_URL
        api = urlopen(COUNTRIES_API_URL)
        data = json.loads(api.read()).get('geonames', None)
        if data is not None:
            self.data = dict((x['countryCode'], x) for x in data)
        else:
            raise CommandError('Error fetching %s' % COUNTRIES_API_URL)
        print 'Creating countries and regions.'

    def handle(self, *args, **options):
        if len(args):
            for code in list(args):
                self.create_country(code)
        else:
            for code in self.data.keys():
                self.create_country(code)

    def create_country(self, code):
        if code in self.data:
            country = self.data.get(code)

            try:
                region = Region.objects.get(
                    code=country['continent'])
            except Region.DoesNotExist:
                region = Region.objects.create(
                    code=country['continent'], name=country['continentName'])

        if not Country.objects.filter(code=code):
            Country.objects.create(
                code=code, name=country['countryName'], region=region)<commit_msg>Add logging to initcountries<commit_after># -*- coding: utf-8 -*-
from __future__ import unicode_literals

import json
from urllib2 import urlopen

from django.utils.encoding import smart_str
from django.core.management.base import CommandError, BaseCommand

from catalog.addresses.models import Region, Country


COUNTRIES_API_URL = \
    'http://api.geonames.org/countryInfoJSON?username=dinoperovic'


class Command(BaseCommand):
    help = 'Create countries and regions automatically from api.geonames.org.'
    args = '<code code code code...>'
    data = None

    def __init__(self, *args, **kwargs):
        super(Command, self).__init__(*args, **kwargs)

    def handle(self, *args, **options):
        print 'Querying database at {}.'.format(COUNTRIES_API_URL)

        api = urlopen(COUNTRIES_API_URL)
        data = json.loads(api.read()).get('geonames', None)
        if data is not None:
            self.data = dict((x['countryCode'], x) for x in data)
        else:
            raise CommandError('Error fetching %s.' % COUNTRIES_API_URL)

        codes = list(args) if len(args) else self.data.keys()
        codes = [x for x in codes if x in self.data]

        created_countries = []
        created_regions = []

        for code in codes:
            country = self.data.get(code)

            try:
                region = Region.objects.get(code=country['continent'])
            except Region.DoesNotExist:
                print 'Creating region: \'{}\' ({}).'.\
                    format(country['continent'],
                           self.get_unicode(country['continentName']))
                region = Region.objects.create(code=country['continent'],
                                               name=country['continentName'])
                created_regions.append(country['continent'])

            if not Country.objects.filter(code=code):
                print 'Creating country: \'{}\' ({}).'.\
                    format(code, self.get_unicode(country['countryName']))

                Country.objects.create(code=code, name=country['countryName'],
                                       region=region)
                created_countries.append(code)

        print 'Done! Created {} countries, and {} regions.'.\
            format(len(created_countries), len(created_regions))

    def get_unicode(self, string):
        try:
            return '{}'.format(smart_str(string))
        except UnicodeDecodeError:
            return 'Can\'t want to deal with unicode errors right now, sry!'
{'fraction_non_alphanumeric': 0.11483031148303115, 'fraction_numerical': 0.0009298000929800093, 'mean_word_length': 2.269756838905775, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 3, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7245240', 'n_tokens_mistral': 1212, 'n_tokens_neox': 1151, 'n_words': 290}