<commit_before>from collections import defaultdict
import urllib
from mozillians.geo.models import Country, Region, City, Geocoding
import requests

from product_details import product_details


from django.core.management.base import BaseCommand
from mozillians.users.models import UserProfile
from mozillians.settings import MAPBOX_MAP_ID

COUNTRIES = product_details.get_regions('en-US')


def country_code_to_mapbox_name(code):
    try:
        country = Country.objects.get(code=code)
    except Country.DoesNotExist:
        country_name = COUNTRIES[code.lower()]
        # Change country name to match mapbox
        # This is not a political statement, it's merely
        # using the same name as Mapbox has already chosen.
        if country_name == 'Guinea-Bissau':
            country_name = 'Guinea Bissau'
        elif country_name == 'Hong Kong':
            country_name = 'Hong Kong S.A.R.'
        elif country_name == 'Macedonia, F.Y.R. of':
            country_name = 'Macedonia'
        elif country_name == 'Russian Federation':
            country_name = 'Russia'
        elif country_name == 'Occupied Palestinian Territory':
            country_name = 'Palestine'
        return country_name
    else:
        return country.name


def massage_results(country_code, results):
    """
    Given a list of results returned by mapbox.

    Return a dictionary that might have entries 'city', 'region',
    and/or 'country'. Only include an entry if the results contained
    exactly one instance of that type.

    E.g. if we got back five results, but the only country in any
    of them was Albania, then the dictionary returned will have a 'country'
    entry with the data for Albania.

    The idea is that sometimes the results from Mapbox don't look unique,
    but enough of the results are unique to be useful to us.

    Each entry we return is an instance of the appropriate model.

    :param results:
    :return:
    """
    print(results)

    country_code = country_code.lower()

    try:
        country = Country.objects.get(code=country_code)
    except Country.DoesNotExist:
        country_name = country_code_to_mapbox_name(country_code)
        country = None
    else:
        country_name = country.name

    countries = {}
    regions = {}
    cities = {}

    sets = defaultdict(set)
    for result in results:
        # Convert list of dictionaries to a dictionary keyed by the type
        result = dict([(item['type'], item) for item in result])

        if 'country' not in result:
            print('SKIPPING result - no country')
            continue

        # Ignore any result that isn't in this user's country
        # FIXME: This breaks if our country names don't match Mapbox's, and they probably
        # don't all match.
        result_name = result['country']['name']
        if result_name != country_name:
            print('SKIPPING result - wrong country - %s != %s' % (result['country']['name'],
                                                                  country_name))
            continue

        if 'city' in result:
            name = result['city']['name']
            cities[name] = result['city']
            sets['city'].add(name)
        if 'province' in result:
            name = result['province']['name']
            regions[name] = result['province']
            sets['region'].add(name)
        if 'country' in result:
            name = result['country']['name']
            countries[name] = result['country']
            sets['country'].add(name)
    retval = {}

    city = region = None
    if not country and len(sets['country']) == 1:
        name = list(sets['country'])[0]
        item = countries[name]
        country, unused = Country.objects.get_or_create(
            mapbox_id=item['id'],
            defaults=dict(
                name=item['name'],
                code=country_code,
                mapbox_id=item['id'],
            )
        )
    if country:
        retval['country'] = country

    # Can't do anything with a region without a country
    if country and len(sets['region']) == 1:
        name = list(sets['region'])[0]
        item = regions[name]
        region, unused = Region.objects.get_or_create(
            mapbox_id=item['id'],
            defaults=dict(
                name=item['name'],
                country=country,
                ),
            )
        retval['region'] = region

    # Can't do anything with a city without a country
    # (Region is optional, though)
    if country and len(sets['city']) == 1:
        name = list(sets['city'])[0]
        item = cities[name]
        if 'lat' in item and 'lon' in item:
            print('Get or create city: %r' % item)
            city, created = City.objects.get_or_create(
                mapbox_id=item['id'],
                defaults=dict(
                    name=item['name'],
                    region=region,
                    country=country,
                    lat=item['lat'],
                    lng=item['lon'],
                )
            )
            retval['city'] = city
        else:
            print('NO LAT, LONG for city %s' % item['name'])

    return retval


class Command(BaseCommand):
    args = '(no args)'
    help = 'Geocode users'

    def handle(self, *args, **options):
        print('Here we go!')

        map_id = MAPBOX_MAP_ID

        if not map_id:
            raise Exception('MAPBOX_MAP_ID is not defined')

        # Reset mozillians from some country
        def reset_country(code):
            UserProfile.objects.filter(country=code, geo_coded=True).update(geo_coded=False)
            Geocoding.objects.filter(country=code).delete()

            print('COUNTRIES["%s"] = %r' % (code, COUNTRIES[code]))
            print(repr(country_code_to_mapbox_name(code)))

        # Apply known data from geocoding model
        for item in Geocoding.objects.all():
            item.apply()<commit_msg>[Fix Bug 1029326] Prevent duplicate region and city cases for geocode migrate.<commit_after>from collections import defaultdict
import urllib
from mozillians.geo.models import Country, Region, City, Geocoding
import requests

from product_details import product_details


from django.core.management.base import BaseCommand
from mozillians.users.models import UserProfile
from mozillians.settings import MAPBOX_MAP_ID

COUNTRIES = product_details.get_regions('en-US')


def country_code_to_mapbox_name(code):
    try:
        country = Country.objects.get(code=code)
    except Country.DoesNotExist:
        country_name = COUNTRIES[code.lower()]
        # Change country name to match mapbox
        # This is not a political statement, it's merely
        # using the same name as Mapbox has already chosen.
        if country_name == 'Guinea-Bissau':
            country_name = 'Guinea Bissau'
        elif country_name == 'Hong Kong':
            country_name = 'Hong Kong S.A.R.'
        elif country_name == 'Macedonia, F.Y.R. of':
            country_name = 'Macedonia'
        elif country_name == 'Russian Federation':
            country_name = 'Russia'
        elif country_name == 'Occupied Palestinian Territory':
            country_name = 'Palestine'
        return country_name
    else:
        return country.name


def massage_results(country_code, results):
    """
    Given a list of results returned by mapbox.

    Return a dictionary that might have entries 'city', 'region',
    and/or 'country'. Only include an entry if the results contained
    exactly one instance of that type.

    E.g. if we got back five results, but the only country in any
    of them was Albania, then the dictionary returned will have a 'country'
    entry with the data for Albania.

    The idea is that sometimes the results from Mapbox don't look unique,
    but enough of the results are unique to be useful to us.

    Each entry we return is an instance of the appropriate model.

    :param results:
    :return:
    """
    print(results)

    country_code = country_code.lower()

    try:
        country = Country.objects.get(code=country_code)
    except Country.DoesNotExist:
        country_name = country_code_to_mapbox_name(country_code)
        country = None
    else:
        country_name = country.name

    countries = {}
    regions = {}
    cities = {}

    sets = defaultdict(set)
    for result in results:
        # Convert list of dictionaries to a dictionary keyed by the type
        result = dict([(item['type'], item) for item in result])

        if 'country' not in result:
            print('SKIPPING result - no country')
            continue

        # Ignore any result that isn't in this user's country
        # FIXME: This breaks if our country names don't match Mapbox's, and they probably
        # don't all match.
        result_name = result['country']['name']
        if result_name != country_name:
            print('SKIPPING result - wrong country - %s != %s' % (result['country']['name'],
                                                                  country_name))
            continue

        if 'city' in result:
            name = result['city']['name']
            cities[name] = result['city']
            sets['city'].add(name)
        if 'province' in result:
            name = result['province']['name']
            regions[name] = result['province']
            sets['region'].add(name)
        if 'country' in result:
            name = result['country']['name']
            countries[name] = result['country']
            sets['country'].add(name)
    retval = {}

    city = region = None
    if not country and len(sets['country']) == 1:
        name = list(sets['country'])[0]
        item = countries[name]
        country, unused = Country.objects.get_or_create(
            mapbox_id=item['id'],
            defaults=dict(
                name=item['name'],
                code=country_code,
                mapbox_id=item['id'],
            )
        )
    if country:
        retval['country'] = country

    # Can't do anything with a region without a country
    if country and len(sets['region']) == 1:
        name = list(sets['region'])[0]
        item = regions[name]
        try:
            region = Region.objects.get(mapbox_id=item['id'])
        except Region.DoesNotExist:
            region, unused = Region.objects.get_or_create(
                name=item['name'], country=country,
                defaults=dict(mapbox_id=item['id']),
            )
            retval['region'] = region

    # Can't do anything with a city without a country
    # (Region is optional, though)
    if country and len(sets['city']) == 1:
        name = list(sets['city'])[0]
        item = cities[name]
        if 'lat' in item and 'lon' in item:
            print('Get or create city: %r' % item)
            try:
                city = City.objects.get(mapbox_id=item['id'])
            except City.DoesNotExist:
                city, created = City.objects.get_or_create(
                    name=item['name'], region=region, country=country,
                    defaults=dict(
                        mapbox_id=item['id'],
                        lat=item['lat'],
                        lng=item['lon'],
                    )
                )
                retval['city'] = city
        else:
            print('NO LAT, LONG for city %s' % item['name'])

    return retval


class Command(BaseCommand):
    args = '(no args)'
    help = 'Geocode users'

    def handle(self, *args, **options):
        print('Here we go!')

        map_id = MAPBOX_MAP_ID

        if not map_id:
            raise Exception('MAPBOX_MAP_ID is not defined')

        # Reset mozillians from some country
        def reset_country(code):
            UserProfile.objects.filter(country=code, geo_coded=True).update(geo_coded=False)
            Geocoding.objects.filter(country=code).delete()

            print('COUNTRIES["%s"] = %r' % (code, COUNTRIES[code]))
            print(repr(country_code_to_mapbox_name(code)))

        # Apply known data from geocoding model
        for item in Geocoding.objects.all():
            item.apply()
{'fraction_non_alphanumeric': 0.10138248847926268, 'fraction_numerical': 0.001563528637261356, 'mean_word_length': 2.0994644223412395, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5386598', 'n_tokens_mistral': 3392, 'n_tokens_neox': 3260, 'n_words': 997}