<commit_before># This command is for updating the aspirants in Mzalendo from the IEBC
# API - it relies on already-imported aspirants having the external_id
# field of their Position set to the IEBC candidate code.

from collections import defaultdict
import csv
import datetime
import errno
import hmac
import hashlib
import itertools
import json
import os
import re
import requests
import sys

from django.core.management.base import NoArgsCommand, CommandError
from django.template.defaultfilters import slugify

from django_date_extensions.fields import ApproximateDate

from settings import IEBC_API_ID, IEBC_API_SECRET
from optparse import make_option

from core.models import Place, PlaceKind, Person, ParliamentarySession, Position, PositionTitle, Organisation, OrganisationKind

from iebc_api import *

before_import_date = datetime.date(2013, 2, 7)

data_directory = os.path.join(sys.path[0], 'kenya', '2013-election-data')

# Calling these 'corrections' may not be quite right.  There are
# naming discrepancies between the documents published by the IEBC and
# the IEBC API in spellings of ward names in particular.  This maps
# the IEBC API version to what we have in the API (which for wards was
# derived from "Final Constituencies and Wards Description.pdf").

place_name_corrections = {}

with open(os.path.join(data_directory, 'wards-names-matched.csv')) as fp:
    reader = csv.reader(fp)
    for api_name, db_name in reader:
        if api_name and db_name:
            place_name_corrections[api_name] = db_name

party_name_corrections = {}

with open(os.path.join(data_directory, 'party-names-matched.csv')) as fp:
    reader = csv.reader(fp)
    for api_name, db_name in reader:
        if api_name and db_name:
            party_name_corrections[api_name] = db_name

def get_matching_party(party_name, **options):
    party_name_to_use = party_name_corrections.get(party_name, party_name)
    # print "looking for '%s'" % (party_name_to_use,)
    matching_parties = Organisation.objects.filter(kind__slug='party',
                                                   name__iexact=party_name_to_use)
    if not matching_parties:
        matching_parties = Organisation.objects.filter(kind__slug='party',
                                                       name__istartswith=party_name_to_use)
    if len(matching_parties) == 0:
        party_name_for_creation = party_name_to_use.title()
        new_party = Organisation(name=party_name_for_creation,
                                 slug=slugify(party_name_for_creation),
                                 started=ApproximateDate(datetime.date.today().year),
                                 ended=None,
                                 kind=OrganisationKind.objects.get(slug='party'))
        maybe_save(new_party, **options)
        return new_party
    elif len(matching_parties) == 1:
        return matching_parties[0]
    else:
        raise Exception, "Multiple parties matched %s" % (party_name_to_use,)

def get_matching_place(place_name, place_kind, parliamentary_session):
    place_name_to_use = place_name_corrections.get(place_name, place_name)
    # We've normalized ward names to have a single space on either
    # side of a / or a -, so change API ward names to match:
    if place_kind.slug in ('ward', 'county'):
        place_name_to_use = re.sub(r'(\w) *([/-]) *(\w)', '\\1 \\2 \\3', place_name_to_use)
    # As with other place matching scripts here, look for the
    # slugified version to avoid problems with different separators:
    place_slug = slugify(place_name_to_use)
    if place_kind.slug == 'ward':
        place_slug = 'ward-' + place_slug
    elif place_kind.slug == 'county':
        place_slug += '-county'
    elif place_kind.slug == 'constituency':
        place_slug += '-2013'
    matching_places = Place.objects.filter(slug=place_slug,
                                           kind=place_kind,
                                           parliamentary_session=parliamentary_session)
    if not matching_places:
        raise Exception, "Found no place that matched: '%s' <%s> <%s>" % (place_slug, place_kind, parliamentary_session)
    elif len(matching_places) > 1:
        raise Exception, "Multiple places found that matched: '%s' <%s> <%s> - they were: %s" % (place_slug, place_kind, parliamentary_session, ",".join(str(p for p in matching_places)))
    else:
        return matching_places[0]

def make_new_person(candidate, **options):
    legal_name = normalize_name((candidate.get('other_name', None) or '').title())
    if legal_name:
        legal_name += ' '
    legal_name += normalize_name((candidate.get('surname', None) or '').title())
    slug_to_use = slugify(legal_name)
    suffix = 2
    while True:
        try:
            Person.objects.get(slug=slug_to_use)
        except Person.DoesNotExist:
            # Then this slug_to_use is fine, so just break:
            break
        slug_to_use = re.sub('-?\d*$', '', slug_to_use) + '-' + str(suffix)
        suffix += 1
    new_person = Person(legal_name=legal_name, slug=slug_to_use)
    maybe_save(new_person, **options)
    return new_person

def update_parties(person, api_party, **options):
    current_party_positions = person.position_set.all().currently_active().filter(title__slug='member').filter(organisation__kind__slug='party')
    if 'name' in api_party:
        # Then we should be checking that a valid party membership
        # exists, or create a new one otherwise:
        api_party_name = api_party['name']
        mz_party = get_matching_party(api_party_name, **options)
        need_to_create_party_position = True
        for party_position in (p for p in current_party_positions if p.organisation == mz_party):
            # If there's a current position in this party, that's fine
            # - just make sure that the end_date is 'future':
            party_position.end_date = ApproximateDate(future=True)
            maybe_save(party_position, **options)
            need_to_create_party_position = False
        for party_position in (p for p in current_party_positions if p.organisation != mz_party):
            # These shouldn't be current any more - end them when we
            # got the new data:
            party_position.end_date = yesterday_approximate_date
            maybe_save(party_position, **options)
        if need_to_create_party_position:
            new_position = Position(title=PositionTitle.objects.get(name='Member'),
                                    organisation=mz_party,
                                    category='political',
                                    person=person,
                                    start_date=today_approximate_date,
                                    end_date=ApproximateDate(future=True))
            maybe_save(new_position, **options)
    else:
        # If there's no party specified, end all current party positions:
        for party_position in current_party_positions:
            party_position.end_date = yesterday_approximate_date
            maybe_save(party_position, **options)

def update_candidates_for_place(place_name,
                                place_kind,
                                parliamentary_session,
                                title,
                                race_type,
                                candidates,
                                same_person_checker,
                                **options):
    place = get_matching_place(place_name, place_kind, parliamentary_session)

    # Find the current aspirants:
    current_aspirants = Position.objects.filter(place=place, title=title).currently_active()
    
    print "%s %s %s %s %s" % (place_name,
                              place_kind,
                              parliamentary_session,
                              title,
                              race_type)

    def full_name(c):
        return normalize_name(c['other_name']) + " " + normalize_name(c['surname'])

    code_to_existing_aspirant = dict((a.external_id, a) for a in current_aspirants if a.external_id)
    code_to_current_candidates = dict((c['code'], c) for c in candidates)

    existing_aspirant_codes = set(code_to_existing_aspirant.keys())
    current_candidate_codes = set(code_to_current_candidates.keys())

    # print "existing_aspirant_codes", existing_aspirant_codes
    # print "current_candidate_codes", current_candidate_codes

    existing_aspirants_to_remove = existing_aspirant_codes - current_candidate_codes
    new_candidates_to_add = current_candidate_codes - existing_aspirant_codes

    for code in new_candidates_to_add:
        candidate = code_to_current_candidates[code]
        print "  would add:", full_name(candidate)
        first_names = normalize_name(candidate['other_name'] or '')
        surname = normalize_name(candidate['surname'] or '')
        person = get_person_from_names(first_names, surname)
        print "  returned person is:", person
        if person:
            same_person = same_person_checker.check_same_and_update(candidate,
                                                                    place,
                                                                    race_type,
                                                                    person)
            if same_person is None:
                return False # To indicate this update didn't complete...
        if not person:
            person = make_new_person(candidate, **options)

        assert(person)

        update_parties(person, candidate['party'], **options)

        aspirant_position_properties = {
            'organisation': Organisation.objects.get(name='REPUBLIC OF KENYA'),
            'place': place,
            'person': person,
            'title': title,
            'category': 'political'}

        # First see if that aspirant position already exists:
        existing_matching_aspirant_positions = Position.objects.filter(**aspirant_position_properties).currently_active()

        if existing_matching_aspirant_positions:
            # If it does, make sure that the IEBC code is set in<commit_msg>Set any missing IEBC API codes when updating from the API<commit_after># This command is for updating the aspirants in Mzalendo from the IEBC
# API - it relies on already-imported aspirants having the external_id
# field of their Position set to the IEBC candidate code.

from collections import defaultdict
import csv
import datetime
import errno
import hmac
import hashlib
import itertools
import json
import os
import re
import requests
import sys

from django.core.management.base import NoArgsCommand, CommandError
from django.template.defaultfilters import slugify

from django_date_extensions.fields import ApproximateDate

from settings import IEBC_API_ID, IEBC_API_SECRET
from optparse import make_option

from core.models import Place, PlaceKind, Person, ParliamentarySession, Position, PositionTitle, Organisation, OrganisationKind

from iebc_api import *

before_import_date = datetime.date(2013, 2, 7)

data_directory = os.path.join(sys.path[0], 'kenya', '2013-election-data')

# Calling these 'corrections' may not be quite right.  There are
# naming discrepancies between the documents published by the IEBC and
# the IEBC API in spellings of ward names in particular.  This maps
# the IEBC API version to what we have in the API (which for wards was
# derived from "Final Constituencies and Wards Description.pdf").

place_name_corrections = {}

with open(os.path.join(data_directory, 'wards-names-matched.csv')) as fp:
    reader = csv.reader(fp)
    for api_name, db_name in reader:
        if api_name and db_name:
            place_name_corrections[api_name] = db_name

party_name_corrections = {}

with open(os.path.join(data_directory, 'party-names-matched.csv')) as fp:
    reader = csv.reader(fp)
    for api_name, db_name in reader:
        if api_name and db_name:
            party_name_corrections[api_name] = db_name

def get_matching_party(party_name, **options):
    party_name_to_use = party_name_corrections.get(party_name, party_name)
    # print "looking for '%s'" % (party_name_to_use,)
    matching_parties = Organisation.objects.filter(kind__slug='party',
                                                   name__iexact=party_name_to_use)
    if not matching_parties:
        matching_parties = Organisation.objects.filter(kind__slug='party',
                                                       name__istartswith=party_name_to_use)
    if len(matching_parties) == 0:
        party_name_for_creation = party_name_to_use.title()
        new_party = Organisation(name=party_name_for_creation,
                                 slug=slugify(party_name_for_creation),
                                 started=ApproximateDate(datetime.date.today().year),
                                 ended=None,
                                 kind=OrganisationKind.objects.get(slug='party'))
        maybe_save(new_party, **options)
        return new_party
    elif len(matching_parties) == 1:
        return matching_parties[0]
    else:
        raise Exception, "Multiple parties matched %s" % (party_name_to_use,)

def get_matching_place(place_name, place_kind, parliamentary_session):
    place_name_to_use = place_name_corrections.get(place_name, place_name)
    # We've normalized ward names to have a single space on either
    # side of a / or a -, so change API ward names to match:
    if place_kind.slug in ('ward', 'county'):
        place_name_to_use = re.sub(r'(\w) *([/-]) *(\w)', '\\1 \\2 \\3', place_name_to_use)
    # As with other place matching scripts here, look for the
    # slugified version to avoid problems with different separators:
    place_slug = slugify(place_name_to_use)
    if place_kind.slug == 'ward':
        place_slug = 'ward-' + place_slug
    elif place_kind.slug == 'county':
        place_slug += '-county'
    elif place_kind.slug == 'constituency':
        place_slug += '-2013'
    matching_places = Place.objects.filter(slug=place_slug,
                                           kind=place_kind,
                                           parliamentary_session=parliamentary_session)
    if not matching_places:
        raise Exception, "Found no place that matched: '%s' <%s> <%s>" % (place_slug, place_kind, parliamentary_session)
    elif len(matching_places) > 1:
        raise Exception, "Multiple places found that matched: '%s' <%s> <%s> - they were: %s" % (place_slug, place_kind, parliamentary_session, ",".join(str(p for p in matching_places)))
    else:
        return matching_places[0]

def make_new_person(candidate, **options):
    legal_name = normalize_name((candidate.get('other_name', None) or '').title())
    if legal_name:
        legal_name += ' '
    legal_name += normalize_name((candidate.get('surname', None) or '').title())
    slug_to_use = slugify(legal_name)
    suffix = 2
    while True:
        try:
            Person.objects.get(slug=slug_to_use)
        except Person.DoesNotExist:
            # Then this slug_to_use is fine, so just break:
            break
        slug_to_use = re.sub('-?\d*$', '', slug_to_use) + '-' + str(suffix)
        suffix += 1
    new_person = Person(legal_name=legal_name, slug=slug_to_use)
    maybe_save(new_person, **options)
    return new_person

def update_parties(person, api_party, **options):
    current_party_positions = person.position_set.all().currently_active().filter(title__slug='member').filter(organisation__kind__slug='party')
    if 'name' in api_party:
        # Then we should be checking that a valid party membership
        # exists, or create a new one otherwise:
        api_party_name = api_party['name']
        mz_party = get_matching_party(api_party_name, **options)
        need_to_create_party_position = True
        for party_position in (p for p in current_party_positions if p.organisation == mz_party):
            # If there's a current position in this party, that's fine
            # - just make sure that the end_date is 'future':
            party_position.end_date = ApproximateDate(future=True)
            maybe_save(party_position, **options)
            need_to_create_party_position = False
        for party_position in (p for p in current_party_positions if p.organisation != mz_party):
            # These shouldn't be current any more - end them when we
            # got the new data:
            party_position.end_date = yesterday_approximate_date
            maybe_save(party_position, **options)
        if need_to_create_party_position:
            new_position = Position(title=PositionTitle.objects.get(name='Member'),
                                    organisation=mz_party,
                                    category='political',
                                    person=person,
                                    start_date=today_approximate_date,
                                    end_date=ApproximateDate(future=True))
            maybe_save(new_position, **options)
    else:
        # If there's no party specified, end all current party positions:
        for party_position in current_party_positions:
            party_position.end_date = yesterday_approximate_date
            maybe_save(party_position, **options)

def update_candidates_for_place(place_name,
                                place_kind,
                                parliamentary_session,
                                title,
                                race_type,
                                candidates,
                                same_person_checker,
                                **options):
    place = get_matching_place(place_name, place_kind, parliamentary_session)

    # Find the current aspirants:
    current_aspirants = Position.objects.filter(place=place, title=title).currently_active()
    
    print "%s %s %s %s %s" % (place_name,
                              place_kind,
                              parliamentary_session,
                              title,
                              race_type)

    def full_name(c):
        return normalize_name(c['other_name']) + " " + normalize_name(c['surname'])

    code_to_existing_aspirant = dict((a.external_id, a) for a in current_aspirants if a.external_id)
    code_to_current_candidates = dict((c['code'], c) for c in candidates)

    existing_aspirant_codes = set(code_to_existing_aspirant.keys())
    current_candidate_codes = set(code_to_current_candidates.keys())

    # print "existing_aspirant_codes", existing_aspirant_codes
    # print "current_candidate_codes", current_candidate_codes

    existing_aspirants_to_remove = existing_aspirant_codes - current_candidate_codes
    new_candidates_to_add = current_candidate_codes - existing_aspirant_codes

    for code in new_candidates_to_add:
        candidate = code_to_current_candidates[code]
        print "  would add:", full_name(candidate)
        first_names = normalize_name(candidate['other_name'] or '')
        surname = normalize_name(candidate['surname'] or '')
        person = get_person_from_names(first_names, surname)
        print "  returned person is:", person
        # If that person was an existing current aspirant, they'll
        # just need to have the IEBC candidate code set:
        iebc_code_just_needed_setting = False
        for matching_existing_aspirant_position in current_aspirants.filter(person=person):
            if matching_existing_aspirant_position.external_id:
                if matching_existing_aspirant_position.external_id != code:
                    print "     original: '%s'" % (matching_existing_aspirant_position.external_id,)
                    print "    candidate: '%s'" % (code,)
                    format_tuple = (matching_existing_aspirant_position.person,
                                    matching_existing_aspirant_position.external_id,
                                    person,
                                    code)
                    message = "There was an existing candidate (%s - %s) with a name match for (%s - %s) but different codes" % format_tuple
                    # raise Exception, message
                    print message
            else:
                print "  setting a missing IEBC code on", matching_existing_aspirant_position
                matching_existing_aspirant_position.external_id = code
                maybe_save(matching_existing_aspirant_position, **options)
                update_parties(person, candidate['party'], **options)
                iebc_code_just_needed_setting = True
        if iebc_code_just_needed_setting:
            continue
        # If we have a person match, but they weren't a current
        # aspirant in this race, then be careful that we're not
        # mismatching using the SamePersonChecker:
        if person:
            same_person = same_person_checker.check_same_and_update(candidate,
                                                                    place,
                                                                    race_type,
                                                                    person)
            if same_person is None:
                return False # To indicate this update didn't complete...
        if not person:
            person = make_new_person(candidate, **options)

        assert(person)

        update_parties(person, candidate['party'], **options)

        aspirant_position_properties = {
            'organisation': Organisation.objects.get(name='REPUBLIC OF KENYA'),
            'place': place,
            'person': person,
            'title': title,
            'category': 'political'}

        # First see if that aspirant position already exists:
        existing_matching_aspirant_positions = Position.objects.filter(**aspirant_position_properties).currently_active()

        if existing_matching_aspirant_positions:
            # If it does, make sure that the IEBC code is set in
{'fraction_non_alphanumeric': 0.0795428675499704, 'fraction_numerical': 0.002276556026043801, 'mean_word_length': 2.4042157470551766, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 49, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5937520', 'n_tokens_mistral': 6165, 'n_tokens_neox': 5919, 'n_words': 1523}