<commit_before># Take the scraped PPC data and try to import it into PopIt.
#
# It looks for someone with the same name by searching PopIt, and, if
# matches are found, asks the user whether they're the same person.
# This decision is recorded in a JSON file so you should only need to
# make that choice once.
#
# TODO: add a --noinput option, which won't ask for input, but instead
# output text and return an error; this will be useful for running
# this job from cron - if you get an email from cron, you'll need to
# run the script manually to make those decisions.

import os
from os.path import join
import re
import json

from django.core.management.base import BaseCommand
from django.conf import settings

import requests

from candidates.update import PersonParseMixin, PersonUpdateMixin
from candidates.static_data import MapItData
from candidates.views import CandidacyMixin

party_slug_to_popit_party = {
    'labour': {
        'name': 'Labour Party',
        'id': 'party:53',
    },
    'conservatives': {
        'name': 'Conservative Party',
        'id': 'party:52',
    },
    'libdem': {
        'name': 'Liberal Democrats',
        'id': 'party:90',
    }
}

constituency_corrections = {
    'Berwick': 'Berwick-upon-Tweed',
    'Chester': 'City of Chester',
    'Dover and Deal': 'Dover',
    'Hull North': 'Kingston upon Hull North',
    'Hull West and Hessle': 'Kingston upon Hull West and Hessle',
    'Isle of Wright': 'Isle of Wight',
    'Richmond': 'Richmond (Yorks)',
    'Stratford-upon-Avon': 'Stratford-on-Avon',
    'West Cornwall and Scilly': 'St Ives',
}

def cons_key(s):
    return re.sub(r',', '', s).lower()

constituency_lookup = {
    cons_key(k): v for k, v in
    MapItData.constituencies_2010_name_map.items()
}

def get_constituency_from_name(constituency_name):
    name = re.sub(r'\xA0', ' ', constituency_name)
    name = re.sub(r' & ', ' and ', name)
    name = constituency_corrections.get(name, name)
    name = re.sub(r'(?i)candidate for ', '', name)
    mapit_data = constituency_lookup[cons_key(name)]
    post_id = str(mapit_data['id'])
    return {
        'post_id': post_id,
        'name': mapit_data['name'],
        'mapit_url': 'http://mapit.mysociety.org/area/{0}'.format(post_id),
    }

ppc_data_directory = join(
    settings.BASE_DIR,
    'data',
    'UK-Political-Parties',
    'ppcs',
    'json'
)

human_decisions_filename = join(
    settings.BASE_DIR,
    'data',
    'human-ppc-decisions.json'
)

def get_human_decisions():
    with open(human_decisions_filename) as f:
        return json.load(f)

def write_human_decisions(human_decisions):
    output = json.dumps(human_decisions, indent=4, sort_keys=True)
    output = re.sub(r'(?ms)\s*$', '', output)
    with open(human_decisions_filename, 'w') as f:
        f.write(output)

def record_human_decision(ppc_dict, popit_person, same):
    human_decisions = get_human_decisions()
    human_decisions.append({
        'same_person': same,
        'ppc': ppc_dict,
        'popit_person': popit_person,
    })
    write_human_decisions(human_decisions)

def get_human_decision(ppc_data):
    fields_to_match = ['party_slug', 'constituency', 'name']
    matching_decisions = []
    for decision in get_human_decisions():
        if all(decision['ppc'][k] == ppc_data[k] for k in fields_to_match):
            matching_decisions.append(decision)
    return matching_decisions

def merge_person_data(old_data, new_data):
    data = old_data.copy()
    standing_in = data.get('standing_in', {})
    party_memberships = data.get('party_memberships', {})
    standing_in.update(new_data.get('standing_in', {}))
    party_memberships.update(new_data.get('party_memberships', {}))
    data.update(new_data)
    data['standing_in'] = standing_in
    data['party_memberships'] = party_memberships
    return data

def get_ppc_url_from_popit_result(popit_result):
    links = popit_result.get('links', [])
    for link in links:
        if link.get('note') == 'party PPC page':
            return link.get['url']
    return None

class Command(CandidacyMixin, PersonParseMixin, PersonUpdateMixin, BaseCommand):
    help = "Import scraped PPC data"

    # Currently unused:
    #
    #     "region",
    #     "full_url",
    #     "path",
    #     "phone",
    #     "slug",
    #     "address",
    #     "current_mp",
    #     "blog_url",
    #     "campaign_url",
    #     "biography_url",

    def get_person_data_from_ppc(self, ppc_data):
        return {
            'name': ppc_data.get('name', ''),
            'email': ppc_data.get('email', ''),
            'birth_date': None,
            'twitter_username': ppc_data.get('twitter_username', ''),
            'facebook_personal_url': ppc_data.get('facebook_url', ''),
            'homepage_url': ppc_data.get('homepage_url', ''),
            'party_ppc_page_url': ppc_data.get('full_url', ''),
            'standing_in': {
                '2015': ppc_data['constituency_object'],
            },
            'party_memberships': {
                '2015': ppc_data['party_object'],
            }
        }

    def update_popit_person(self, popit_person_id, ppc_data):
        # Get the existing data first:
        person_data = self.get_person(popit_person_id)
        previous_versions = person_data.pop('versions')
        new_person_data = self.get_person_data_from_ppc(ppc_data)
        # Remove any empty keys, we don't want to overwrite exiting
        # data with nothing:
        keys = new_person_data.keys()
        for key in keys:
            if not new_person_data[key]:
                del new_person_data[key]
        merged_person_data = merge_person_data(person_data, new_person_data)
        change_metadata = self.get_change_metadata(
            None,
            'Updated candidate from official PPC data ({0})'.format(ppc_data['party_slug']),
        )
        return self.update_person(
            merged_person_data,
            change_metadata,
            previous_versions,
        )

    def add_popit_person(self, ppc_data):
        change_metadata = self.get_change_metadata(
            None,
            'Created new candidate from official PPC data ({0})'.format(ppc_data['party_slug']),
        )
        person_data = self.get_person_data_from_ppc(ppc_data)
        return self.create_person(person_data, change_metadata)

    def already_added_for_2015(self, ppc_data, popit_result):
        popit_constituency_name_2015 = popit_result['standing_in'].get('2015', {}).get('name')
        popit_party_id_2015 = popit_result['party_memberships'].get('2015', {}).get('id')
        return (ppc_data['name'] == popit_result['name'] and
                ppc_data['party_object']['id'] == popit_party_id_2015 and
                ppc_data['constituency_object']['name'] == popit_constituency_name_2015)<commit_msg>Make sure the PPC importer doesn't overwrite existing data<commit_after># Take the scraped PPC data and try to import it into PopIt.
#
# It looks for someone with the same name by searching PopIt, and, if
# matches are found, asks the user whether they're the same person.
# This decision is recorded in a JSON file so you should only need to
# make that choice once.
#
# TODO: add a --noinput option, which won't ask for input, but instead
# output text and return an error; this will be useful for running
# this job from cron - if you get an email from cron, you'll need to
# run the script manually to make those decisions.

import os
from os.path import join
import re
import json

from django.core.management.base import BaseCommand
from django.conf import settings

import requests

from candidates.update import PersonParseMixin, PersonUpdateMixin
from candidates.static_data import MapItData
from candidates.views import CandidacyMixin

party_slug_to_popit_party = {
    'labour': {
        'name': 'Labour Party',
        'id': 'party:53',
    },
    'conservatives': {
        'name': 'Conservative Party',
        'id': 'party:52',
    },
    'libdem': {
        'name': 'Liberal Democrats',
        'id': 'party:90',
    }
}

constituency_corrections = {
    'Berwick': 'Berwick-upon-Tweed',
    'Chester': 'City of Chester',
    'Dover and Deal': 'Dover',
    'Hull North': 'Kingston upon Hull North',
    'Hull West and Hessle': 'Kingston upon Hull West and Hessle',
    'Isle of Wright': 'Isle of Wight',
    'Richmond': 'Richmond (Yorks)',
    'Stratford-upon-Avon': 'Stratford-on-Avon',
    'West Cornwall and Scilly': 'St Ives',
}

def cons_key(s):
    return re.sub(r',', '', s).lower()

constituency_lookup = {
    cons_key(k): v for k, v in
    MapItData.constituencies_2010_name_map.items()
}

def get_constituency_from_name(constituency_name):
    name = re.sub(r'\xA0', ' ', constituency_name)
    name = re.sub(r' & ', ' and ', name)
    name = constituency_corrections.get(name, name)
    name = re.sub(r'(?i)candidate for ', '', name)
    mapit_data = constituency_lookup[cons_key(name)]
    post_id = str(mapit_data['id'])
    return {
        'post_id': post_id,
        'name': mapit_data['name'],
        'mapit_url': 'http://mapit.mysociety.org/area/{0}'.format(post_id),
    }

ppc_data_directory = join(
    settings.BASE_DIR,
    'data',
    'UK-Political-Parties',
    'ppcs',
    'json'
)

human_decisions_filename = join(
    settings.BASE_DIR,
    'data',
    'human-ppc-decisions.json'
)

def get_human_decisions():
    with open(human_decisions_filename) as f:
        return json.load(f)

def write_human_decisions(human_decisions):
    output = json.dumps(human_decisions, indent=4, sort_keys=True)
    output = re.sub(r'(?ms)\s*$', '', output)
    with open(human_decisions_filename, 'w') as f:
        f.write(output)

def record_human_decision(ppc_dict, popit_person, same):
    human_decisions = get_human_decisions()
    human_decisions.append({
        'same_person': same,
        'ppc': ppc_dict,
        'popit_person': popit_person,
    })
    write_human_decisions(human_decisions)

def get_human_decision(ppc_data):
    fields_to_match = ['party_slug', 'constituency', 'name']
    matching_decisions = []
    for decision in get_human_decisions():
        if all(decision['ppc'][k] == ppc_data[k] for k in fields_to_match):
            matching_decisions.append(decision)
    return matching_decisions

def merge_person_data(old_data, new_data):
    data = old_data.copy()
    standing_in = data.get('standing_in', {})
    party_memberships = data.get('party_memberships', {})
    standing_in.update(new_data.get('standing_in', {}))
    party_memberships.update(new_data.get('party_memberships', {}))
    data.update(new_data)
    data['standing_in'] = standing_in
    data['party_memberships'] = party_memberships
    return data

def get_ppc_url_from_popit_result(popit_result):
    links = popit_result.get('links', [])
    for link in links:
        if link.get('note') == 'party PPC page':
            return link.get['url']
    return None

class Command(CandidacyMixin, PersonParseMixin, PersonUpdateMixin, BaseCommand):
    help = "Import scraped PPC data"

    # Currently unused:
    #
    #     "region",
    #     "full_url",
    #     "path",
    #     "phone",
    #     "slug",
    #     "address",
    #     "current_mp",
    #     "blog_url",
    #     "campaign_url",
    #     "biography_url",

    def get_person_data_from_ppc(self, ppc_data):
        return {
            'name': ppc_data.get('name', ''),
            'email': ppc_data.get('email', ''),
            'birth_date': None,
            'twitter_username': ppc_data.get('twitter_username', ''),
            'facebook_personal_url': ppc_data.get('facebook_url', ''),
            'homepage_url': ppc_data.get('homepage_url', ''),
            'party_ppc_page_url': ppc_data.get('full_url', ''),
            'standing_in': {
                '2015': ppc_data['constituency_object'],
            },
            'party_memberships': {
                '2015': ppc_data['party_object'],
            }
        }

    def update_popit_person(self, popit_person_id, ppc_data):
        # Get the existing data first:
        person_data = self.get_person(popit_person_id)
        previous_versions = person_data.pop('versions')
        new_person_data = self.get_person_data_from_ppc(ppc_data)
        # Remove any empty keys, we don't want to overwrite exiting
        # data with nothing:
        keys = new_person_data.keys()
        warnings = []
        for key in keys:
            if not new_person_data[key]:
                del new_person_data[key]
            # Also make sure that we don't overwrite any existing
            # fields that are filled in with different values:
            if key not in ('standing_in', 'party_memberships'):
                new_person_data_value = new_person_data.get(key)
                person_data_value = person_data.get(key)
                if person_data_value and new_person_data_value and new_person_data_value != person_data_value:
                    warnings.append(u"not overwriting {0}".format(person_data_value))
                    warnings.append(u"with the new value {0}".format(new_person_data_value))
                    del new_person_data[key]
        if warnings:
            print "Warnings for person/{0} {1}".format(
                popit_person_id, person_data['name']
            )
            for warning in warnings:
                print "  ...", warning
        merged_person_data = merge_person_data(person_data, new_person_data)
        change_metadata = self.get_change_metadata(
            None,
            'Updated candidate from official PPC data ({0})'.format(ppc_data['party_slug']),
        )
        return self.update_person(
            merged_person_data,
            change_metadata,
            previous_versions,
        )

    def add_popit_person(self, ppc_data):
        change_metadata = self.get_change_metadata(
            None,
            'Created new candidate from official PPC data ({0})'.format(ppc_data['party_slug']),
        )
        person_data = self.get_person_data_from_ppc(ppc_data)
        return self.create_person(person_data, change_metadata)

    def already_added_for_2015(self, ppc_data, popit_result):
        popit_constituency_name_2015 = popit_result['standing_in'].get('2015', {}).get('name')
        popit_party_id_2015 = popit_result['party_memberships'].get('2015', {}).get('id')
        return (ppc_data['name'] == popit_result['name'] and
                ppc_data['party_object']['id'] == popit_party_id_2015 and
                ppc_data['constituency_object']['name'] == popit_constituency_name_2015)
{'fraction_non_alphanumeric': 0.11977773204363037, 'fraction_numerical': 0.007271729436783975, 'mean_word_length': 2.9711250340506674, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5008722', 'n_tokens_mistral': 4839, 'n_tokens_neox': 4696, 'n_words': 1033}