<commit_before>import os
import pickle
from collections import defaultdict

import requests
from django.conf import settings
from django.core.management import BaseCommand

from cl.lib.sunburnt import SolrInterface
from cl.people_db.import_judges.courtid_levels import courtid2statelevel
from cl.people_db.models import Person

leveldict = {
    'H': 'J',  # State Supreme Court
    'M': 'K',  # State Appellate Court
    'L': 'D',  # Lower Court
}

url_template = (
    'http://api.followthemoney.org/?'
    'f-core=1&'
    'c-exi=1&'
    'gro=c-t-id&'
    'mode=json&'
    'APIKey={key}&'
    's={state}&'
    'c-r-ot={level}&'
    'y={year}'
)


def make_dict_of_ftm_eids(use_pickle=True):
    """Build up a dictionary mapping jurisdiction IDs to candidates in those
    locations
    """
    pickle_location = '/tmp/eid_lists.pkl'
    if use_pickle:
        if os.path.isfile(pickle_location):
            with open(pickle_location, 'r') as f:
                print("Loading pickled candidate list. Read the command "
                      "documentation if this is not desired.")
                return pickle.load(f)
        else:
            print("Unable to find pickle file.")

    candidate_eid_lists = defaultdict(list)

    for courtid, (state, level) in courtid2statelevel.items():
        if level != 'H':
            # We only want high courts.
            continue
        for year in range(1989, 2017):
            url = url_template.format(
                key=settings.FTM_KEY,
                state=state,
                level=leveldict[level],
                year=year,
            )
            print("Getting url at: %s" % url)
            data = requests.get(url).json()

            if data['records'] == ['No Records']:
                print('  No records found in court %s and year %s.' % (
                    courtid,
                    year,
                ))
                continue
            print('  Found %s records in court %s and year %s' % (
                len(data['records']),
                courtid,
                year,
            ))

            for item in data['records']:
                # add an eid, name, year tuple to this court's list
                candidate_eid_lists[courtid].append({
                    'eid': item['Candidate_Entity']['id'],
                    'name': item['Candidate_Entity']['Candidate_Entity'],
                    'total': float(item['Total_$']['Total_$']),
                    'year': year,
                })

    if use_pickle:
        with open(pickle_location, 'w') as f:
            print("Creating pickle file at: %s" % pickle_location)
            pickle.dump(candidate_eid_lists, f)
    return candidate_eid_lists


def clear_old_values(do_it, debug):
    """Clear out the old values in the ftm fields. If debug or do_it is False,
    don't clear the values.
    """
    if not do_it or debug:
        return
    print("Clearing out all old values in FTM fields.")
    Person.objects.all().update(ftm_eid='', ftm_total_received=None)


def print_stats(match_stats, candidate_eid_lists):
    """Print the stats."""
    print("\n#########")
    print("# Stats #")
    print("#########")
    print("Finished matching judges:")
    for k, v in match_stats.items():
        print(" - %s had %s matches" % (v, k))
    ftm_judge_count = 0
    for v in candidate_eid_lists.values():
        ftm_judge_count += len(v)
    print("\nThere were %s judges in FTM that we matched "
          "against." % ftm_judge_count)


def update_judges_by_solr(candidate_id_map, debug):
    """Update judges by looking up each entity from FTM in Solr."""
    conn = SolrInterface(settings.SOLR_PEOPLE_URL, mode='r')
    match_stats = defaultdict(int)
    # These IDs are ones that cannot be updated due to being identified as
    # problematic in FTM's data.
    blacklisted_ids = defaultdict(set)
    for court_id, candidate_list in candidate_id_map.items():
        for candidate in candidate_list:
            # Look up the candidate in Solr.
            print("\nDoing: %s" % candidate['name'])
            name = (' AND '.join([word for word in candidate['name'].split() if
                                 len(word) > 1])).replace(',', '')
            results = conn.raw_query(**{
                'caller': 'ftm_update_judges_by_solr',
                'fq': [
                    'name:(%s)' % name,
                    'court_exact:%s' % court_id,
                    # This filters out Sr/Jr problems by insisting on recent
                    # positions. 1980 is arbitrary, based on testing.
                    'date_start:[1980-12-31T23:59:59Z TO *]',
                ],
                'q': "*:*",
            }).execute()

            if len(results) == 0:
                match_stats[len(results)] += 1
                print("  Found no matches.")

            elif len(results) == 1:
                match_stats[len(results)] += 1
                print("  Found one match: %s" % results[0]['name'])

                # Get the person from the DB and update them.
                pk = results[0]['id']
                if pk in blacklisted_ids:
                    continue
                p = Person.objects.get(pk=pk)
                if p.ftm_eid and p.ftm_eid != candidate['eid']:
                    print("  Found values in ftm database fields. This "
                          "indicates a duplicate in FTM.")

                    blacklisted_ids[p.pk].add(candidate['eid'])
                    blacklisted_ids[p.pk].add(p.ftm_eid)
                    p.ftm_eid = ""
                    p.ftm_total_received = None
                    if not debug:
                        p.save()
                else:
                    # No major problems. Proceed.
                    p.ftm_eid = candidate['eid']
                    p.ftm_total_received = candidate['total']
                    if not debug:
                        p.save()

            elif len(results) > 1:
                match_stats[len(results)] += 1
                print("  Found more than one match: %s" % results)

    print_stats(match_stats, candidate_id_map)
    print("Blacklisted IDs: %s" % blacklisted_ids)


class Command(BaseCommand):<commit_msg>Better support for multiple positions across jurisdictions.<commit_after>import os
import pickle
from collections import defaultdict

import requests
from django.conf import settings
from django.core.management import BaseCommand

from cl.lib.sunburnt import SolrInterface
from cl.people_db.import_judges.courtid_levels import courtid2statelevel
from cl.people_db.models import Person

leveldict = {
    'H': 'J',  # State Supreme Court
    'M': 'K',  # State Appellate Court
    'L': 'D',  # Lower Court
}

url_template = (
    'http://api.followthemoney.org/?'
    'f-core=1&'
    'c-exi=1&'
    'gro=c-t-id&'
    'mode=json&'
    'APIKey={key}&'
    's={state}&'
    'c-r-ot={level}&'
    'y={year}'
)


def make_dict_of_ftm_eids(use_pickle=True):
    """Build up a dictionary mapping jurisdiction IDs to candidates in those
    locations
    """
    pickle_location = '/tmp/eid_lists.pkl'
    if use_pickle:
        if os.path.isfile(pickle_location):
            with open(pickle_location, 'r') as f:
                print("Loading pickled candidate list. Read the command "
                      "documentation if this is not desired.")
                return pickle.load(f)
        else:
            print("Unable to find pickle file.")

    candidate_eid_lists = defaultdict(list)

    for courtid, (state, level) in courtid2statelevel.items():
        if level != 'H':
            # We only want high courts.
            continue
        for year in range(1989, 2017):
            url = url_template.format(
                key=settings.FTM_KEY,
                state=state,
                level=leveldict[level],
                year=year,
            )
            print("Getting url at: %s" % url)
            data = requests.get(url).json()

            if data['records'] == ['No Records']:
                print('  No records found in court %s and year %s.' % (
                    courtid,
                    year,
                ))
                continue
            print('  Found %s records in court %s and year %s' % (
                len(data['records']),
                courtid,
                year,
            ))

            for item in data['records']:
                # add an eid, name, year tuple to this court's list
                candidate_eid_lists[courtid].append({
                    'eid': item['Candidate_Entity']['id'],
                    'name': item['Candidate_Entity']['Candidate_Entity'],
                    'total': float(item['Total_$']['Total_$']),
                    'year': year,
                })

    if use_pickle:
        with open(pickle_location, 'w') as f:
            print("Creating pickle file at: %s" % pickle_location)
            pickle.dump(candidate_eid_lists, f)
    return candidate_eid_lists


def clear_old_values(do_it, debug):
    """Clear out the old values in the ftm fields. If debug or do_it is False,
    don't clear the values.
    """
    if not do_it or debug:
        return
    print("Clearing out all old values in FTM fields.")
    Person.objects.all().update(ftm_eid='', ftm_total_received=None)


def print_stats(match_stats, candidate_eid_lists):
    """Print the stats."""
    print("\n#########")
    print("# Stats #")
    print("#########")
    print("Finished matching judges:")
    for k, v in match_stats.items():
        print(" - %s had %s matches" % (v, k))
    ftm_judge_count = 0
    for v in candidate_eid_lists.values():
        ftm_judge_count += len(v)
    print("\nThere were %s judges in FTM that we matched "
          "against." % ftm_judge_count)


def update_judges_by_solr(candidate_id_map, debug):
    """Update judges by looking up each entity from FTM in Solr."""
    conn = SolrInterface(settings.SOLR_PEOPLE_URL, mode='r')
    match_stats = defaultdict(int)
    # These IDs are ones that cannot be updated due to being identified as
    # problematic in FTM's data.
    blacklisted_ids = defaultdict(set)
    for court_id, candidate_list in candidate_id_map.items():
        for candidate in candidate_list:
            # Look up the candidate in Solr.
            print("\nDoing: %s" % candidate['name'])
            name = (' AND '.join([word for word in candidate['name'].split() if
                                 len(word) > 1])).replace(',', '')
            results = conn.raw_query(**{
                'caller': 'ftm_update_judges_by_solr',
                'fq': [
                    'name:(%s)' % name,
                    'court_exact:%s' % court_id,
                    # This filters out Sr/Jr problems by insisting on recent
                    # positions. 1980 is arbitrary, based on testing.
                    'date_start:[1980-12-31T23:59:59Z TO *]',
                ],
                'q': "*:*",
            }).execute()

            if len(results) == 0:
                match_stats[len(results)] += 1
                print("  Found no matches.")

            elif len(results) == 1:
                match_stats[len(results)] += 1
                print("  Found one match: %s" % results[0]['name'])

                # Get the person from the DB and update them.
                pk = results[0]['id']
                if pk in blacklisted_ids:
                    continue
                p = Person.objects.get(pk=pk)
                if p.ftm_eid:
                    if p.ftm_eid != candidate['eid']:
                        print("  Found values in ftm database fields. This "
                              "indicates a duplicate in FTM.")

                        blacklisted_ids[p.pk].add(candidate['eid'])
                        blacklisted_ids[p.pk].add(p.ftm_eid)
                        p.ftm_eid = ""
                        p.ftm_total_received = None
                    else:
                        print("  Found values with matching EID. Adding "
                              "amounts, since this indicates multiple "
                              "jurisdictions that the judge was in.")
                        p.ftm_total_received += candidate['total']
                    if not debug:
                        p.save()
                else:
                    # No major problems. Proceed.
                    p.ftm_eid = candidate['eid']
                    p.ftm_total_received = candidate['total']
                    if not debug:
                        p.save()

            elif len(results) > 1:
                match_stats[len(results)] += 1
                print("  Found more than one match: %s" % results)

    print_stats(match_stats, candidate_id_map)
    print("Blacklisted IDs: %s" % blacklisted_ids)


class Command(BaseCommand):
{'fraction_non_alphanumeric': 0.11343260310282997, 'fraction_numerical': 0.00623684415685663, 'mean_word_length': 1.6994949494949494, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '409137', 'n_tokens_mistral': 3843, 'n_tokens_neox': 3573, 'n_words': 992}