<commit_before> #!/usr/bin/env python

# gets ICPSR ID for every member

# options:
#  --cache: load from cache if present on disk (default: true)
#  --bioguide: load only one legislator, by his/her bioguide ID
#  --congress: do *only* updates for legislators serving in specific congress

import utils
from utils import load_data, save_data, parse_date
import string
import csv
import unicodedata

def run():

    # default to caching
    cache = utils.flags().get('cache', True)
    force = not cache


    only_bioguide = utils.flags().get('bioguide', None)
    congress = utils.flags().get('congress',None)


    data_files = []

    print("Loading %s..." % "legislators-current.yaml")
    legislators = load_data("legislators-current.yaml")
    data_files.append((legislators,"legislators-current.yaml"))
    print("Loading %s..." % "legislators-historical.yaml")
    legislators = load_data("legislators-historical.yaml")
    data_files.append((legislators,"legislators-historical.yaml"))

    # load member data from vote view
    if congress == None:
        raise Exception("the --congress flag is required")
    elif int(congress) < 10 and int(congress) > 0:
        url_senate = "https://voteview.com/static/data/out/members/S00%s_members.csv" % congress
        url_house = "https://voteview.com/static/data/out/members/H00%s_members.csv" % congress
    elif int(congress) < 100 and int(congress) >= 10:
        url_senate = "https://voteview.com/static/data/out/members/S0%s_members.csv" % congress
        url_house = "https://voteview.com/static/data/out/members/H0%s_members.csv" % congress
    elif int(congress) >= 100:
        url_senate = "https://voteview.com/static/data/out/members/S%s_members.csv" % congress
        url_house = "https://voteview.com/static/data/out/members/H%s_members.csv" % congress
    else:
        raise Exception("no data for congress " + congress)

    senate_destination = "icpsr/source/senate_rollcall%s.txt" % congress
    senate_data = utils.download(url_senate, senate_destination, force)

    house_destination = "icpsr/source/house_rollcall%s.txt" % congress
    house_data = utils.download(url_house, house_destination, force)

    error_log = csv.writer(open("cache/errors/mismatch/mismatch_%s.csv" % congress, "w"))
    error_log.writerow(["error_type","matches","icpsr_name","icpsr_state","is_territory","old_id","new_id"])



    read_files = [("sen",senate_data),("rep",house_data)]
    print("Running for congress " + congress)
    for read_file in read_files:
        for data_file in data_files:
            for legislator in data_file[0]:
                num_matches = 0
                write_id = ""
            #     # # this can't run unless we've already collected a bioguide for this person
            #     bioguide = legislator["id"].get("bioguide", None)
            #     # if we've limited this to just one bioguide, skip over everyone else
            #     if only_bioguide and (bioguide != only_bioguide):
            #         continue
            #     #if not in currently read chamber, skip
            #     chamber = legislator['terms'][len(legislator['terms'])-1]['type']
            #     if chamber != read_file[1]:
            #         continue
            #
            #     #only run for selected congress
            #     latest_congress = utils.congress_from_legislative_year(utils.legislative_year(parse_date(legislator['terms'][len(legislator['terms'])-1]['start'])))
            #     if chamber == "sen":
            #         congresses = [latest_congress,latest_congress+1,latest_congress+2]
            #     else:
            #         congresses =[latest_congress]
            #
            #     if int(congress) not in congresses:
            #         continue
            #
            #     # pull data to match from yaml
            #
            #     last_name_unicode = legislator['name']['last'].upper().strip().replace('\'','')
            #     last_name = unicodedata.normalize('NFD', str(last_name_unicode)).encode('ascii', 'ignore')
            #     state = utils.states[legislator['terms'][len(legislator['terms'])-1]['state']].upper()[:7].strip()
            #     # select icpsr source data based on more recent chamber
            #
            #     lines = read_file[0].split('\n')
            #     for line in lines:
            #         # parse source data<commit_msg>added key,value to read_files loop, simplified getting last name by removing endcoding<commit_after> #!/usr/bin/env python

# gets ICPSR ID for every member

# options:
#  --cache: load from cache if present on disk (default: true)
#  --bioguide: load only one legislator, by his/her bioguide ID
#  --congress: do *only* updates for legislators serving in specific congress

import utils
from utils import load_data, save_data, parse_date
import string
import csv
import unicodedata

def run():

    # default to caching
    cache = utils.flags().get('cache', True)
    force = not cache


    only_bioguide = utils.flags().get('bioguide', None)
    congress = utils.flags().get('congress',None)


    data_files = []

    print("Loading %s..." % "legislators-current.yaml")
    legislators = load_data("legislators-current.yaml")
    data_files.append((legislators,"legislators-current.yaml"))
    print("Loading %s..." % "legislators-historical.yaml")
    legislators = load_data("legislators-historical.yaml")
    data_files.append((legislators,"legislators-historical.yaml"))

    # load member data from vote view
    if congress == None:
        raise Exception("the --congress flag is required")
    elif int(congress) < 10 and int(congress) > 0:
        url_senate = "https://voteview.com/static/data/out/members/S00%s_members.csv" % congress
        url_house = "https://voteview.com/static/data/out/members/H00%s_members.csv" % congress
    elif int(congress) < 100 and int(congress) >= 10:
        url_senate = "https://voteview.com/static/data/out/members/S0%s_members.csv" % congress
        url_house = "https://voteview.com/static/data/out/members/H0%s_members.csv" % congress
    elif int(congress) >= 100:
        url_senate = "https://voteview.com/static/data/out/members/S%s_members.csv" % congress
        url_house = "https://voteview.com/static/data/out/members/H%s_members.csv" % congress
    else:
        raise Exception("no data for congress " + congress)

    senate_destination = "icpsr/source/senate_rollcall%s.txt" % congress
    senate_data = utils.download(url_senate, senate_destination, force)

    house_destination = "icpsr/source/house_rollcall%s.txt" % congress
    house_data = utils.download(url_house, house_destination, force)

    error_log = csv.writer(open("cache/errors/mismatch/mismatch_%s.csv" % congress, "w"))
    error_log.writerow(["error_type","matches","icpsr_name","icpsr_state","is_territory","old_id","new_id"])



    read_files = [("sen",senate_data),("rep",house_data)]
    print("Running for congress " + congress)
    for read_file_chamber,read_file_content in read_files:
        for data_file in data_files:
            for legislator in data_file[0]:
                num_matches = 0
                write_id = ""
                # this can't run unless we've already collected a bioguide for this person
                bioguide = legislator["id"].get("bioguide", None)
                # if we've limited this to just one bioguide, skip over everyone else
                if only_bioguide and (bioguide != only_bioguide):
                    continue
                #if not in currently read chamber, skip
                chamber = legislator['terms'][len(legislator['terms'])-1]['type']
                if chamber != read_file_chamber:
                    continue

                #only run for selected congress
                latest_congress = utils.congress_from_legislative_year(utils.legislative_year(parse_date(legislator['terms'][len(legislator['terms'])-1]['start'])))
                if chamber == "sen":
                    congresses = [latest_congress,latest_congress+1,latest_congress+2]
                else:
                    congresses =[latest_congress]

                if int(congress) not in congresses:
                    continue

                # pull data to match from yaml

                last_name = legislator['name']['last'].upper()
                state = utils.states[legislator['terms'][len(legislator['terms'])-1]['state']].upper()[:7].strip()

            #     # # this can't run unless we've already collected a bioguide for this person
            #     bioguide = legislator["id"].get("bioguide", None)
            #     # if we've limited this to just one bioguide, skip over everyone else
            #     if only_bioguide and (bioguide != only_bioguide):
            #         continue
            #     #if not in currently read chamber, skip
            #     chamber = legislator['terms'][len(legislator['terms'])-1]['type']
            #     if chamber != read_file[1]:
            #         continue
            #
            #     #only run for selected congress
            #     latest_congress = utils.congress_from_legislative_year(utils.legislative_year(parse_date(legislator['terms'][len(legislator['terms'])-1]['start'])))
            #     if chamber == "sen":
            #         congresses = [latest_congress,latest_congress+1,latest_congress+2]
            #     else:
            #         congresses =[latest_congress]
            #
            #     if int(congress) not in congresses:
            #         continue
            #
            #     # pull data to match from yaml
            #
            #     last_name_unicode = legislator['name']['last'].upper().strip().replace('\'','')
            #     last_name = unicodedata.normalize('NFD', str(last_name_unicode)).encode('ascii', 'ignore')
            #     state = utils.states[legislator['terms'][len(legislator['terms'])-1]['state']].upper()[:7].strip()
            #     # select icpsr source data based on more recent chamber
            #
            #     lines = read_file[0].split('\n')
            #     for line in lines:
            #         # parse source data
{'fraction_non_alphanumeric': 0.12415216750221174, 'fraction_numerical': 0.005897965202005308, 'mean_word_length': 2.503443526170799, 'pattern_counts': {'":': 3, '<': 7, '<?xml version=': 0, '>': 9, 'https://': 12, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1835635', 'n_tokens_mistral': 3144, 'n_tokens_neox': 3046, 'n_words': 639}