<commit_before>#!/usr/bin/env python

# gets CRP id for every member with a bioguide ID:

# options:
#  --cache: load from cache if present on disk (default: true)
#  --current: do *only* current legislators (default: true)
#  --historical: do *only* historical legislators (default: false)

import lxml.html, StringIO
import datetime
import re
import utils
import urllib2
import requests
from utils import download, load_data, save_data, parse_date
import json

debug = utils.flags().get('debug', False)

# default to caching
cache = utils.flags().get('cache', True)
force = not cache

# pick either current or historical
# order is important here, since current defaults to true
if utils.flags().get('historical', False):
  filename = "legislators-historical.yaml"
elif utils.flags().get('current', True):
  filename = "legislators-current.yaml"
else:
  print "No legislators selected."
  exit(0)

print "Loading %s..." % filename
legislators = load_data(filename)

api_file = open('cache/sunlight_api_key.txt','r')
api_key = api_file.read()

for m in legislators:
    url_BG = "http://transparencydata.com/api/1.0/entities/id_lookup.json?bioguide_id="
    if m["id"].has_key("bioguide"):
        url_BG += m["id"]["bioguide"]
        url_BG += "&apikey="+api_key
        root = lxml.html.parse(url_BG)
        jsondata = json.loads(root.find(".//p").text)
        try:
            IE_ID = jsondata[0]['id']
        except:
            continue
        url_CRP = "http://transparencydata.com/api/1.0/entities/"
        url_CRP += IE_ID
        url_CRP += ".json?apikey=" + api_key
        root2 = lxml.html.parse(url_CRP).getroot()
        jsondata2 = json.loads(root2.find(".//body").text_content())
        try:
            m["id"]["opensecrets"] = jsondata2['external_ids'][0]['id']
        except:
            continue

print "Saving data to %s..." % filename
save_data(legislators, filename)<commit_msg>Refactor to use json directly for IE download<commit_after>#!/usr/bin/env python

# gets CRP id for every member with a bioguide ID:

# options:
#  --cache: load from cache if present on disk (default: true)
#  --current: do *only* current legislators (default: true)
#  --historical: do *only* historical legislators (default: false)

import lxml.html, StringIO
import datetime
import re
import utils
import urllib2
import requests
from utils import download, load_data, save_data, parse_date
import json

debug = utils.flags().get('debug', False)

# default to caching
cache = utils.flags().get('cache', True)
force = not cache


only_bioguide = utils.flags().get('bioguide', None)


# pick either current or historical
# order is important here, since current defaults to true
if utils.flags().get('historical', False):
  filename = "legislators-historical.yaml"
elif utils.flags().get('current', True):
  filename = "legislators-current.yaml"
else:
  print "No legislators selected."
  exit(0)


print "Loading %s..." % filename
legislators = load_data(filename)


api_file = open('cache/sunlight_api_key.txt','r')
api_key = api_file.read()


for m in legislators:

    # this can't run unless we've already collected a bioguide for this person
    bioguide = m["id"].get("bioguide", None)
    if not bioguide:
        continue

    # if we've limited this to just one bioguide, skip over everyone else
    if only_bioguide and (bioguide != only_bioguide):
        continue

    url_BG = "http://transparencydata.com/api/1.0/entities/id_lookup.json?bioguide_id="
    url_BG += bioguide
    url_BG += "&apikey="+api_key

    root = lxml.html.parse(url_BG)

    destination = "legislators/influence_explorer/%s.json" % bioguide
    body = utils.download(url_BG, destination, force)

    jsondata = json.loads(body)
    try:
        IE_ID = jsondata[0]['id']
        print "Found IE ID: %s" % IE_ID
    except:
        continue
    url_CRP = "http://transparencydata.com/api/1.0/entities/"
    url_CRP += IE_ID
    url_CRP += ".json?apikey=" + api_key
    root2 = lxml.html.parse(url_CRP).getroot()
    jsondata2 = json.loads(root2.find(".//body").text_content())
    try:
        m["id"]["opensecrets"] = jsondata2['external_ids'][0]['id']
    except:
        continue

print "Saving data to %s..." % filename
save_data(legislators, filename)
{'fraction_non_alphanumeric': 0.13422661024917726, 'fraction_numerical': 0.005641748942172073, 'mean_word_length': 3.5703544575725026, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 6}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7512023', 'n_tokens_mistral': 1423, 'n_tokens_neox': 1417, 'n_words': 369}