<commit_before>import requests
from bs4 import BeautifulSoup
from login import keys

ARTICLE_SEARCH_URL = 'http://api.nytimes.com/svc/search/v2/articlesearch.json?api-key={key}'
SUNLIGHT_CONGRESS_URL = 'http://congress.api.sunlightfoundation.com/{method}?{query}&apikey={key}'

def get_json(url):
    return requests.get(url).json()

def sunlight_url(base, method, query, key):
    return base.format(method=method, query=query, key=key)

def sunlight_query(**kwargs):
    return '&'.join(key+'='+value for key,value in kwargs.items())

def save_url(url, path, name):
    r = requests.get(url)
    text = BeautifulSoup(r.text).get_text()
    with open(path + '/' + name + '.txt', 'w') as f:
        f.write(text)

query = sunlight_query(congress='113', per_page='50')
#query = ''

url = sunlight_url(SUNLIGHT_CONGRESS_URL, 'bills', query, keys['sunlight'])

bills = get_json(url)

#qHealth = sunlight_query(congress='113', query='health care')
#bHealth = get_json(sunlight_url(SUNLIGHT_CONGRESS_URL, 'bills/search', qHealth, keys['sunlight']))

def save_bills(bs):
    for b in enumerate(b['results']):
        try:
            link = b['last_version']['urls']['html']
            number = b['number']
            title = b['official_title']
            save_url(link, 'bill_text', str(number))
        except:
            pass<commit_msg>`main()` loops over bill pages<commit_after>import requests
from bs4 import BeautifulSoup
from login import keys

ARTICLE_SEARCH_URL = 'http://api.nytimes.com/svc/search/v2/articlesearch.json?api-key={key}'
SUNLIGHT_CONGRESS_URL = 'http://congress.api.sunlightfoundation.com/{method}?{query}&apikey={key}'

def get_json(url):
    return requests.get(url).json()

def sunlight_url(base, method, query, key):
    return base.format(method=method, query=query, key=key)

def sunlight_query(**kwargs):
    return '&'.join(key+'='+value for key,value in kwargs.items())

def save_url(url, path, name):
    r = requests.get(url)
    text = BeautifulSoup(r.text).get_text()
    with open(path + '/' + name + '.txt', 'w') as f:
        f.write(text)
        print('saved: ' + url)


#qHealth = sunlight_query(congress='113', query='health care')
#bHealth = get_json(sunlight_url(SUNLIGHT_CONGRESS_URL, 'bills/search', qHealth, keys['sunlight']))

def save_bills(bs):
    for b in bs['results']:
        number = b['number']
        title = b['official_title']
        if b.get('last_version', None):
            link = b['last_version']['urls']['html']
            save_url(link, 'bill_text', str(number))

def main():
    MAX_PAGES = 500
    loop = 0
    while loop < MAX_PAGES:
        query = sunlight_query(congress='113', per_page='50', page=str(loop))
        url = sunlight_url(SUNLIGHT_CONGRESS_URL, 'bills', query, keys['sunlight'])
        bills = get_json(url)
        save_bills(bills)
        loop = bills['page']['page'] + 1
    return loop
{'fraction_non_alphanumeric': 0.1608318890814558, 'fraction_numerical': 0.008665511265164644, 'mean_word_length': 4.019130434782609, 'pattern_counts': {'":': 0, '<': 4, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5138945', 'n_tokens_mistral': 1022, 'n_tokens_neox': 996, 'n_words': 198}