<commit_before>import requests
from requests.sessions import Session
from BeautifulSoup import BeautifulSoup
from urllib import urlencode
import csv
import os
from cStringIO import StringIO
from string import ascii_lowercase
from boto.s3.connection import S3Connection
from boto.s3.key import Key

AWS_KEY = os.environ['AWS_ACCESS_KEY']
AWS_SECRET = os.environ['AWS_SECRET_KEY']

CONT_GET_PARAMS = {
    'AddressSearchType': 'Starts with',
    'Archived': 'false',
    'CitySearchType': 'Starts with',
    'ContributionType': 'All Types',
    'DownloadListType': 'Receipts',
    'EmployerSearchType': 'Starts with',
    'FirstNameSearchType': 'Starts with',
    'LastOnlyNameSearchType': 'Starts with',
    'LastOnlyName': '',
    'FirstName': '',
    'Address': '',
    'City': '',
    'State': '',
    'Zip': '',
    'ZipThru': '',
    'Occupation': '',
    'Employer': '',
    'VendorLastOnlyName': '',
    'VendorFirstName': '',
    'VendorAddress': '',
    'VendorCity': '',
    'VendorState': '',
    'VendorZip': '',
    'VendorZipThru': '',
    'LinkedQuery': 'false',
    'OccupationSearchType': 'Starts with',
    'OrderBy': 'Date Received - most recent first',
    'PurposeState': 'Starts with',
    'QueryType': 'Contrib',
    'RcvDate': None,
    'RcvDateThru': None,
    'VendorAddressSearchType': 'Starts with',
    'VendorCitySearchType': 'Starts with',
    'VendorFirstNameSearchType': 'Starts with',
    'VendorLastOnlyNameSearchType': 'Starts with',
    'OtherReceiptsDescriptionSearchType': '',
    'OtherReceiptsDescription': '',
    'Purpose': '',
    'Amount': '',
    'AmountThru': ''
}

COMM_GET_PARAMS = {
    'DownloadListType': 'Committees',
    'Active': 'true',
    'NameSearchType': 'Starts with',
    'Name': None,
    'AddressSearchType': 'Starts with',
    'Address': '',
    'CitySearchType': 'Starts with',
    'City': '',
    'State': '',
    'Zip': '',
    'ZipThru': '',
    'CommitteeID': '',
    'CmteTypeSearch': 'Select a Type',
    'OrderBy': 'Committee Name - A to Z'
}

EXP_GET_PARAMS = {
    'DownloadListType': 'Expenditures',
    'ExpenditureSearchType': 'All',
    'ExpenditureType': 'All Types',
    'LastOnlyNameSearchType': 'Starts with',
    'LastOnlyName': '',
    'FirstNameSearchType': 'Starts with',
    'FirstName': '',
    'AddressSearchType': 'Starts with',
    'Address': '',
    'CitySearchType': 'Starts with',
    'City': '',
    'State': '',
    'Zip': '',
    'ZipThru': '',
    'CandidateNameSearchType': 'Starts with',
    'CandidateName': '',
    'OfficeSearchType': 'Starts with',
    'Office': '',
    'Opposing': '',
    'Supporting': '',
    'PurposeSearchType': 'Starts with',
    'Purpose': '',
    'BeneficiarySearchType': 'Starts with',
    'Beneficiary': '',
    'Amount': '',
    'AmountThru': '',
    'ExpendedDate': None,
    'ExpendedDateThru': None,
    'Archived': 'false',
    'QueryType': 'Expend',
    'LinkedQuery': 'false',
    'OrderBy': 'Last or Only Name - A to Z',
}

BASE_URL = 'http://www.elections.state.il.us/CampaignDisclosure'

ALPHA_NUM = [l for l in ascii_lowercase] + [n for n in range(10)]

def fetch_data(dl_type=None, **kwargs):
    """ 
    Fetch Receipts, Expenditures, and Committees. 
    dl_type is one of those three choices. 
    kwargs depend on the choice. 
    Receipts and Expenditures need start_date and end_date for search.
    Committees need a name_start kwarg to pass into the search.
    
    Seems like the maximum that you can get is about 250,000 records at a time.
    """
    s = Session()
    if dl_type == 'Receipts':
        CONT_GET_PARAMS['RcvDate'] = kwargs['start_date']
        CONT_GET_PARAMS['RcvDateThru'] = kwargs['end_date']
        url = '%s/DownloadList.aspx?%s' % (BASE_URL, urlencode(CONT_GET_PARAMS))
    elif dl_type == 'Committees':
        COMM_GET_PARAMS['Name'] = kwargs['name_start']
    elif dl_type == 'Expenditures':
        EXP_GET_PARAMS['ExpendedDate'] = kwargs['start_date']
        EXP_GET_PARAMS['ExpendedDateThru'] = kwargs['end_date']
        url = '%s/DownloadList.aspx?%s' % (BASE_URL, urlencode(EXP_GET_PARAMS))
    g = s.get(url)
    soup = BeautifulSoup(g.content)
    view_state = soup.find('input', attrs={'id': '__VIEWSTATE'}).get('value')
    event_val = soup.find('input', attrs={'id': '__EVENTVALIDATION'}).get('value')
    d = {
        '__VIEWSTATE': view_state,
        '__EVENTVALIDATION': event_val,
        '__EVENTTARGET': 'ctl00$ContentPlaceHolder1$btnText',
        'ctl00$pnlMenu_CollapsiblePanelExtender_ClientState': 'true',
        'ctl00$AccordionStateBoardMenu_AccordionExtender_ClientState': '0',
        'ctl00$mtbSearch': '',
        'ctl00$AccordionPaneStateBoardMenu_content$AccordionMainContent_AccordionExtender_ClientState': '-1',
        'hiddenInputToUpdateATBuffer_CommonToolkitScripts': '1',
        '__EVENTARGUMENT': '',
    }
    dl_page = s.post(url, data=d)
    return dl_page.content

def load_disclosure(dl_type, start, end):
    conn = S3Connection(AWS_KEY, AWS_SECRET)
    bucket = conn.get_bucket('il-elections')
    for year in range(start, end):
        start_date = '1/1/%s' % year
        end_date = '12/31/%s' % year
        print 'Saving %s %s' % (year, dl_type)
        content = fetch_data(dl_type=dl_type, start_date=start_date, end_date=end_date)
        k = Key(bucket)
        k.key = '%s/%s_%s.tsv' % (dl_type, year, dl_type.lower())
        k.set_contents_from_string(content)
        k.make_public()
    return None

if __name__ == "__main__":
    from datetime import datetime
    this_year = datetime.now().year
    for dl_type in ['Expenditures', 'Receipts']:
        load_disclosure(dl_type, this_year, this_year + 1)<commit_msg>Scraping all receipts and expenditures every day<commit_after>import requests
from requests.sessions import Session
from BeautifulSoup import BeautifulSoup
from urllib import urlencode
import csv
import os
from cStringIO import StringIO
from string import ascii_lowercase
from boto.s3.connection import S3Connection
from boto.s3.key import Key

AWS_KEY = os.environ['AWS_ACCESS_KEY']
AWS_SECRET = os.environ['AWS_SECRET_KEY']

CONT_GET_PARAMS = {
    'AddressSearchType': 'Starts with',
    'Archived': 'false',
    'CitySearchType': 'Starts with',
    'ContributionType': 'All Types',
    'DownloadListType': 'Receipts',
    'EmployerSearchType': 'Starts with',
    'FirstNameSearchType': 'Starts with',
    'LastOnlyNameSearchType': 'Starts with',
    'LastOnlyName': '',
    'FirstName': '',
    'Address': '',
    'City': '',
    'State': '',
    'Zip': '',
    'ZipThru': '',
    'Occupation': '',
    'Employer': '',
    'VendorLastOnlyName': '',
    'VendorFirstName': '',
    'VendorAddress': '',
    'VendorCity': '',
    'VendorState': '',
    'VendorZip': '',
    'VendorZipThru': '',
    'LinkedQuery': 'false',
    'OccupationSearchType': 'Starts with',
    'OrderBy': 'Date Received - most recent first',
    'PurposeState': 'Starts with',
    'QueryType': 'Contrib',
    'RcvDate': None,
    'RcvDateThru': None,
    'VendorAddressSearchType': 'Starts with',
    'VendorCitySearchType': 'Starts with',
    'VendorFirstNameSearchType': 'Starts with',
    'VendorLastOnlyNameSearchType': 'Starts with',
    'OtherReceiptsDescriptionSearchType': '',
    'OtherReceiptsDescription': '',
    'Purpose': '',
    'Amount': '',
    'AmountThru': ''
}

COMM_GET_PARAMS = {
    'DownloadListType': 'Committees',
    'Active': 'true',
    'NameSearchType': 'Starts with',
    'Name': None,
    'AddressSearchType': 'Starts with',
    'Address': '',
    'CitySearchType': 'Starts with',
    'City': '',
    'State': '',
    'Zip': '',
    'ZipThru': '',
    'CommitteeID': '',
    'CmteTypeSearch': 'Select a Type',
    'OrderBy': 'Committee Name - A to Z'
}

EXP_GET_PARAMS = {
    'DownloadListType': 'Expenditures',
    'ExpenditureSearchType': 'All',
    'ExpenditureType': 'All Types',
    'LastOnlyNameSearchType': 'Starts with',
    'LastOnlyName': '',
    'FirstNameSearchType': 'Starts with',
    'FirstName': '',
    'AddressSearchType': 'Starts with',
    'Address': '',
    'CitySearchType': 'Starts with',
    'City': '',
    'State': '',
    'Zip': '',
    'ZipThru': '',
    'CandidateNameSearchType': 'Starts with',
    'CandidateName': '',
    'OfficeSearchType': 'Starts with',
    'Office': '',
    'Opposing': '',
    'Supporting': '',
    'PurposeSearchType': 'Starts with',
    'Purpose': '',
    'BeneficiarySearchType': 'Starts with',
    'Beneficiary': '',
    'Amount': '',
    'AmountThru': '',
    'ExpendedDate': None,
    'ExpendedDateThru': None,
    'Archived': 'false',
    'QueryType': 'Expend',
    'LinkedQuery': 'false',
    'OrderBy': 'Last or Only Name - A to Z',
}

BASE_URL = 'http://www.elections.state.il.us/CampaignDisclosure'

ALPHA_NUM = [l for l in ascii_lowercase] + [n for n in range(10)]

def fetch_data(dl_type=None, **kwargs):
    """ 
    Fetch Receipts, Expenditures, and Committees. 
    dl_type is one of those three choices. 
    kwargs depend on the choice. 
    Receipts and Expenditures need start_date and end_date for search.
    Committees need a name_start kwarg to pass into the search.
    
    Seems like the maximum that you can get is about 250,000 records at a time.
    """
    s = Session()
    if dl_type == 'Receipts':
        CONT_GET_PARAMS['RcvDate'] = kwargs['start_date']
        CONT_GET_PARAMS['RcvDateThru'] = kwargs['end_date']
        url = '%s/DownloadList.aspx?%s' % (BASE_URL, urlencode(CONT_GET_PARAMS))
    elif dl_type == 'Committees':
        COMM_GET_PARAMS['Name'] = kwargs['name_start']
    elif dl_type == 'Expenditures':
        EXP_GET_PARAMS['ExpendedDate'] = kwargs['start_date']
        EXP_GET_PARAMS['ExpendedDateThru'] = kwargs['end_date']
        url = '%s/DownloadList.aspx?%s' % (BASE_URL, urlencode(EXP_GET_PARAMS))
    g = s.get(url)
    soup = BeautifulSoup(g.content)
    view_state = soup.find('input', attrs={'id': '__VIEWSTATE'}).get('value')
    event_val = soup.find('input', attrs={'id': '__EVENTVALIDATION'}).get('value')
    d = {
        '__VIEWSTATE': view_state,
        '__EVENTVALIDATION': event_val,
        '__EVENTTARGET': 'ctl00$ContentPlaceHolder1$btnText',
        'ctl00$pnlMenu_CollapsiblePanelExtender_ClientState': 'true',
        'ctl00$AccordionStateBoardMenu_AccordionExtender_ClientState': '0',
        'ctl00$mtbSearch': '',
        'ctl00$AccordionPaneStateBoardMenu_content$AccordionMainContent_AccordionExtender_ClientState': '-1',
        'hiddenInputToUpdateATBuffer_CommonToolkitScripts': '1',
        '__EVENTARGUMENT': '',
    }
    dl_page = s.post(url, data=d)
    return dl_page.content

def load_disclosure(dl_type, start, end):
    conn = S3Connection(AWS_KEY, AWS_SECRET)
    bucket = conn.get_bucket('il-elections')
    for year in range(start, end):
        start_date = '1/1/%s' % year
        end_date = '12/31/%s' % year
        print 'Saving %s %s' % (year, dl_type)
        content = fetch_data(dl_type=dl_type, start_date=start_date, end_date=end_date)
        k = Key(bucket)
        k.key = '%s/%s_%s.tsv' % (dl_type, year, dl_type.lower())
        k.set_contents_from_string(content)
        k.make_public()
    return None

if __name__ == "__main__":
    from datetime import datetime
    this_year = datetime.now().year
    for dl_type in ['Expenditures', 'Receipts']:
        load_disclosure(dl_type, 1989, this_year + 1)
{'fraction_non_alphanumeric': 0.15559471365638766, 'fraction_numerical': 0.006167400881057269, 'mean_word_length': 3.650143383859074, 'pattern_counts': {'":': 2, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 10, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3649545', 'n_tokens_mistral': 3987, 'n_tokens_neox': 3867, 'n_words': 805}