<commit_before>#!/usr/bin/env python
#
# A parser for the 2008 earmarks data (XLS format) from http://taxpayer.net/
# 
# This script depends on xls2list which will convert the excel file to a 2d array. 
# It then does some trivial parsing of each field and outputs the data in a few ways.
#
# author: <NAME> (<EMAIL>)

import sys, re
from xls2list import *

HOUSE_REQUEST = 1
SENATE_REQUEST = 2
PRE_REDUCTION_AMT = 3
FINAL_AMOUNT = 4
BUDGET_REQUEST = 5
DESCRIPTION = 6
CITY_LOCATION=7
COUNTY=8
STATE=9
BILL=10
BILL_SECTION=11
BILL_SUBSECTION=12
PROJECT_HEADING=13
HOUSE_MEMBER = 14
HOUSE_PARTY = 15
HOUSE_STATE = 16
SENATE_MEMBER = 18
SENATE_PARTY= 19
SENATE_STATE = 20
PRESIDENTIAL_EARMARKS=21
UNDISCLOSED=22
INTENDED_RECIP=23
NOTES=24

class Earmark(object):
    houseRequest = None
    senateRequest = None
    preReductionAmount = None
    finalAmount = None
    budgetRequest = None
    description = None
    cityLocation = None
    county = None
    state = None
    bill = None
    billSection = None
    billSubsection = None
    projectHeading = None
    houseMembers = []
    #housePartys = [] leaving out house and state, hoping we know this already with the name(s) alone
    #houseStates = []
    senateMembers = []
    #senatePartys = []
    #senateStates = []
    presidentialEarmarks = None
    undisclosed = None
    intendedRecipient = None
    notes = None
    
    def __init__(self):
        pass
    
    def __str__(self):
        result = ["============================="]
        for attr in dir(self):
            if attr.startswith("__"):
                continue
            result.append(attr +  ' = ' + str(self.__getattribute__(attr)))
        return "\n".join(result)


def earmarkFromRow(row):
    e = Earmark()
    e.houseRequest = row[HOUSE_REQUEST]
    e.senateRequest = row[SENATE_REQUEST]
    e.preReductionAmount = row[PRE_REDUCTION_AMT]
    e.finalAmount = row[FINAL_AMOUNT]
    e.budgetRequest = row[BUDGET_REQUEST]
    e.description = row[DESCRIPTION]
    e.cityLocation = row[CITY_LOCATION]
    e.county = row[COUNTY]
    e.state = row[STATE]
    e.bill = row[BILL]
    e.billSection = row[BILL_SECTION]
    e.billSubsection = row[BILL_SUBSECTION]
    e.projectHeading = row[PROJECT_HEADING]
    if isinstance(row[HOUSE_MEMBER], basestring):<commit_msg>appended 'Dollars' to vars that represent them<commit_after>#!/usr/bin/env python
#
# A parser for the 2008 earmarks data (XLS format) from http://taxpayer.net/
# 
# This script depends on xls2list which will convert the excel file to a 2d array. 
# It then does some trivial parsing of each field and outputs the data in a few ways.
#
# author: <NAME> (<EMAIL>)

import sys, re
from xls2list import *

HOUSE_REQUEST = 1
SENATE_REQUEST = 2
PRE_REDUCTION_AMT = 3
FINAL_AMOUNT = 4
BUDGET_REQUEST = 5
DESCRIPTION = 6
CITY_LOCATION=7
COUNTY=8
STATE=9
BILL=10
BILL_SECTION=11
BILL_SUBSECTION=12
PROJECT_HEADING=13
HOUSE_MEMBER = 14
HOUSE_PARTY = 15
HOUSE_STATE = 16
SENATE_MEMBER = 18
SENATE_PARTY= 19
SENATE_STATE = 20
PRESIDENTIAL_EARMARKS=21
UNDISCLOSED=22
INTENDED_RECIP=23
NOTES=24

class Earmark(object):
    houseRequestDollars = None
    senateRequestDollars = None
    preReductionAmountDollars = None
    finalAmountDollars = None
    budgetRequestDollars = None
    description = None
    cityLocation = None
    county = None
    state = None
    bill = None
    billSection = None
    billSubsection = None
    projectHeading = None
    houseMembers = []
    #housePartys = [] leaving out house and state, hoping we know this already with the name(s) alone
    #houseStates = []
    senateMembers = []
    #senatePartys = []
    #senateStates = []
    presidentialEarmarks = None
    undisclosed = None
    intendedRecipient = None
    notes = None
    
    def __init__(self):
        pass
    
    def __str__(self):
        result = ["============================="]
        for attr in dir(self):
            if attr.startswith("__"):
                continue
            result.append(attr +  ' = ' + str(self.__getattribute__(attr)))
        return "\n".join(result)


def earmarkFromRow(row):
    e = Earmark()
    e.houseRequestDollars = row[HOUSE_REQUEST]
    e.senateRequestDollars = row[SENATE_REQUEST]
    e.preReductionAmountDollars = row[PRE_REDUCTION_AMT]
    e.finalAmountDollars = row[FINAL_AMOUNT]
    e.budgetRequestDollars = row[BUDGET_REQUEST]
    e.description = row[DESCRIPTION]
    e.cityLocation = row[CITY_LOCATION]
    e.county = row[COUNTY]
    e.state = row[STATE]
    e.bill = row[BILL]
    e.billSection = row[BILL_SECTION]
    e.billSubsection = row[BILL_SUBSECTION]
    e.projectHeading = row[PROJECT_HEADING]
    if isinstance(row[HOUSE_MEMBER], basestring):
{'fraction_non_alphanumeric': 0.09963806685118161, 'fraction_numerical': 0.01873536299765808, 'mean_word_length': 3.321987120515179, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3079858', 'n_tokens_mistral': 1696, 'n_tokens_neox': 1613, 'n_words': 395}