<commit_before>import csv
import yaml
import random
import time
import urllib2
import json

class PoliticalData():

    SPREADSHEET_CACHE_TIMEOUT = 60 # seconds

    overrides_data = {}     # the google spreadsheet overrides for each campaign
    scrape_times = {}       # the last google scrape timestamps for campaigns

    cache_handler = None
    campaigns = None
    legislators = None
    districts = None
    debug_mode = False

    def __init__(self, cache_handler, debug_mode):
        """
        Load data in database
        """
        legislators = []

        self.cache_handler = cache_handler
        self.debug_mode = debug_mode

        with open('data/legislators.csv') as f:
            reader = csv.DictReader(f)

            for legislator in reader:
                if legislator['senate_class']:
                    legislator['chamber'] = 'senate'
                else:
                    legislator['chamber'] = 'house'

                legislators.append(legislator)

        legislators = [l for l in legislators if l['in_office'] == '1']

        districts = []

        with open('data/districts.csv') as f:
            reader = csv.DictReader(
                f, fieldnames=['zipcode', 'state', 'district_number'])

            for district in reader:
                districts.append(district)

        with open('data/campaigns.yaml', 'r') as f:
            campaigns = {c['id']: c for c in yaml.load(f.read())}

        self.campaigns = campaigns
        self.legislators = legislators
        self.districts = districts

    def get_campaign(self, campaign_id):
        if campaign_id in self.campaigns:
            return dict(self.campaigns['default'],
                        **self.campaigns[campaign_id])

    def get_senators(self, districts, get_one=False):
        states = [d['state'] for d in districts]

        senators = [l for l in self.legislators
                if l['chamber'] == 'senate'
                and l['state'] in states]

        random.shuffle(senators)    # mix it up! always do this :)

        if get_one:
            return [random.choice(senators)]
        else:
            return senators

    def get_house_members(self, districts, get_one=False):
        states = [d['state'] for d in districts]
        district_numbers = [d['district_number'] for d in districts]

        reps = [l for l in self.legislators
                if l['chamber'] == 'house'
                and l['state'] in states
                and l['district'] in district_numbers]

        if get_one:
            return [random.choice(reps)]
        else:
            return reps

    def locate_member_ids(self, zipcode, campaign):
        """get congressional member ids from zip codes to districts data"""
        local_districts = [d for d in self.districts
                           if d['zipcode'] == str(zipcode)]
        member_ids = []

        individual_target = campaign.get('target_member_id', None)

        if individual_target:
            member_ids = [individual_target]
            return member_ids

        target_senate = campaign.get('target_senate')
        target_house_first = campaign.get('target_house_first')
        target_house = campaign.get('target_house')

        # Instantiate some extra variables related to state-specific overrides
        target_individual = None
        state = None
        first_call_name = None
        first_call_number = None

        # check if there's a google spreadsheet with state-specific overrides
        if self.has_special_overrides(local_districts, campaign):

            overrides = self.get_override_values(local_districts, campaign)

            target_senate = overrides['target_senate']
            target_house_first = overrides['target_house_first']
            target_house = overrides['target_house']
            target_individual = overrides['target_individual']
            first_call_name = overrides['first_call_name']
            first_call_number = overrides['first_call_number']
            state = overrides['_STATE_ABBREV']

        # filter list by campaign target_house, target_senate
        if target_senate and not target_house_first:
            sens = [s['bioguide_id'] for s
                        in self.get_senators(local_districts, campaign.get('only_call_1_sen', False))]
            member_ids.extend(sens)

        if target_house:
            reps = [h['bioguide_id'] for h
                       in self.get_house_members(local_districts, campaign.get('only_call_1_rep', False))]
            member_ids.extend(reps)

        if target_senate and target_house_first:
            member_ids.extend([s['bioguide_id'] for s
                       in self.get_senators(local_districts, campaign.get('only_call_1_sen', False))])

        if campaign.get('randomize_order', False):
            random.shuffle(member_ids)

        # if targeting an individual by name, pop them to the front of the list
        # JL NOTE ~ Tony C=>A<=rdenas (C001097) has bad data, unicode warning
        if target_individual != None and target_individual != "":
            for l in self.legislators:
                if l['lastname'] == target_individual and l['state'] == state:<commit_msg>debug statements, defensive coding<commit_after>import csv
import yaml
import random
import time
import urllib2
import json

class PoliticalData():

    SPREADSHEET_CACHE_TIMEOUT = 60 # seconds

    overrides_data = {}     # the google spreadsheet overrides for each campaign
    scrape_times = {}       # the last google scrape timestamps for campaigns

    cache_handler = None
    campaigns = None
    legislators = None
    districts = None
    debug_mode = False

    def __init__(self, cache_handler, debug_mode):
        """
        Load data in database
        """
        legislators = []

        self.cache_handler = cache_handler
        self.debug_mode = debug_mode

        with open('data/legislators.csv') as f:
            reader = csv.DictReader(f)

            for legislator in reader:
                if legislator['senate_class']:
                    legislator['chamber'] = 'senate'
                else:
                    legislator['chamber'] = 'house'

                legislators.append(legislator)

        legislators = [l for l in legislators if l['in_office'] == '1']

        districts = []

        with open('data/districts.csv') as f:
            reader = csv.DictReader(
                f, fieldnames=['zipcode', 'state', 'district_number'])

            for district in reader:
                districts.append(district)

        with open('data/campaigns.yaml', 'r') as f:
            campaigns = {c['id']: c for c in yaml.load(f.read())}

        self.campaigns = campaigns
        self.legislators = legislators
        self.districts = districts

    def get_campaign(self, campaign_id):
        if campaign_id in self.campaigns:
            return dict(self.campaigns['default'],
                        **self.campaigns[campaign_id])

    def get_senators(self, districts, get_one=False):
        states = [d['state'] for d in districts]

        senators = [l for l in self.legislators
                if l['chamber'] == 'senate'
                and l['state'] in states]

        random.shuffle(senators)    # mix it up! always do this :)

        if senators and get_one:
            return [random.choice(senators)]
        else:
            return senators

    def get_house_members(self, districts, get_one=False):
        states = [d['state'] for d in districts]
        district_numbers = [d['district_number'] for d in districts]

        reps = [l for l in self.legislators
                if l['chamber'] == 'house'
                and l['state'] in states
                and l['district'] in district_numbers]

        if reps and get_one:
            return [random.choice(reps)]
        else:
            return reps

    def locate_member_ids(self, zipcode, campaign):
        """get congressional member ids from zip codes to districts data"""
        local_districts = [d for d in self.districts
                           if d['zipcode'] == str(zipcode)]
        member_ids = []

        individual_target = campaign.get('target_member_id', None)

        if individual_target:
            member_ids = [individual_target]
            return member_ids

        target_senate = campaign.get('target_senate')
        target_house_first = campaign.get('target_house_first')
        target_house = campaign.get('target_house')

        # Instantiate some extra variables related to state-specific overrides
        target_individual = None
        state = None
        first_call_name = None
        first_call_number = None

        # check if there's a google spreadsheet with state-specific overrides
        if self.has_special_overrides(local_districts, campaign):

            overrides = self.get_override_values(local_districts, campaign)

            target_senate = overrides['target_senate']
            target_house_first = overrides['target_house_first']
            target_house = overrides['target_house']
            target_individual = overrides['target_individual']
            first_call_name = overrides['first_call_name']
            first_call_number = overrides['first_call_number']
            state = overrides['_STATE_ABBREV']

        # filter list by campaign target_house, target_senate
        if target_senate and not target_house_first:
            sens = [s['bioguide_id'] for s
                        in self.get_senators(local_districts, campaign.get('only_call_1_sen', False))]
            if self.debug_mode:
                print "got %s sens" % sens
            member_ids.extend(sens)

        if target_house:
            reps = [h['bioguide_id'] for h
                       in self.get_house_members(local_districts, campaign.get('only_call_1_rep', False))]
            if self.debug_mode:
                print "got %s reps" % reps
            member_ids.extend(reps)

        if target_senate and target_house_first:
            sens = [s['bioguide_id'] for s
                       in self.get_senators(local_districts, campaign.get('only_call_1_sen', False))]
            if self.debug_mode:
                print "got %s sens" % sens
            member_ids.extend(sens)


        if campaign.get('randomize_order', False):
            random.shuffle(member_ids)

        # if targeting an individual by name, pop them to the front of the list
        # JL NOTE ~ <NAME>=>A<=rdenas (C001097) has bad data, unicode warning
        if target_individual != None and target_individual != "":
            for l in self.legislators:
                if l['lastname'] == target_individual and l['state'] == state:
{'fraction_non_alphanumeric': 0.08958003538504516, 'fraction_numerical': 0.002421082037433653, 'mean_word_length': 2.1495601173020527, 'pattern_counts': {'":': 2, '<': 6, '<?xml version=': 0, '>': 6, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6371383', 'n_tokens_mistral': 3104, 'n_tokens_neox': 2894, 'n_words': 774}