<commit_before>"""
FEC data loader.
"""
from __future__ import with_statement
import itertools, datetime
import web
import tools
from tools import db
from parse import fec_cobol, fec_csv, fec_crude_csv
import psycopg2 # @@sigh
import cgitb
import glob

fec2pol = {}
def load_fec_ids():
    with db.transaction():
        db.delete('politician_fec_ids', '1=1')
        fh = iter(file('../data/crawl/opensecrets/FEC_CRP_ID.tsv'))
        header = fh.next()
        for line in fh:
            fec_id, crp_id = line.split()
            if tools.opensecretsp(crp_id):
                fec2pol[fec_id] = tools.opensecretsp(crp_id)
                db.insert('politician_fec_ids',
                  seqname=False,
                  politician_id=tools.opensecretsp(crp_id),
                  fec_id=fec_id)

def load_fec_cans():
    for can in fec_cobol.parse_cansum():
        can = web.storage(can)
        if can.candidate_id in fec2pol:
            pol_id = fec2pol[can.candidate_id]

            total = float(can.total_receipts)
            if total == 0.0:
                print "Oops:", pol_id, total, can.total_receipts, can.total_disbursements, can.contrib_from_candidate, can.total_indiv_contrib, can.contrib_from_other_pc
                continue
            db.update('politician', where='id = $pol_id', vars=locals(),
              money_raised = can.total_receipts,
              pct_spent = can.total_disbursements/total,
              pct_self = can.contrib_from_candidate/total,
              pct_indiv = can.total_indiv_contrib/total,
              pct_pac = can.contrib_from_other_pc/total
            )

def load_fec_committees():
    db.delete('committee', '1=1')
    for f in fec_cobol.parse_committees(reverse=True):
        f = web.storage(f)
        try:
            db.insert('committee', seqname=False,
              id = f.committee_id,
              name = f.committee_name,
              treasurer = f.treasurer_name,
              street1 = f.street_one,
              street2 = f.street_two,
              city = f.city,
              state = f.state,
              zip = f.zip,
              connected_org_name = f.connected_org_name,
              candidate_id = f.candidate_id
            )
        except psycopg2.IntegrityError:
            pass # already imported

def load_fec_contributions():
    t = db.transaction(); n = 0
    db.delete('contribution', '1=1')
    for f in fec_cobol.parse_contributions():
        f = web.storage(f)
        f.occupation = f.occupation.replace('N/A', '')
        if '/' in f.occupation:
            employer, occupation = f.occupation.split('/', 1)
        else:
            employer = ''
            occupation = f.occupation
        
        try:
            datetime.date(*[int(x) for x in f.date.split('-')])
        except ValueError:
            f.date = None
        
        db.insert('contribution',
          fec_record_id = f.get('fec_record_id'),
          microfilm_loc = f.microfilm_loc,
          recipient_id = f.filer_id,
          name = f.name,
          street = f.get('street'),
          city = f.city,
          state = f.state,
          zip = f.zip,
          occupation = occupation,
          employer = employer,
          employer_stem = tools.stemcorpname(employer),
          committee = f.from_id or None,
          sent = f.date,
          amount = f.amount
        )
        n += 1
        if n % 10000 == 0: t.commit(); t = db.transaction(); print n


def load_fec_efilings(filepattern=fec_crude_csv.DEFAULT_EFILINGS_FILEPATTERN):
    for f, schedules in fec_crude_csv.parse_efilings(glob.glob(filepattern)):
        for s in schedules:
            if s.get('type') == 'contribution':
                # XXX all this code for politician_id is currently
                # dead, does nothing useful
                politician_id = None
                if f.get('candidate_fec_id'):
                    fec_id = f['candidate_fec_id']
                    pol_fec_id = list(db.select('politician_fec_ids',
                                                where='fec_id=$fec_id',
                                                vars=locals()))
                    if pol_fec_id and len(pol_fec_id) == 1:<commit_msg>Fixed two bugs in load/fec.py.<commit_after>"""
FEC data loader.
"""
from __future__ import with_statement
import itertools, datetime
import web
import tools
from tools import db
from parse import fec_cobol, fec_csv, fec_crude_csv
import psycopg2 # @@sigh
import cgitb
import glob

fec2pol = {}
def load_fec_ids():
    with db.transaction():
        db.delete('politician_fec_ids', '1=1')
        fh = iter(file('../data/crawl/opensecrets/FEC_CRP_ID.tsv'))
        header = fh.next()
        for line in fh:
            fec_id, crp_id = line.split()
            if tools.opensecretsp(crp_id):
                fec2pol[fec_id] = tools.opensecretsp(crp_id)
                db.insert('politician_fec_ids',
                  seqname=False,
                  politician_id=tools.opensecretsp(crp_id),
                  fec_id=fec_id)

def load_fec_cans():
    for can in fec_cobol.parse_cansum():
        can = web.storage(can)
        if can.candidate_id in fec2pol:
            pol_id = fec2pol[can.candidate_id]

            total = float(can.total_receipts)
            if total == 0.0:
                print "Oops:", pol_id, total, can.total_receipts, can.total_disbursements, can.contrib_from_candidate, can.total_indiv_contrib, can.contrib_from_other_pc
                continue
            db.update('politician', where='id = $pol_id', vars=locals(),
              money_raised = can.total_receipts,
              pct_spent = can.total_disbursements/total,
              pct_self = can.contrib_from_candidate/total,
              pct_indiv = can.total_indiv_contrib/total,
              pct_pac = can.contrib_from_other_pc/total
            )

def load_fec_committees():
    db.delete('contribution', '1=1')
    db.delete('committee', '1=1')
    for f in fec_cobol.parse_committees(reverse=True):
        f = web.storage(f)
        try:
            db.insert('committee', seqname=False,
              id = f.committee_id,
              name = f.committee_name,
              treasurer = f.treasurer_name,
              street1 = f.street_one,
              street2 = f.street_two,
              city = f.city,
              state = f.state,
              zip = f.zip,
              connected_org_name = f.connected_org_name,
              candidate_id = f.candidate_id
            )
        except psycopg2.IntegrityError:
            pass # already imported

def load_fec_contributions():
    t = db.transaction(); n = 0
    db.delete('contribution', '1=1')
    for f in fec_cobol.parse_contributions():
        f = web.storage(f)
        f.occupation = f.occupation.replace('N/A', '')
        if '/' in f.occupation:
            employer, occupation = f.occupation.split('/', 1)
        else:
            employer = ''
            occupation = f.occupation
        
        try:
            datetime.date(*[int(x) for x in f.date.split('-')])
        except ValueError:
            f.date = None
        
        db.insert('contribution',
          fec_record_id = f.get('fec_record_id'),
          microfilm_loc = f.microfilm_loc,
          recipient_id = f.filer_id,
          name = f.name,
          street = f.get('street'),
          city = f.city,
          state = f.state,
          zip = f.zip,
          occupation = occupation,
          employer = employer,
          employer_stem = tools.stemcorpname(employer),
          committee = f.from_id or None,
          sent = f.date,
          amount = f.amount
        )
        n += 1
        if n % 10000 == 0: t.commit(); t = db.transaction(); print n
    t.commit()


def load_fec_efilings(filepattern=fec_crude_csv.DEFAULT_EFILINGS_FILEPATTERN):
    for f, schedules in fec_crude_csv.parse_efilings(glob.glob(filepattern)):
        for s in schedules:
            if s.get('type') == 'contribution':
                # XXX all this code for politician_id is currently
                # dead, does nothing useful
                politician_id = None
                if f.get('candidate_fec_id'):
                    fec_id = f['candidate_fec_id']
                    pol_fec_id = list(db.select('politician_fec_ids',
                                                where='fec_id=$fec_id',
                                                vars=locals()))
                    if pol_fec_id and len(pol_fec_id) == 1:
{'fraction_non_alphanumeric': 0.10283896807633408, 'fraction_numerical': 0.006361173283072211, 'mean_word_length': 1.7808712741565673, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 8, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3696965', 'n_tokens_mistral': 2763, 'n_tokens_neox': 2659, 'n_words': 524}