<commit_before># django independent entry process; no db abstraction; this is built for postgresql

import psycopg2, sys, time

sys.path.append('/projects/realtimefec/src/realtimefec/fecreader/fecreader')
sys.path.append('/projects/realtimefec/src/realtimefec/fecreader/')

from parsing.filing import filing
from parsing.form_parser import form_parser, ParserMissingError
from local_settings import DATABASES
from form_mappers import *

from write_csv_to_db import CSV_dumper

from fec_import_logging import fec_logger
from hstore_helpers import dict_to_hstore


class FilingHeaderDoesNotExist(Exception):
    pass
    
class FilingHeaderAlreadyProcessed(Exception):
    pass
    
def get_connection():
    dbname = DATABASES['default']['NAME']
    pw = DATABASES['default']['PASSWORD']
    user = DATABASES['default']['USER']
    host = DATABASES['default']['HOST']
    port = DATABASES['default']['PORT']
    if not host:
        host = 'localhost'
    if not port:
        port = 5432
        
    conn_string = "host='%s' dbname='%s' user='%s' password='%s' port='%s'" % (host, dbname, user, pw, port)
    conn = psycopg2.connect(conn_string)
    return conn


def process_body_row(linedict, filingnum, header_id, is_amended, cd, filer_id):
    form = linedict['form_parser']
    #print "processing form type: %s" % (form)
    if form=='SchA':
        skeda_from_skedadict(linedict, filingnum, header_id, is_amended, cd)

    elif form=='SchB':
        skedb_from_skedbdict(linedict, filingnum, header_id, is_amended, cd)                        

    elif form=='SchE':
        skede_from_skededict(linedict, filingnum, header_id, is_amended, cd)

    # Treat 48-hour contribution notices like sked A.
    # Requires special handling for amendment, since these are superceded
    # by regular F3 forms. 
    elif form=='F65':
        skeda_from_f65(linedict, filingnum, header_id, is_amended, cd)

    # disclosed donor to non-commmittee. Sorta rare, but.. 
    elif form=='F56':
        skeda_from_f56(linedict, filingnum, header_id, is_amended, cd)

    # disclosed electioneering donor
    elif form=='F92':
        skeda_from_f92(linedict, filingnum, header_id, is_amended, cd)   

    # inaugural donors
    elif form=='F132':
        skeda_from_f132(linedict, filingnum, header_id, is_amended, cd)                    

    #inaugural refunds
    elif form=='F133':
        skeda_from_f133(linedict, filingnum, header_id, is_amended, cd)                    

    # IE's disclosed by non-committees. Note that they use this for * both * quarterly and 24- hour notices. There's not much consistency with this--be careful with superceding stuff. 
    elif form=='F57':
        skede_from_f57(linedict, filingnum, header_id, is_amended, cd)

    # Its another kind of line. Just dump it in Other lines.
    else:
        otherline_from_line(linedict, filingnum, header_id, is_amended, cd, filer_id)



def process_filing_body(filingnum, fp=None, logger=None):
    
    
    #It's useful to pass the form parser in when running in bulk so we don't have to keep creating new ones. 
    if not fp:
      fp = form_parser()
      
    if not logger:
        logger=fec_logger()
    msg = "process_filing_body: Starting # %s" % (filingnum)
    #print msg
    logger.info(msg)
      
    connection = get_connection()
    cursor = connection.cursor()
    cmd = "select id, is_superceded from formdata_filing_header where filing_number=%s" % (filingnum)
    cursor.execute(cmd)
    
    cd = CSV_dumper(connection)
    
    result = cursor.fetchone()
    if not result:
        msg = 'process_filing_body: Couldn\'t find a filing header for filing %s' % (filingnum)
        logger.error(msg)
        raise FilingHeaderDoesNotExist(msg)
        
    # will throw a TypeError if it's missing.
    header_id = result[0]
    is_amended = result[1]
    
    cmd = "select data_is_processed from fec_alerts_new_filing where filing_number=%s" % (filingnum)
    cursor.execute(cmd)
    result = cursor.fetchone()
    is_already_processed = result[0]
    if is_already_processed:
        msg = 'process_filing_body: This filing has already been entered.'
        logger.error(msg)
        raise FilingHeaderAlreadyProcessed(msg)
    
    #print "Processing filing %s" % (filingnum)
    f1 = filing(filingnum)
    form = f1.get_form_type()
    version = f1.get_version()
    filer_id = f1.get_filer_id()
    
    # only parse forms that we're set up to read
    
    if not fp.is_allowed_form(form):
        if verbose:
            msg = "process_filing_body: Not a parseable form: %s - %s" % (form, filingnum)
            # print msg
            logger.error(msg)
        return None
        
    linenum = 0
    while True:
        linenum += 1
        row = f1.get_body_row()
        if not row:
            break
        
        #print "row is %s" % (row)
        #print "\n\n\nForm is %s" % form
        try:
            linedict = fp.parse_form_line(row, version)
            #print "\n\n\nform is %s" % form
            process_body_row(linedict, filingnum, header_id, is_amended, cd, filer_id)
        except ParserMissingError:
            msg = 'process_filing_body: Unknown line type in filing %s line %s: type=%s Skipping.' % (filingnum, linenum, row[0])
            logger.warn(msg)
            continue
        
    # commit all the leftovers
    cd.commit_all()
    cd.close()
    counter = cd.get_counter()
    total_rows = 0
    for i in counter:
        total_rows += counter[i]
        
    msg = "process_filing_body: Filing # %s Total rows: %s Tally is: %s" % (filingnum, total_rows, counter)
    # print msg
    logger.info(msg)
    
    # save the line number count -- which is counter in the above -- as an hstore in the original 
    header_data = dict_to_hstore(counter)
    cmd = "update formdata_filing_header set header_data='%s'::hstore where filing_number=%s" % (header_data, filingnum)
    cursor.execute(cmd)
    
    # mark file as having been entered. 
    cmd = "update fec_alerts_new_filing set data_is_processed = True where filing_number=%s" % (filingnum)
    cursor.execute(cmd)<commit_msg>put lines present in correct row<commit_after># django independent entry process; no db abstraction; this is built for postgresql

import psycopg2, sys, time

sys.path.append('/projects/realtimefec/src/realtimefec/fecreader/fecreader')
sys.path.append('/projects/realtimefec/src/realtimefec/fecreader/')

from parsing.filing import filing
from parsing.form_parser import form_parser, ParserMissingError
from local_settings import DATABASES
from form_mappers import *

from write_csv_to_db import CSV_dumper

from fec_import_logging import fec_logger
from hstore_helpers import dict_to_hstore


class FilingHeaderDoesNotExist(Exception):
    pass
    
class FilingHeaderAlreadyProcessed(Exception):
    pass
    
def get_connection():
    dbname = DATABASES['default']['NAME']
    pw = DATABASES['default']['PASSWORD']
    user = DATABASES['default']['USER']
    host = DATABASES['default']['HOST']
    port = DATABASES['default']['PORT']
    if not host:
        host = 'localhost'
    if not port:
        port = 5432
        
    conn_string = "host='%s' dbname='%s' user='%s' password='%s' port='%s'" % (host, dbname, user, pw, port)
    conn = psycopg2.connect(conn_string)
    return conn


def process_body_row(linedict, filingnum, header_id, is_amended, cd, filer_id):
    form = linedict['form_parser']
    #print "processing form type: %s" % (form)
    if form=='SchA':
        skeda_from_skedadict(linedict, filingnum, header_id, is_amended, cd)

    elif form=='SchB':
        skedb_from_skedbdict(linedict, filingnum, header_id, is_amended, cd)                        

    elif form=='SchE':
        skede_from_skededict(linedict, filingnum, header_id, is_amended, cd)

    # Treat 48-hour contribution notices like sked A.
    # Requires special handling for amendment, since these are superceded
    # by regular F3 forms. 
    elif form=='F65':
        skeda_from_f65(linedict, filingnum, header_id, is_amended, cd)

    # disclosed donor to non-commmittee. Sorta rare, but.. 
    elif form=='F56':
        skeda_from_f56(linedict, filingnum, header_id, is_amended, cd)

    # disclosed electioneering donor
    elif form=='F92':
        skeda_from_f92(linedict, filingnum, header_id, is_amended, cd)   

    # inaugural donors
    elif form=='F132':
        skeda_from_f132(linedict, filingnum, header_id, is_amended, cd)                    

    #inaugural refunds
    elif form=='F133':
        skeda_from_f133(linedict, filingnum, header_id, is_amended, cd)                    

    # IE's disclosed by non-committees. Note that they use this for * both * quarterly and 24- hour notices. There's not much consistency with this--be careful with superceding stuff. 
    elif form=='F57':
        skede_from_f57(linedict, filingnum, header_id, is_amended, cd)

    # Its another kind of line. Just dump it in Other lines.
    else:
        otherline_from_line(linedict, filingnum, header_id, is_amended, cd, filer_id)



def process_filing_body(filingnum, fp=None, logger=None):
    
    
    #It's useful to pass the form parser in when running in bulk so we don't have to keep creating new ones. 
    if not fp:
      fp = form_parser()
      
    if not logger:
        logger=fec_logger()
    msg = "process_filing_body: Starting # %s" % (filingnum)
    #print msg
    logger.info(msg)
      
    connection = get_connection()
    cursor = connection.cursor()
    cmd = "select id, is_superceded from formdata_filing_header where filing_number=%s" % (filingnum)
    cursor.execute(cmd)
    
    cd = CSV_dumper(connection)
    
    result = cursor.fetchone()
    if not result:
        msg = 'process_filing_body: Couldn\'t find a filing header for filing %s' % (filingnum)
        logger.error(msg)
        raise FilingHeaderDoesNotExist(msg)
        
    # will throw a TypeError if it's missing.
    header_id = result[0]
    is_amended = result[1]
    
    cmd = "select data_is_processed from fec_alerts_new_filing where filing_number=%s" % (filingnum)
    cursor.execute(cmd)
    result = cursor.fetchone()
    is_already_processed = result[0]
    if is_already_processed:
        msg = 'process_filing_body: This filing has already been entered.'
        logger.error(msg)
        raise FilingHeaderAlreadyProcessed(msg)
    
    #print "Processing filing %s" % (filingnum)
    f1 = filing(filingnum)
    form = f1.get_form_type()
    version = f1.get_version()
    filer_id = f1.get_filer_id()
    
    # only parse forms that we're set up to read
    
    if not fp.is_allowed_form(form):
        if verbose:
            msg = "process_filing_body: Not a parseable form: %s - %s" % (form, filingnum)
            # print msg
            logger.error(msg)
        return None
        
    linenum = 0
    while True:
        linenum += 1
        row = f1.get_body_row()
        if not row:
            break
        
        #print "row is %s" % (row)
        #print "\n\n\nForm is %s" % form
        try:
            linedict = fp.parse_form_line(row, version)
            #print "\n\n\nform is %s" % form
            process_body_row(linedict, filingnum, header_id, is_amended, cd, filer_id)
        except ParserMissingError:
            msg = 'process_filing_body: Unknown line type in filing %s line %s: type=%s Skipping.' % (filingnum, linenum, row[0])
            logger.warn(msg)
            continue
        
    # commit all the leftovers
    cd.commit_all()
    cd.close()
    counter = cd.get_counter()
    total_rows = 0
    for i in counter:
        total_rows += counter[i]
        
    msg = "process_filing_body: Filing # %s Total rows: %s Tally is: %s" % (filingnum, total_rows, counter)
    # print msg
    logger.info(msg)
    
    # save the line number count -- which is counter in the above -- as an hstore in the original 
    header_data = dict_to_hstore(counter)
    cmd = "update formdata_filing_header set lines_present='%s'::hstore where filing_number=%s" % (header_data, filingnum)
    cursor.execute(cmd)
    
    # mark file as having been entered. 
    cmd = "update fec_alerts_new_filing set data_is_processed = True where filing_number=%s" % (filingnum)
    cursor.execute(cmd)
{'fraction_non_alphanumeric': 0.09592952116812138, 'fraction_numerical': 0.008320417652337058, 'mean_word_length': 2.8626339004410837, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2395879', 'n_tokens_mistral': 4134, 'n_tokens_neox': 3930, 'n_words': 1074}