<commit_before>"""
Harvest the Agronomic Data into the ISU Database
"""
import util
import sys
import gdata.docs.client
import ConfigParser
import psycopg2

YEAR = sys.argv[1]

config = ConfigParser.ConfigParser()
config.read('mytokens.cfg')

pgconn = psycopg2.connect(database='sustainablecorn',
                          host=config.get('database', 'host'))
pcursor = pgconn.cursor()

# Get me a client, stat
spr_client = util.get_spreadsheet_client(config)
docs_client = util.get_docs_client(config)

query = gdata.docs.client.DocsQuery(show_collections='false', 
                                    title='Agronomic Data')
feed = docs_client.GetAllResources(query=query)

for entry in feed:
    if entry.get_resource_type() != 'spreadsheet':
        continue
    spreadsheet = util.Spreadsheet(docs_client, spr_client, entry)
    spreadsheet.get_worksheets()
    worksheet = spreadsheet.worksheets[YEAR]
    worksheet.get_cell_feed()
    siteid = spreadsheet.title.split()[0]
    for col in range(1, worksheet.cols+1):
        val = worksheet.get_cell_value(1, col)
        if val is None:
            continue
        if val.find("PlotID") == 0:
            plotidcol = col
        if val.find("AGR") != 0:
            continue
        varname = val
        for row in range(4,worksheet.rows+1):
            plotid = worksheet.get_cell_value(row, plotidcol)
            if plotid is None:
                continue
            val = worksheet.get_cell_value(row, col)
            try:
                pcursor.execute("""
                    INSERT into agronomic_data(site, plotid, varname, year, value)
                    values (%s, %s, %s, %s, %s)
                    """, (siteid, plotid, varname, YEAR, val))
            except Exception, exp:
                print 'HARVEST_AGRONOMIC TRACEBACK'
                print exp
                print '%s %s %s %s' % (siteid, plotid, varname, val)
                sys.exit()

pcursor.close()
pgconn.commit()
pgconn.close()<commit_msg>improve traceback print<commit_after>"""
Harvest the Agronomic Data into the ISU Database
"""
import util
import sys
import gdata.docs.client
import ConfigParser
import psycopg2

YEAR = sys.argv[1]

config = ConfigParser.ConfigParser()
config.read('mytokens.cfg')

pgconn = psycopg2.connect(database='sustainablecorn',
                          host=config.get('database', 'host'))
pcursor = pgconn.cursor()

# Get me a client, stat
spr_client = util.get_spreadsheet_client(config)
docs_client = util.get_docs_client(config)

query = gdata.docs.client.DocsQuery(show_collections='false', 
                                    title='Agronomic Data')
feed = docs_client.GetAllResources(query=query)

for entry in feed:
    if entry.get_resource_type() != 'spreadsheet':
        continue
    spreadsheet = util.Spreadsheet(docs_client, spr_client, entry)
    spreadsheet.get_worksheets()
    worksheet = spreadsheet.worksheets[YEAR]
    worksheet.get_cell_feed()
    siteid = spreadsheet.title.split()[0]
    for col in range(1, worksheet.cols+1):
        val = worksheet.get_cell_value(1, col)
        if val is None:
            continue
        if val.find("PlotID") == 0:
            plotidcol = col
        if val.find("AGR") != 0:
            continue
        varname = val
        for row in range(4,worksheet.rows+1):
            plotid = worksheet.get_cell_value(row, plotidcol)
            if plotid is None:
                continue
            val = worksheet.get_cell_value(row, col)
            try:
                pcursor.execute("""
                    INSERT into agronomic_data(site, plotid, varname, year, value)
                    values (%s, %s, %s, %s, %s)
                    """, (siteid, plotid, varname, YEAR, val))
            except Exception, exp:
                print 'HARVEST_AGRONOMIC TRACEBACK'
                print exp
                print '%s %s %s %s %s' % (YEAR, siteid, plotid, repr(varname), 
                                          repr(val))
                sys.exit()

pcursor.close()
pgconn.commit()
pgconn.close()
{'fraction_non_alphanumeric': 0.10770750988142293, 'fraction_numerical': 0.005434782608695652, 'mean_word_length': 2.2186009538950717, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '346092', 'n_tokens_mistral': 1271, 'n_tokens_neox': 1215, 'n_words': 289}