<commit_before>"""
Process the ENV output to the database, for a given date
"""
import sys
import datetime
import glob
import os
import psycopg2
import numpy as np
idep = psycopg2.connect(database='idep', host='iemdb')
icursor = idep.cursor()

IDEPHOME = "/i"
SCENARIO = int(sys.argv[1])

TODAY = datetime.date.today()

def do(date, process_all):
    """ Process for this date, if process_all is true, then do it all!"""
    if process_all:
        print("Deleting all results_by_huc12 for scenario: %s" % (SCENARIO,))
        icursor.execute("""DELETE from results_by_huc12 WHERE 
                            scenario = %s""", (SCENARIO,))
    else:
        icursor.execute("""DELETE from results_by_huc12 WHERE 
                            valid = %s and scenario = %s""", (date, SCENARIO))
        if icursor.rowcount != 0:
            print '... env2database.py removed %s rows for date: %s' % (
                                                    icursor.rowcount, date)
    # Compute dictionary of slope lengths
    lengths = {}
    icursor.execute("""
    SELECT huc_12, fpath, ST_Length(geom) from flowpaths where
    scenario = %s
    """, (SCENARIO,))
    for row in icursor:
        lengths["%s_%s" % (row[0], row[1])] = row[2]
    
    
    os.chdir("/i/%s/env" % (SCENARIO,))
    hits = 0
    count = 0
    for huc8 in glob.glob("*"):
        os.chdir(huc8)
        for huc4 in glob.glob("*"):
            count += 1
            os.chdir(huc4)
            huc12 = huc8+huc4
            runs = 0
            data = {}
            for fn in glob.glob("*.env"):
                runs += 1
                for line in open(fn):
                    tokens = line.strip().split()
                    if (len(tokens) < 5 or tokens[0] in ['day', '---'] 
                        or line.find('******') > -1):
                        continue
                    ts = datetime.date(2006 + int(tokens[2]), 
                                            int(tokens[1]), int(tokens[0]))
                    if not process_all and date != ts:
                        continue
                    if not data.has_key(ts):
                        data[ts] = {'runoff': [],
                                    'loss': [],
                                    'precip': [],
                                    'delivery': []}
                    data[ts]['runoff'].append( float(tokens[4]) )
                    data[ts]['loss'].append( float(tokens[6]) )
                    data[ts]['precip'].append( float(tokens[3]) )
                    data[ts]['delivery'].append( float(tokens[12]) /
                                                 lengths[fn[:-4]])
            if runs > 0:
                for ts in data.keys():
                    # Don't ingest data from the future!
                    if ts >= TODAY:
                        continue
                    avgloss = sum(data[ts]['loss']) / float(runs)
                    avgprecip = np.average(data[ts]['precip'])
                    avgrunoff = sum(data[ts]['runoff']) / float(runs)
                    avgdelivery = sum(data[ts]['delivery']) / float(runs)
                    icursor.execute("""
                    INSERT into results_by_huc12(huc_12, valid, 
                    min_precip, avg_precip, max_precip,
                    min_loss, avg_loss, max_loss,
                    min_delivery, avg_delivery, max_delivery,
                    min_runoff, avg_runoff, max_runoff, scenario) VALUES
                    (%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s)
                    """, (huc12, ts, min(data[ts]['precip']), 
                          avgprecip, max(data[ts]['precip']),
                    min(data[ts]['loss']), avgloss, max(data[ts]['loss']), 
                    min(data[ts]['delivery']), avgdelivery, max(data[ts]['delivery']), 
                          min(data[ts]['runoff']), 
                          avgrunoff, max(data[ts]['runoff']), SCENARIO))
                    hits += 1
            os.chdir("..")
        os.chdir("..")
    
    print '... env2database.py %s produced %s/%s huc12s with erosion' % (
                                            date, hits, count)
 
def update_properties(date):
    """ Update database properties to let us know we ran this date! """
    icursor.execute("""UPDATE properties SET value = %s 
    WHERE key = 'last_date'""", (date,))
    if icursor.rowcount == 0:
        icursor.execute("""INSERT into properties(key, value) VALUES
        (%s, %s)""", ('last_date', date))
 
def main():
    """ go main go """
    if len(sys.argv) == 5:
        ts = datetime.date( int(sys.argv[2]), int(sys.argv[3]), 
                             int(sys.argv[4]))
    else:
        ts = datetime.date.today() - datetime.timedelta(days=1)

    do(ts, (len(sys.argv) == 3 and sys.argv[2] == 'all'))
    if SCENARIO == 0:
        update_properties(ts)
        
 
if __name__ == '__main__':
    # Do something
    main()
    icursor.close()<commit_msg>don't set last date when we are manually run, refs #3<commit_after>"""
Process the ENV output to the database, for a given date
"""
import sys
import datetime
import glob
import os
import psycopg2
import numpy as np
idep = psycopg2.connect(database='idep', host='iemdb')
icursor = idep.cursor()

IDEPHOME = "/i"
SCENARIO = int(sys.argv[1])

TODAY = datetime.date.today()

def do(date, process_all):
    """ Process for this date, if process_all is true, then do it all!"""
    if process_all:
        print("Deleting all results_by_huc12 for scenario: %s" % (SCENARIO,))
        icursor.execute("""DELETE from results_by_huc12 WHERE 
                            scenario = %s""", (SCENARIO,))
    else:
        icursor.execute("""DELETE from results_by_huc12 WHERE 
                            valid = %s and scenario = %s""", (date, SCENARIO))
        if icursor.rowcount != 0:
            print '... env2database.py removed %s rows for date: %s' % (
                                                    icursor.rowcount, date)
    # Compute dictionary of slope lengths
    lengths = {}
    icursor.execute("""
    SELECT huc_12, fpath, ST_Length(geom) from flowpaths where
    scenario = %s
    """, (SCENARIO,))
    for row in icursor:
        lengths["%s_%s" % (row[0], row[1])] = row[2]
    
    
    os.chdir("/i/%s/env" % (SCENARIO,))
    hits = 0
    count = 0
    for huc8 in glob.glob("*"):
        os.chdir(huc8)
        for huc4 in glob.glob("*"):
            count += 1
            os.chdir(huc4)
            huc12 = huc8+huc4
            runs = 0
            data = {}
            for fn in glob.glob("*.env"):
                runs += 1
                for line in open(fn):
                    tokens = line.strip().split()
                    if (len(tokens) < 5 or tokens[0] in ['day', '---'] 
                        or line.find('******') > -1):
                        continue
                    ts = datetime.date(2006 + int(tokens[2]), 
                                            int(tokens[1]), int(tokens[0]))
                    if not process_all and date != ts:
                        continue
                    if not data.has_key(ts):
                        data[ts] = {'runoff': [],
                                    'loss': [],
                                    'precip': [],
                                    'delivery': []}
                    data[ts]['runoff'].append( float(tokens[4]) )
                    data[ts]['loss'].append( float(tokens[6]) )
                    data[ts]['precip'].append( float(tokens[3]) )
                    data[ts]['delivery'].append( float(tokens[12]) /
                                                 lengths[fn[:-4]])
            if runs > 0:
                for ts in data.keys():
                    # Don't ingest data from the future!
                    if ts >= TODAY:
                        continue
                    avgloss = sum(data[ts]['loss']) / float(runs)
                    avgprecip = np.average(data[ts]['precip'])
                    avgrunoff = sum(data[ts]['runoff']) / float(runs)
                    avgdelivery = sum(data[ts]['delivery']) / float(runs)
                    icursor.execute("""
                    INSERT into results_by_huc12(huc_12, valid, 
                    min_precip, avg_precip, max_precip,
                    min_loss, avg_loss, max_loss,
                    min_delivery, avg_delivery, max_delivery,
                    min_runoff, avg_runoff, max_runoff, scenario) VALUES
                    (%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s)
                    """, (huc12, ts, min(data[ts]['precip']), 
                          avgprecip, max(data[ts]['precip']),
                    min(data[ts]['loss']), avgloss, max(data[ts]['loss']), 
                    min(data[ts]['delivery']), avgdelivery, max(data[ts]['delivery']), 
                          min(data[ts]['runoff']), 
                          avgrunoff, max(data[ts]['runoff']), SCENARIO))
                    hits += 1
            os.chdir("..")
        os.chdir("..")
    
    print '... env2database.py %s produced %s/%s huc12s with erosion' % (
                                            date, hits, count)
 
def update_properties(date):
    """ Update database properties to let us know we ran this date! """
    icursor.execute("""UPDATE properties SET value = %s 
    WHERE key = 'last_date'""", (date,))
    if icursor.rowcount == 0:
        icursor.execute("""INSERT into properties(key, value) VALUES
        (%s, %s)""", ('last_date', date))
 
def main():
    """ go main go """
    if len(sys.argv) == 5:
        ts = datetime.date( int(sys.argv[2]), int(sys.argv[3]), 
                             int(sys.argv[4]))
    else:
        ts = datetime.date.today() - datetime.timedelta(days=1)

    do(ts, (len(sys.argv) == 3 and sys.argv[2] == 'all'))
    # Only update the last processing date when this script is run without
    # arguments, which is a realtime run
    if SCENARIO == 0 and len(sys.argv) == 2:
        update_properties(ts)
        
 
if __name__ == '__main__':
    # Do something
    main()
    icursor.close()
{'fraction_non_alphanumeric': 0.14438396833250866, 'fraction_numerical': 0.013062840178129639, 'mean_word_length': 1.4977755808205635, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 22, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2330322', 'n_tokens_mistral': 3156, 'n_tokens_neox': 2949, 'n_words': 708}