<commit_before>"""
 We need to look at the raw SNET datafiles and see what the daily precip should
 have been.  Due to complex issues with clocks, we should see the largest
 value in the raw data file and use that for the daily precip value stored in
 the summary database

 Runs from: RUN_MIDNIGHT.sh

"""
import psycopg2
import mx.DateTime
import sys
import os
from pyiem.network import Table as NetworkTable
nt = NetworkTable(["KCCI", "KIMT", "KELO"])

IEM = psycopg2.connect(database='iem', host='iemdb')
icursor = IEM.cursor()


def process(ts):
    for nwsli in nt.sts.keys():
        nwnid = nt.sts[nwsli]['nwn_id']
        #
        fp = "/mesonet/ARCHIVE/raw/snet/%s/%s.dat" % (ts.strftime("%Y_%m/%d"),
                                                      nwnid)
        if not os.path.isfile(fp):
            # print 'Missing: %s' % (fp,)
            continue

        maxPrecip = -99
        lines = open(fp).readlines()
        for line in lines[-10:]:
            tokens = line.split(",")
            pDay = float(tokens[9][:-2])
            if (pDay > maxPrecip):
                maxPrecip = pDay

        if maxPrecip < 0:
            continue

        sql = """UPDATE summary_%s s SET pday = %s FROM stations t
            WHERE t.id = '%s' and t.iemid = s.iemid
            and day = '%s' """ % (ts.year, maxPrecip, nwsli,
                                  ts.strftime("%Y-%m-%d"))
        icursor.execute(sql)

ts = mx.DateTime.now() - mx.DateTime.RelativeDateTime(days=1)
if (len(sys.argv) == 4):
    ts = mx.DateTime.DateTime(int(sys.argv[1]), int(sys.argv[2]),
                              int(sys.argv[3]))
process(ts)
"""
sts = mx.DateTime.DateTime(2004,1,1)
ets = mx.DateTime.DateTime(2005,7,25)
interval = mx.DateTime.RelativeDateTime(days=1)
now = sts
while now < ets:
    print now
    process(now)
    now += interval
"""
icursor.close()
IEM.commit()<commit_msg>look at more data for SchoolNet Precip QC check<commit_after>"""
 We need to look at the raw SNET datafiles and see what the daily precip should
 have been.  Due to complex issues with clocks, we should see the largest
 value in the raw data file and use that for the daily precip value stored in
 the summary database

 Runs from: RUN_MIDNIGHT.sh

"""
import psycopg2
import datetime
import sys
import os
from pyiem.network import Table as NetworkTable
nt = NetworkTable(["KCCI", "KIMT", "KELO"])

IEM = psycopg2.connect(database='iem', host='iemdb')
icursor = IEM.cursor()


def process(ts):
    for nwsli in nt.sts.keys():
        nwnid = nt.sts[nwsli]['nwn_id']
        #
        fp = "/mesonet/ARCHIVE/raw/snet/%s/%s.dat" % (ts.strftime("%Y_%m/%d"),
                                                      nwnid)
        if not os.path.isfile(fp):
            # print 'Missing: %s' % (fp,)
            continue

        maxPrecip = -99
        lines = open(fp).readlines()
        for line in lines[-100:]:
            tokens = line.split(",")
            pDay = float(tokens[9][:-2])
            if (pDay > maxPrecip):
                maxPrecip = pDay

        if maxPrecip < 0:
            continue

        sql = """UPDATE summary_%s s SET pday = %s FROM stations t
            WHERE t.id = '%s' and t.iemid = s.iemid
            and day = '%s' """ % (ts.year, maxPrecip, nwsli,
                                  ts.strftime("%Y-%m-%d"))
        icursor.execute(sql)

ts = datetime.datetime.now() - datetime.timedelta(days=1)
if (len(sys.argv) == 4):
    ts = datetime.datetime(int(sys.argv[1]), int(sys.argv[2]),
                           int(sys.argv[3]))
process(ts)
"""
sts = mx.DateTime.DateTime(2004,1,1)
ets = mx.DateTime.DateTime(2005,7,25)
interval = mx.DateTime.RelativeDateTime(days=1)
now = sts
while now < ets:
    print now
    process(now)
    now += interval
"""
icursor.close()
IEM.commit()
{'fraction_non_alphanumeric': 0.13531266421439833, 'fraction_numerical': 0.014976353126642144, 'mean_word_length': 2.159336099585062, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 6, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4772471', 'n_tokens_mistral': 1280, 'n_tokens_neox': 1229, 'n_words': 345}