<commit_before>"""
<NAME> kindly provides a monthly file of his QC'd COOP observations
This script processes them into something we can insert into the IEM database
"""

import sys
import re
import datetime
import string

"""
This is not good, but necessary.  We translate some sites into others, so to
maintain a long term record.
"""
stconv = {
  'IA6199': 'IA6200', # Oelwein
  'IA3288': 'IA3290', # Glenwood
  'IA4963': 'IA8266', # Lowden becomes Tipton
  'IA7892': 'IA4049', # Stanley becomes Independence
  'IA0214': 'IA0213', # Anamosa
  'IA2041': 'IA3980', # Dakota City becomes Humboldt
}

year = int(sys.argv[1])
month = int(sys.argv[2])
fn = "/mesonet/data/harry/%s/SCIA%s%02i.txt" % (year, str(year)[2:], month)
print "Processing File: ", fn

lines = open(fn, 'r').readlines()

alldata = open('/tmp/harry.sql', 'w')
alldata.write("""BEGIN;
-- Remove anything old that may be in alldata_tmp
DELETE from alldata_tmp;
COPY alldata_tmp from STDIN with null as 'Null';
""")

hits = 0
misses = 0
for line in lines:
    tokens = re.split(",", line)
    if len(tokens) < 15 or len(tokens[2]) != 4:
        misses += 1
        continue
    if len(tokens[0]) == 0 or tokens[2] == 'YR' or tokens[0] == 'YR':
        continue
    stid = tokens[0]
    dbid = "%s%04.0f" % ("IA", int(stid))
    dbid = stconv.get(dbid, dbid)
    yr = int(tokens[2])
    mo = int(tokens[3])
    dy = int(tokens[4])
    hi = string.strip(tokens[6])
    lo = string.strip(tokens[8])
    pr = string.strip(tokens[12])
    sf = string.strip(tokens[14])
    sd = string.strip(tokens[16])
    if (pr == "T"): pr = 0.0001
    if (sf == "T"): sf = 0.0001
    if (sd == "T"): sd = 0.0001
    if (sf == "M"): sf = ""
    if (sd == "M"): sd = ""
    if (hi == "M"): hi = ""
    if (lo == "M"): lo = ""
    if (pr == "M"): pr = ""
    if (pr == "C"): pr = ""
    if (sf == "C"): sf = ""
    if (sd == "C"): sd = ""

    if (sf == ""): sf = 0
    if (sd == ""): sd = 0
    if (pr == ""): pr = 0
    if (hi == ""): hi = "Null"
    if (lo == ""): lo = "Null"

    ts = datetime.datetime(yr, mo, dy)
    day = ts.strftime("%Y-%m-%d")
    sday = ts.strftime("%m%d")
 
    alldata.write("%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\tf\tNull\tNull\tNull\n" % (
                        dbid, day,  hi, lo, pr, sf, sday, yr, mo,sd) )
    hits += 1

print '    got %s good lines %s bad lines' % (hits, misses)

alldata.write("""\.
-- Now we need to clear the old estimates away
INSERT into alldata_estimates SELECT * from alldata_ia WHERE year = %s
 and month = %s;
DELETE from alldata_ia WHERE year = %s and month = %s;
INSERT into alldata_ia SELECT * from alldata_tmp;<commit_msg>need another null column for HRRR<commit_after>"""
<NAME> kindly provides a monthly file of his QC'd COOP observations
This script processes them into something we can insert into the IEM database
"""

import sys
import re
import datetime
import string

"""
This is not good, but necessary.  We translate some sites into others, so to
maintain a long term record.
"""
stconv = {
  'IA6199': 'IA6200', # Oelwein
  'IA3288': 'IA3290', # Glenwood
  'IA4963': 'IA8266', # Lowden becomes Tipton
  'IA7892': 'IA4049', # Stanley becomes Independence
  'IA0214': 'IA0213', # Anamosa
  'IA2041': 'IA3980', # Dakota City becomes Humboldt
}

year = int(sys.argv[1])
month = int(sys.argv[2])
fn = "/mesonet/data/harry/%s/SCIA%s%02i.txt" % (year, str(year)[2:], month)
print "Processing File: ", fn

lines = open(fn, 'r').readlines()

alldata = open('/tmp/harry.sql', 'w')
alldata.write("""BEGIN;
-- Remove anything old that may be in alldata_tmp
DELETE from alldata_tmp;
COPY alldata_tmp from STDIN with null as 'Null';
""")

hits = 0
misses = 0
for line in lines:
    tokens = re.split(",", line)
    if len(tokens) < 15 or len(tokens[2]) != 4:
        misses += 1
        continue
    if len(tokens[0]) == 0 or tokens[2] == 'YR' or tokens[0] == 'YR':
        continue
    stid = tokens[0]
    dbid = "%s%04.0f" % ("IA", int(stid))
    dbid = stconv.get(dbid, dbid)
    yr = int(tokens[2])
    mo = int(tokens[3])
    dy = int(tokens[4])
    hi = string.strip(tokens[6])
    lo = string.strip(tokens[8])
    pr = string.strip(tokens[12])
    sf = string.strip(tokens[14])
    sd = string.strip(tokens[16])
    if (pr == "T"): pr = 0.0001
    if (sf == "T"): sf = 0.0001
    if (sd == "T"): sd = 0.0001
    if (sf == "M"): sf = ""
    if (sd == "M"): sd = ""
    if (hi == "M"): hi = ""
    if (lo == "M"): lo = ""
    if (pr == "M"): pr = ""
    if (pr == "C"): pr = ""
    if (sf == "C"): sf = ""
    if (sd == "C"): sd = ""

    if (sf == ""): sf = 0
    if (sd == ""): sd = 0
    if (pr == ""): pr = 0
    if (hi == ""): hi = "Null"
    if (lo == ""): lo = "Null"

    ts = datetime.datetime(yr, mo, dy)
    day = ts.strftime("%Y-%m-%d")
    sday = ts.strftime("%m%d")
 
    alldata.write("%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\tf\tNull\tNull\tNull\tNull\n" % (
                        dbid, day,  hi, lo, pr, sf, sday, yr, mo,sd) )
    hits += 1

print '    got %s good lines %s bad lines' % (hits, misses)

alldata.write("""\.
-- Now we need to clear the old estimates away
INSERT into alldata_estimates SELECT * from alldata_ia WHERE year = %s
 and month = %s;
DELETE from alldata_ia WHERE year = %s and month = %s;
INSERT into alldata_ia SELECT * from alldata_tmp;
{'fraction_non_alphanumeric': 0.18863293051359517, 'fraction_numerical': 0.03700906344410876, 'mean_word_length': 2.8805860805860806, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5313923', 'n_tokens_mistral': 2191, 'n_tokens_neox': 2064, 'n_words': 616}