<commit_before># Send products from AFOS database to pyWWA

import psycopg2
import os
AFOS = psycopg2.connect(database='afos', host='iemdb', port=5555, user='nobody')
acursor = AFOS.cursor('streamer')


acursor.execute("""
    SELECT data, entered at time zone 'UTC', source, pil from products
    WHERE source = 'KOKX'""")
for i, row in enumerate(acursor):
    mydir = "KOKX/%s" % (row[3],)
    if not os.path.isdir(mydir):
        os.makedirs(mydir)
    o = open('%s/%s_%s_%s.txt' % (mydir, row[1].strftime("%Y%m%d%H%M"), row[2], row[3]),
             'a')
    o.write(row[0])
    o.write('\r\r\n\003')
    o.close()<commit_msg>dump metadata in util script<commit_after># Send products from AFOS database to pyWWA

import psycopg2
import os
AFOS = psycopg2.connect(database='afos', host='iemdb', port=5555,
                        user='nobody')
acursor = AFOS.cursor('streamer')

o = open('KUNR.txt', 'w')

acursor.execute("""
    SELECT pil, entered at time zone 'UTC', source, pil from products
    WHERE source = 'KUNR'""")
for i, row in enumerate(acursor):
    uri = "http://mesonet.agron.iastate.edu/wx/afos/p.php?pil=%s&e=%s" % (
                    row[0], row[1].strftime("%Y%m%d%H%M"))
    o.write("%s,%s,%s,%s\n" % (row[1].strftime("%Y-%m-%d %H:%M"),
                               row[1].hour, row[0], uri))
    # skip below
    continue
    mydir = "KOKX/%s" % (row[3],)
    if not os.path.isdir(mydir):
        os.makedirs(mydir)
    o = open('%s/%s_%s_%s.txt' % (mydir, row[1].strftime("%Y%m%d%H%M"),
                                  row[2], row[3]),
             'a')
    o.write(row[0])
    o.write('\r\r\n\003')
    o.close()

o.close()
{'fraction_non_alphanumeric': 0.19855159927579963, 'fraction_numerical': 0.019915509957754977, 'mean_word_length': 2.751131221719457, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1506679', 'n_tokens_mistral': 671, 'n_tokens_neox': 645, 'n_words': 132}