<commit_before>"""Compute the archive start time of a HADS/DCP network"""
from pyiem.network import Table as NetworkTable
import sys
import psycopg2
import datetime

THISYEAR = datetime.datetime.now().year
HADSDB = psycopg2.connect(database='hads', host='iemdb')
MESOSITEDB = psycopg2.connect(database='mesosite', host='iemdb')


def do(network, sid):
    cursor = HADSDB.cursor()
    running = None
    # We work backwards
    for yr in range(THISYEAR, 2001, -1):
        cursor.execute("""
            SELECT min(valid) from raw""" + str(yr) + """
            WHERE station = %s
        """, (sid,))
        minv = cursor.fetchone()[0]
        if minv is None:
            return running
        running = minv
    return running


def main(argv):
    """Go main Go"""
    network = argv[1]
    nt = NetworkTable(network)
    for sid in nt.sts.keys():
        sts = do(network, sid)
        if sts is None:
            continue
        if (nt.sts[sid]['archive_begin'] is None or
                nt.sts[sid]['archive_begin'] != sts):
            osts = nt.sts[sid]['archive_begin']
            f = "%Y-%m-%d %H:%M"
            print(("%s [%s] new sts: %s OLD sts: %s"
                   ) % (sid, network, sts.strftime(f),
                        osts.strftime(f) if osts is not None else 'null'))
            cursor = MESOSITEDB.cursor()
            cursor.execute("""UPDATE stations SET archive_begin = %s
            WHERE id = %s and network = %s""", (sts, sid, network))
            cursor.close()
            MESOSITEDB.commit()

if __name__ == '__main__':
    main(sys.argv)
<commit_msg>compute HADS archive begin differently/better<commit_after>"""Compute the archive start time of a HADS/DCP network"""
from pyiem.network import Table as NetworkTable
import sys
import psycopg2
import datetime

THISYEAR = datetime.datetime.now().year
HADSDB = psycopg2.connect(database='hads', host='iemdb')
MESOSITEDB = psycopg2.connect(database='mesosite', host='iemdb')


def do(network, sid):
    cursor = HADSDB.cursor()
    for yr in range(2002, THISYEAR + 1):
        cursor.execute("""
            SELECT min(valid) from raw""" + str(yr) + """
            WHERE station = %s
        """, (sid,))
        minv = cursor.fetchone()[0]
        if minv is not None:
            return minv


def main(argv):
    """Go main Go"""
    network = argv[1]
    nt = NetworkTable(network)
    for sid in nt.sts.keys():
        sts = do(network, sid)
        if sts is None:
            continue
        if (nt.sts[sid]['archive_begin'] is None or
                nt.sts[sid]['archive_begin'] != sts):
            osts = nt.sts[sid]['archive_begin']
            f = "%Y-%m-%d %H:%M"
            print(("%s [%s] new sts: %s OLD sts: %s"
                   ) % (sid, network, sts.strftime(f),
                        osts.strftime(f) if osts is not None else 'null'))
            cursor = MESOSITEDB.cursor()
            cursor.execute("""UPDATE stations SET archive_begin = %s
            WHERE id = %s and network = %s""", (sts, sid, network))
            cursor.close()
            MESOSITEDB.commit()

if __name__ == '__main__':
    main(sys.argv)

{'fraction_non_alphanumeric': 0.13853503184713375, 'fraction_numerical': 0.006369426751592357, 'mean_word_length': 2.1888324873096447, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4872745', 'n_tokens_mistral': 1037, 'n_tokens_neox': 985, 'n_words': 278}