<commit_before>"""Search for a network to generate windroses for

Called from RUN_2AM.sh script
"""
import os
import subprocess
import stat
import datetime

from pandas.io.sql import read_sql
from pyiem.util import get_dbconn, logger

LOG = logger()
CACHEDIR = "/mesonet/share/windrose/"


def do_network(network):
    """Process a network"""
    # Update the STS while we are at it, this will help the database get
    # stuff cached too
    if network.find("_ASOS") > 0:
        subprocess.call(
            "python ../dbutil/compute_asos_sts.py %s" % (network,), shell=True
        )
    if network.find("_DCP") > 0:
        # Special Logic to compute archive period.
        subprocess.call(
            ["python", "../dbutil/compute_hads_sts.py", network],
        )
        # Update stage details.
        subprocess.call(
            ["python", "../hads/process_ahps_xml.py", network],
        )
    subprocess.call(
        "python drive_network_windrose.py %s" % (network,), shell=True
    )


def main():
    """Main"""
    now = datetime.datetime.now()
    with get_dbconn("mesosite") as pgconn:
        df = read_sql(
            """SELECT max(id) as station, network from stations
            WHERE (network ~* 'ASOS' or network = 'AWOS' or network ~* 'DCP')
            and online = 't' GROUP by network ORDER by random()""",
            pgconn,
            index_col="network",
        )
    for network, row in df.iterrows():
        station = row["station"]
        testfn = "%s/%s/%s/%s_yearly.png" % (
            CACHEDIR,
            network,
            station,
            station,
        )
        if not os.path.isfile(testfn):
            LOG.info(
                "Driving network %s because no file for %s", network, station
            )
            do_network(network)
            break
        mtime = os.stat(testfn)[stat.ST_MTIME]
        age = float(now.strftime("%s")) - mtime
        # 250 days in seconds, enough to cover the number of networks
        if age < (250 * 24 * 60 * 60):
            continue
        LOG.info("Driving network %s because of age!", network)
        do_network(network)
        break


if __name__ == "__main__":
    main()
<commit_msg>drive RWIS windroses via cron too<commit_after>"""Search for a network to generate windroses for

Called from RUN_2AM.sh script
"""
import os
import subprocess
import stat
import datetime

from pandas.io.sql import read_sql
from pyiem.util import get_dbconn, logger

LOG = logger()
CACHEDIR = "/mesonet/share/windrose/"


def do_network(network):
    """Process a network"""
    # Update the STS while we are at it, this will help the database get
    # stuff cached too
    if network.find("_ASOS") > 0:
        subprocess.call(["python", "../dbutil/compute_asos_sts.py", network])
    elif network.find("_DCP") > 0:
        # Special Logic to compute archive period.
        subprocess.call(["python", "../dbutil/compute_hads_sts.py", network])
        # Update stage details.
        subprocess.call(["python", "../hads/process_ahps_xml.py", network])
    elif network.find("_RWIS") > 0:
        subprocess.call(
            ["python", "../dbutil/compute_alldata_sts.py", "rwis", network],
        )
    subprocess.call(["python", "drive_network_windrose.py", network])


def main():
    """Main"""
    now = datetime.datetime.now()
    with get_dbconn("mesosite") as pgconn:
        df = read_sql(
            """SELECT max(id) as station, network from stations
            WHERE (network ~* 'ASOS' or network = 'AWOS' or network ~* 'DCP'
            or network ~* 'RWIS')
            and online = 't' GROUP by network ORDER by random()""",
            pgconn,
            index_col="network",
        )
    for network, row in df.iterrows():
        station = row["station"]
        testfn = f"{CACHEDIR}/{network}/{station}/{station}_yearly.png"
        if not os.path.isfile(testfn):
            LOG.info(
                "Driving network %s because no file for %s", network, station
            )
            do_network(network)
            break
        mtime = os.stat(testfn)[stat.ST_MTIME]
        age = float(now.strftime("%s")) - mtime
        # 250 days in seconds, enough to cover the number of networks
        if age < (250 * 24 * 60 * 60):
            continue
        LOG.info("Driving network %s because of age!", network)
        do_network(network)
        break


if __name__ == "__main__":
    main()

{'fraction_non_alphanumeric': 0.12285456187895212, 'fraction_numerical': 0.007000903342366757, 'mean_word_length': 2.204775687409551, 'pattern_counts': {'":': 2, '<': 5, '<?xml version=': 0, '>': 8, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4997792', 'n_tokens_mistral': 1368, 'n_tokens_neox': 1323, 'n_words': 390}