<commit_before>"""Clean up some tables that contain bloaty NWS Text Data

called from RUN_2AM.sh
"""

from pyiem.util import get_dbconn, logger

LOG = logger()


def main():
    """Clean AFOS and friends"""
    pgconn = get_dbconn("afos")
    acursor = pgconn.cursor()

    # reflect changes to docs/datasets/afos.md
    acursor.execute(
        "delete from products WHERE "
        "entered < ('YESTERDAY'::date - '7 days'::interval) and "
        "entered > ('YESTERDAY'::date - '31 days'::interval) and "
        "(pil ~* '^(RR[1-9RSMAZ]|ECM|ECS|ECX|LAV|LEV|MAV|MET|MTR|MEX|NBE|"
        "NBH|NBP|NBS|NBX|STO|HML|WRK|OSO|SCV)' "
        "or pil in ('HPTNCF', 'WTSNCF','WRKTTU','TSTNCF', 'HD3RSA', "
        "'XF03DY', 'XOBUS', 'ECMNC1', 'SYNBOU', 'MISWTM', 'MISWTX', "
        "'MISMA1', 'MISAM1'))"<commit_msg>RWR can be culled, oops<commit_after>"""Clean up some tables that contain bloaty NWS Text Data

called from RUN_2AM.sh
"""

from pyiem.util import get_dbconn, logger

LOG = logger()


def main():
    """Clean AFOS and friends"""
    pgconn = get_dbconn("afos")
    acursor = pgconn.cursor()

    # reflect changes to docs/datasets/afos.md
    acursor.execute(
        "delete from products WHERE "
        "entered < ('YESTERDAY'::date - '7 days'::interval) and "
        "entered > ('YESTERDAY'::date - '31 days'::interval) and "
        "(pil ~* '^(RR[1-9RSMAZ]|ECM|ECS|ECX|LAV|LEV|MAV|MET|MTR|MEX|NBE|"
        "NBH|NBP|NBS|NBX|RWR|STO|HML|WRK|OSO|SCV)' "
        "or pil in ('HPTNCF', 'WTSNCF','WRKTTU','TSTNCF', 'HD3RSA', "
        "'XF03DY', 'XOBUS', 'ECMNC1', 'SYNBOU', 'MISWTM', 'MISWTX', "
        "'MISMA1', 'MISAM1'))"
{'fraction_non_alphanumeric': 0.18369829683698297, 'fraction_numerical': 0.014598540145985401, 'mean_word_length': 3.482288828337875, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '491826', 'n_tokens_mistral': 705, 'n_tokens_neox': 673, 'n_words': 135}