<commit_before>"""Database found apache log entries of autoplot generation timing

Run from RUN_MIDNIGHT.sh
"""
import psycopg2
import datetime
import re

LOGRE = re.compile("Autoplot\[\s*(\d+)\] Timing:\s*(\d+\.\d+)s Key: ([^\s]*)")
LOGFN = '/var/log/mesonet/error_log'


def get_dbendts(cursor):
    cursor.execute("""SELECT max(valid) from autoplot_timing""")
    ts = cursor.fetchone()[0]
    if ts is None:
        ts = datetime.datetime.now() - datetime.timedelta(days=1)
    else:
        ts = ts.replace(tzinfo=None)
    return ts


def find_and_save(cursor, dbendts):
    now = datetime.datetime.now()
    thisyear = now.year
    for line in open(LOGFN):
        tokens = LOGRE.findall(line)
        if len(tokens) != 1:
            continue
        (appid, timing, uri) = tokens[0]
        valid = datetime.datetime.strptime("%s %s" % (thisyear, line[:15]),
                                           '%Y %b %d %H:%M:%S')
        if valid > now or valid <= dbendts:
            continue
        hostname = line.split()[3]
        cursor.execute("""
        INSERT into autoplot_timing (appid, valid, timing, uri, hostname)
        VALUES (%s, %s, %s, %s, %s)
        """, (appid, valid, timing, uri, hostname))


def main():
    mesosite = psycopg2.connect(database='mesosite', host='iemdb')
    cursor = mesosite.cursor()
    dbendts = get_dbendts(cursor)
    find_and_save(cursor, dbendts)
    cursor.close()
    mesosite.commit()

if __name__ == '__main__':
    main()<commit_msg>add warning to autoplot timing miner when no results found<commit_after>"""Database found apache log entries of autoplot generation timing

Run from RUN_MIDNIGHT.sh
"""
from __future__ import print_function
import datetime
import re

import psycopg2

LOGRE = re.compile(r"Autoplot\[\s*(\d+)\] Timing:\s*(\d+\.\d+)s Key: ([^\s]*)")
LOGFN = '/var/log/mesonet/error_log'


def get_dbendts(cursor):
    """Figure out when we have data until"""
    cursor.execute("""SELECT max(valid) from autoplot_timing""")
    ts = cursor.fetchone()[0]
    if ts is None:
        ts = datetime.datetime.now() - datetime.timedelta(days=1)
    else:
        ts = ts.replace(tzinfo=None)
    return ts


def find_and_save(cursor, dbendts):
    """Do work please"""
    now = datetime.datetime.now()
    thisyear = now.year
    inserts = 0
    for line in open(LOGFN):
        tokens = LOGRE.findall(line)
        if len(tokens) != 1:
            continue
        (appid, timing, uri) = tokens[0]
        valid = datetime.datetime.strptime("%s %s" % (thisyear, line[:15]),
                                           '%Y %b %d %H:%M:%S')
        if valid > now or valid <= dbendts:
            continue
        hostname = line.split()[3]
        cursor.execute("""
        INSERT into autoplot_timing (appid, valid, timing, uri, hostname)
        VALUES (%s, %s, %s, %s, %s)
        """, (appid, valid, timing, uri, hostname))
        inserts += 1
    if inserts == 0:
        print("mine_autoplot: WARNING, no entries found for databasing...")


def main():
    """Go Main Go!"""
    mesosite = psycopg2.connect(database='mesosite', host='iemdb')
    cursor = mesosite.cursor()
    dbendts = get_dbendts(cursor)
    find_and_save(cursor, dbendts)
    cursor.close()
    mesosite.commit()


if __name__ == '__main__':
    main()
{'fraction_non_alphanumeric': 0.14736203759854458, 'fraction_numerical': 0.006367495451788963, 'mean_word_length': 2.6533776301218164, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6792088', 'n_tokens_mistral': 1134, 'n_tokens_neox': 1099, 'n_words': 281}