<commit_before>"""Pull in what's available for HFMETAR MADIS data

Run from RUN_10MIN.sh
Run from RUN_40_AFTER.sh for two hours ago
"""
import psycopg2
import os
import sys
import pytz
import netCDF4
import re
import datetime
from pyiem.datatypes import temperature, distance, pressure, speed
from pyiem.observation import Observation
import numpy as np
from metar import Metar
import warnings
warnings.simplefilter('ignore', RuntimeWarning)

LOC2TZ = {}
TIMEZONES = {None: pytz.timezone('UTC')}

pgconn = psycopg2.connect(dbname='iem', host='iemdb', user='nobody')
txn = pgconn.cursor()
txn.execute("""SELECT id, network, tzname from stations
    where network ~* 'ASOS' or network = 'AWOS' or network = 'WTM'
    """)
news = 0
for row in txn:
    LOC2TZ[row['id']] = row['tzname']
    if row['tzname'] not in TIMEZONES:
        try:
            TIMEZONES[row['tzname']] = pytz.timezone(row['tzname'])
        except:
            TIMEZONES[row['tzname']] = pytz.timezone("UTC")


def vsbyfmt(val):
    """ Tricky formatting of vis"""
    if val == 0:
        return 0
    if val <= 0.125:
        return "1/8"
    if val <= 0.25:
        return "1/4"
    if val <= 0.375:
        return "3/8"
    if val <= 0.5:
        return "1/2"
    if val <= 1.1:
        return "1"
    if val <= 1.6:
        return "1 1/2"
    if val <= 2.1:
        return "2"
    if val <= 2.6:
        return "2 1/2"
    return int(val)


def tostring(val):
    return re.sub('\x00', '', val.tostring()).strip()<commit_msg>fix cursor with last commit<commit_after>"""Pull in what's available for HFMETAR MADIS data

Run from RUN_10MIN.sh
Run from RUN_40_AFTER.sh for two hours ago
"""
import psycopg2
import os
import sys
import pytz
import netCDF4
import re
import datetime
from pyiem.datatypes import temperature, distance, pressure, speed
from pyiem.observation import Observation
import numpy as np
from metar import Metar
import warnings
warnings.simplefilter('ignore', RuntimeWarning)

LOC2TZ = {}
TIMEZONES = {None: pytz.timezone('UTC')}

pgconn = psycopg2.connect(dbname='iem', host='iemdb', user='nobody')
txn = pgconn.cursor()
txn.execute("""SELECT id, tzname from stations
    where network ~* 'ASOS' or network = 'AWOS' or network = 'WTM'
    """)
news = 0
for row in txn:
    LOC2TZ[row[0]] = row[1]
    if row[1] not in TIMEZONES:
        try:
            TIMEZONES[row[1]] = pytz.timezone(row[1])
        except:
            TIMEZONES[row[1]] = pytz.timezone("UTC")


def vsbyfmt(val):
    """ Tricky formatting of vis"""
    if val == 0:
        return 0
    if val <= 0.125:
        return "1/8"
    if val <= 0.25:
        return "1/4"
    if val <= 0.375:
        return "3/8"
    if val <= 0.5:
        return "1/2"
    if val <= 1.1:
        return "1"
    if val <= 1.6:
        return "1 1/2"
    if val <= 2.1:
        return "2"
    if val <= 2.6:
        return "2 1/2"
    return int(val)


def tostring(val):
    return re.sub('\x00', '', val.tostring()).strip()
{'fraction_non_alphanumeric': 0.13582743511964948, 'fraction_numerical': 0.03640040444893832, 'mean_word_length': 2.738035264483627, 'pattern_counts': {'":': 0, '<': 19, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5205659', 'n_tokens_mistral': 1131, 'n_tokens_neox': 1068, 'n_words': 320}