<commit_before>"""
 Extract MADIS METAR QC information to the database
"""
import os
import re
import sys
import psycopg2
import netCDF4
import numpy as np
import datetime
import pytz
from pyiem.datatypes import temperature

IEM = psycopg2.connect(database='iem', host='iemdb')
icursor = IEM.cursor()

now = datetime.datetime.utcnow() - datetime.timedelta(hours=3)

fn = "/mesonet/data/madis/metar/%s.nc" % (now.strftime("%Y%m%d_%H00"), )
table = "current_qc"

if not os.path.isfile(fn):
    sys.exit()

nc = netCDF4.Dataset(fn)

ids = nc.variables["stationName"]
nc_tmpk = nc.variables["temperature"]
nc_dwpk = nc.variables["dewpoint"]
nc_alti = nc.variables["altimeter"]
tmpkQCD = nc.variables["temperatureQCD"]
dwpkQCD = nc.variables["dewpointQCD"]
altiQCD = nc.variables["altimeterQCD"]


def figure(val, qcval):
    if qcval > 1000:
        return 'Null'
    tmpf = temperature(val, 'K').value("F")
    qcval = temperature(val + qcval, 'K').value("F")
    return qcval - tmpf


def figureAlti(val, qcval):
    if (qcval > 100000.):
        return 'Null'
    return qcval / 100.0


def check(val):
    if val > 200000.:
        return 'Null'
    return val

found = 0
for j in range(ids.shape[0]):
    sid = ids[j]
    sid = re.sub('\x00', '', sid.tostring())
    if len(sid) < 4:
        continue
    if sid[0] == "K":
        ts = datetime.datetime(1971, 1, 1) + datetime.timedelta(
                                seconds=nc.variables["timeObs"][j])
        ts = ts.replace(tzinfo=pytz.timezone("UTC"))
        (tmpf, tmpf_qc_av, tmpf_qc_sc) = ('Null', 'Null', 'Null')
        (dwpf, dwpf_qc_av, dwpf_qc_sc) = ('Null', 'Null', 'Null')
        (alti, alti_qc_av, alti_qc_sc) = ('Null', 'Null', 'Null')
        if (not np.ma.is_masked(nc_tmpk[j]) and
                not np.ma.is_masked(tmpkQCD[j, 0]) and
                not np.ma.is_masked(tmpkQCD[j, 6])):
            tmpf = check(temperature(nc_tmpk[j], 'K').value('F'))
            tmpf_qc_av = figure(nc_tmpk[j], tmpkQCD[j, 0])
            tmpf_qc_sc = figure(nc_tmpk[j], tmpkQCD[j, 6])
        if (not np.ma.is_masked(nc_dwpk[j]) and
                not np.ma.is_masked(dwpkQCD[j, 0]) and<commit_msg>fix unixtime issue with secs conversion<commit_after>"""
 Extract MADIS METAR QC information to the database
"""
import os
import re
import sys
import psycopg2
import netCDF4
import numpy as np
import datetime
import pytz
from pyiem.datatypes import temperature

IEM = psycopg2.connect(database='iem', host='iemdb')
icursor = IEM.cursor()

now = datetime.datetime.utcnow() - datetime.timedelta(hours=3)

fn = "/mesonet/data/madis/metar/%s.nc" % (now.strftime("%Y%m%d_%H00"), )
table = "current_qc"

if not os.path.isfile(fn):
    sys.exit()

nc = netCDF4.Dataset(fn)

ids = nc.variables["stationName"]
nc_tmpk = nc.variables["temperature"]
nc_dwpk = nc.variables["dewpoint"]
nc_alti = nc.variables["altimeter"]
tmpkQCD = nc.variables["temperatureQCD"]
dwpkQCD = nc.variables["dewpointQCD"]
altiQCD = nc.variables["altimeterQCD"]


def figure(val, qcval):
    if qcval > 1000:
        return 'Null'
    tmpf = temperature(val, 'K').value("F")
    qcval = temperature(val + qcval, 'K').value("F")
    return qcval - tmpf


def figureAlti(val, qcval):
    if (qcval > 100000.):
        return 'Null'
    return qcval / 100.0


def check(val):
    if val > 200000.:
        return 'Null'
    return val

found = 0
for j in range(ids.shape[0]):
    sid = ids[j]
    sid = re.sub('\x00', '', sid.tostring())
    if len(sid) < 4:
        continue
    if sid[0] == "K":
        ts = datetime.datetime(1970, 1, 1) + datetime.timedelta(
                                seconds=nc.variables["timeObs"][j])
        ts = ts.replace(tzinfo=pytz.timezone("UTC"))
        (tmpf, tmpf_qc_av, tmpf_qc_sc) = ('Null', 'Null', 'Null')
        (dwpf, dwpf_qc_av, dwpf_qc_sc) = ('Null', 'Null', 'Null')
        (alti, alti_qc_av, alti_qc_sc) = ('Null', 'Null', 'Null')
        if (not np.ma.is_masked(nc_tmpk[j]) and
                not np.ma.is_masked(tmpkQCD[j, 0]) and
                not np.ma.is_masked(tmpkQCD[j, 6])):
            tmpf = check(temperature(nc_tmpk[j], 'K').value('F'))
            tmpf_qc_av = figure(nc_tmpk[j], tmpkQCD[j, 0])
            tmpf_qc_sc = figure(nc_tmpk[j], tmpkQCD[j, 6])
        if (not np.ma.is_masked(nc_dwpk[j]) and
                not np.ma.is_masked(dwpkQCD[j, 0]) and
{'fraction_non_alphanumeric': 0.15908045977011495, 'fraction_numerical': 0.020229885057471263, 'mean_word_length': 3.028703703703704, 'pattern_counts': {'":': 2, '<': 5, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1202351', 'n_tokens_mistral': 1814, 'n_tokens_neox': 1698, 'n_words': 356}