<commit_before>import sys
import netCDF3
import numpy
import mx.DateTime
from pyIEM import iemdb, mesonet
import Ngl
import constants

i = iemdb.iemdb()
mesosite = i['mesosite']
coop = i['coop']
locs = {}

def load_stationtable():
    sql = """SELECT id, x(geom) as lon, y(geom) as lat from
         stations where network IN ('IACLIMATE')"""
    rs = mesosite.query( sql ).dictresult()
    for i in range(len(rs)):
        locs[ rs[i]['id'].lower() ] = rs[i]

def generic_gridder(rs, idx):
    """
    Generic gridding algorithm for easy variables
    """
    lats = []
    lons = []
    vals = []
    for i in range(len(rs)):
        if rs[i][idx] is not None and locs.has_key(rs[i]['stationid']):
            lats.append(  locs[rs[i]['stationid']]['lat'] )
            lons.append(  locs[rs[i]['stationid']]['lon'] )
            vals.append( rs[i][idx]  )
    if len(vals) < 4:
        print "Only %s observations found for %s, won't grid" % (len(vals),
               idx)
        return None
    grid = Ngl.natgrid(lons, lats, vals, constants.XAXIS, constants.YAXIS)
    if grid is not None:
        return grid.transpose()
    else:
        return None


def grid_day(nc, ts):
    """
    I proctor the gridding of data on an hourly basis
    @param ts Timestamp of the analysis, we'll consider a 20 minute window
    """
    offset = int((ts - (ts + mx.DateTime.RelativeDateTime(month=1,day=1,hour=0))).days)


    sql = """SELECT * from alldata WHERE day = '%s' and
             stationid != 'ia0000' """ % (
         ts.strftime("%Y-%m-%d"), )
    rs = coop.query( sql ).dictresult()
    if len(rs) > 4:
        res = generic_gridder(rs, 'high')
        if res is not None:
            nc.variables['high_tmpk'][offset] = constants.f2k(res)
        res = generic_gridder(rs, 'low')
        if res is not None:
            nc.variables['low_tmpk'][offset] = constants.f2k(res)
        res = generic_gridder(rs, 'precip')
        if res is not None:
            nc.variables['p01d'][offset] = res * 24.5
    else:
        print "%s has %02i entries, FAIL" % (ts.strftime("%Y-%m-%d"), 
            len(rs))

def main(ts):
    # Load up a station table we are interested in
    load_stationtable()

    # Load up our netcdf file!
    nc = netCDF3.Dataset("/mnt/mesonet/data/iemre/%s_mw_daily.nc" % (ts.year,), 'a')
    grid_day(nc , ts)

    nc.close()<commit_msg>make this MW aware and not use networkTable<commit_after>import sys
try:
    import netCDF4 as netCDF3
except:
    import netCDF3
import numpy
import mx.DateTime
from pyIEM import iemdb, mesonet
import Ngl
import constants
import network

i = iemdb.iemdb()
coop = i['coop']

nt = network.Table(('IACLIMATE', 'ILCLIMATE', 'INCLIMATE',
         'OHCLIMATE','MICLIMATE','KYCLIMATE','WICLIMATE','MNCLIMATE',
         'SDCLIMATE','NDCLIMATE','NECLIMATE','KSCLIMATE','MOCLIMATE'))
locs = nt.sts

def generic_gridder(rs, idx):
    """
    Generic gridding algorithm for easy variables
    """
    lats = []
    lons = []
    vals = []
    for i in range(len(rs)):
        stid = rs[i]['stationid'].upper()
        if rs[i][idx] is not None and locs.has_key(stid):
            lats.append(  locs[stid]['lat'] )
            lons.append(  locs[stid]['lon'] )
            vals.append( rs[i][idx]  )
    if len(vals) < 4:
        print "Only %s observations found for %s, won't grid" % (len(vals),
               idx)
        return None
    grid = Ngl.natgrid(lons, lats, vals, constants.XAXIS, constants.YAXIS)
    if grid is not None:
        return grid.transpose()
    else:
        return None


def grid_day(nc, ts):
    """
    I proctor the gridding of data on an hourly basis
    @param ts Timestamp of the analysis, we'll consider a 20 minute window
    """
    offset = int((ts - (ts + mx.DateTime.RelativeDateTime(month=1,day=1,hour=0))).days)


    sql = """SELECT * from alldata WHERE day = '%s' and
             substr(stationid,3,4) != '0000' """ % (
         ts.strftime("%Y-%m-%d"), )
    rs = coop.query( sql ).dictresult()
    if len(rs) > 4:
        res = generic_gridder(rs, 'high')
        if res is not None:
            nc.variables['high_tmpk'][offset] = constants.f2k(res)
        res = generic_gridder(rs, 'low')
        if res is not None:
            nc.variables['low_tmpk'][offset] = constants.f2k(res)
        res = generic_gridder(rs, 'precip')
        if res is not None:
            nc.variables['p01d'][offset] = res * 24.5
    else:
        print "%s has %02i entries, FAIL" % (ts.strftime("%Y-%m-%d"), 
            len(rs))

def main(ts):

    # Load up our netcdf file!
    nc = netCDF3.Dataset("/mnt/mesonet/data/iemre/%s_mw_daily.nc" % (ts.year,), 'a')
    grid_day(nc , ts)

    nc.close()
{'fraction_non_alphanumeric': 0.14946695095948828, 'fraction_numerical': 0.01023454157782516, 'mean_word_length': 2.5191297824456114, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1053878', 'n_tokens_mistral': 1705, 'n_tokens_neox': 1634, 'n_words': 431}