<commit_before># Process CoCoRaHS Stations!

import sys
import urllib2
import os
import mx.DateTime
import pg
import access
dbconn = pg.connect('iem', 'iemdb')

now = mx.DateTime.now() - mx.DateTime.RelativeDateTime(hours=3)

state = sys.argv[1]

url = "http://data.cocorahs.org/Cocorahs/export/exportreports.aspx?ReportType=Daily&dtf=1&Format=CSV&State=%s&ReportDateType=timestamp&Date=%s&TimesInGMT=False" % (state, now.strftime("%m/%d/%Y%%20%H:00%%20%P") ) 
req = urllib2.Request( url )
data = urllib2.urlopen(req).readlines()


# Process Header
header = {}
h = data[0].split(",")
for i in range(len( h )):
    header[ h[i] ] = i

if not header.has_key('StationNumber'):
    sys.exit()

def safeP(v):
    v = v.strip()
    if v == "T":
        return 0.0001
    if v == "NA":
        return -99
    return v

for row in  data[1:]:
    cols = row.split(",")
    id = cols[ header["StationNumber"] ].strip()<commit_msg>make sure values are floats<commit_after># Process CoCoRaHS Stations!

import sys
import urllib2
import os
import mx.DateTime
import pg
import access
dbconn = pg.connect('iem', 'iemdb')

now = mx.DateTime.now() - mx.DateTime.RelativeDateTime(hours=3)

state = sys.argv[1]

url = "http://data.cocorahs.org/Cocorahs/export/exportreports.aspx?ReportType=Daily&dtf=1&Format=CSV&State=%s&ReportDateType=timestamp&Date=%s&TimesInGMT=False" % (state, now.strftime("%m/%d/%Y%%20%H:00%%20%P") ) 
req = urllib2.Request( url )
data = urllib2.urlopen(req).readlines()


# Process Header
header = {}
h = data[0].split(",")
for i in range(len( h )):
    header[ h[i] ] = i

if not header.has_key('StationNumber'):
    sys.exit()

def safeP(v):
    v = v.strip()
    if v == "T":
        return 0.0001
    if v == "NA":
        return -99
    return float(v)

for row in  data[1:]:
    cols = row.split(",")
    id = cols[ header["StationNumber"] ].strip()
{'fraction_non_alphanumeric': 0.17517463729177862, 'fraction_numerical': 0.022568511552928532, 'mean_word_length': 3.938992042440318, 'pattern_counts': {'":': 4, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4115520', 'n_tokens_mistral': 743, 'n_tokens_neox': 717, 'n_words': 155}