<commit_before># Need something to ingest the CTRE provided bridge data
# RSAI4
# RLRI4

import mx.DateTime
import sys
# Run every 3 minutes
if mx.DateTime.now().minute % 3 != 0:
    sys.exit(0)

import urllib2
import csv
import access
import pg
import secret
accessdb = pg.connect('iem', 'iemdb')



# Get Saylorville
req = urllib2.Request("ftp://%s:%s@192.168.127.12/Saylorville_Table3Min_current.dat" % (secret.CTRE_FTPUSER,
                                                        secret.CTRE_FTPPASS))
#try:
data = urllib2.urlopen(req).readlines()
#except:
#  pass
keys = data[0].strip().replace('"', '').split(',')
vals = data[1].strip().replace('"', '').split(',')
d = {}
for i in range(len(vals)):
    d[ keys[i] ] = vals[i]

ts = mx.DateTime.strptime(d['TIMESTAMP'], '%Y-%m-%d %H:%M:%S')

iem = access.Ob( 'RSAI4', "OT")
iem.setObTime( ts )
iem.data['drct'] = d['WindDir']
iem.data['sknt'] = float(d['WS_mph_S_WVT']) / 1.15
iem.data['gust'] = float(d['WS_mph_Max']) / 1.15
iem.updateDatabase( accessdb )

# Get Saylorville
req = urllib2.Request("ftp://%s:%s@192.168.127.12/Red Rock_Table3Min_current.dat" % (secret.CTRE_FTPUSER,
                                                        secret.CTRE_FTPPASS))
#try:
data = urllib2.urlopen(req).readlines()
#except:
#  pass
keys = data[0].strip().replace('"', '').split(',')
vals = data[1].strip().replace('"', '').split(',')
d = {}
for i in range(len(vals)):
    d[ keys[i] ] = vals[i]

ts = mx.DateTime.strptime(d['TIMESTAMP'], '%Y-%m-%d %H:%M:%S')

iem = access.Ob( 'RLRI4', "OT")
iem.setObTime( ts )
iem.data['drct'] = d['WindDir']
iem.data['sknt'] = float(d['WS_mph_S_WVT']) / 1.15
iem.data['gust'] = float(d['WS_mph_Max']) / 1.15
iem.updateDatabase( accessdb )<commit_msg>insert into the LDM too<commit_after># Need something to ingest the CTRE provided bridge data
# RSAI4
# RLRI4

import mx.DateTime
import sys
# Run every 3 minutes
if mx.DateTime.now().minute % 3 != 0:
    sys.exit(0)

import urllib2
import csv
import access
import pg
import secret
import os
accessdb = pg.connect('iem', 'iemdb')

csv = open('/tmp/ctre.txt', 'w')

# Get Saylorville
req = urllib2.Request("ftp://%s:%s@192.168.127.12/Saylorville_Table3Min_current.dat" % (secret.CTRE_FTPUSER,
                                                        secret.CTRE_FTPPASS))
#try:
data = urllib2.urlopen(req).readlines()
#except:
#  pass
keys = data[0].strip().replace('"', '').split(',')
vals = data[1].strip().replace('"', '').split(',')
d = {}
for i in range(len(vals)):
    d[ keys[i] ] = vals[i]

ts = mx.DateTime.strptime(d['TIMESTAMP'], '%Y-%m-%d %H:%M:%S')

iem = access.Ob( 'RSAI4', "OT")
iem.setObTime( ts )
drct = d['WindDir']
iem.data['drct'] = drct
sknt = float(d['WS_mph_S_WVT']) / 1.15
iem.data['sknt'] = sknt
gust = float(d['WS_mph_Max']) / 1.15
iem.data['gust'] = gust
iem.updateDatabase( accessdb )

csv.write("%s,%s,%s,%.1f,%.1f\n" % ('RSAI4', 
            ts.gmtime().strftime("%Y/%m/%d %H:%M:%S"),
      drct, sknt, gust) )


# Get Saylorville
req = urllib2.Request("ftp://%s:%s@192.168.127.12/Red Rock_Table3Min_current.dat" % (secret.CTRE_FTPUSER,
                                                        secret.CTRE_FTPPASS))
#try:
data = urllib2.urlopen(req).readlines()
#except:
#  pass
keys = data[0].strip().replace('"', '').split(',')
vals = data[1].strip().replace('"', '').split(',')
d = {}
for i in range(len(vals)):
    d[ keys[i] ] = vals[i]

ts = mx.DateTime.strptime(d['TIMESTAMP'], '%Y-%m-%d %H:%M:%S')

iem = access.Ob( 'RLRI4', "OT")
iem.setObTime( ts )
drct = d['WindDir']
iem.data['drct'] = drct
sknt = float(d['WS_mph_S_WVT']) / 1.15
iem.data['sknt'] = sknt
gust = float(d['WS_mph_Max']) / 1.15
iem.data['gust'] = gust
iem.updateDatabase( accessdb )

csv.write("%s,%s,%s,%.1f,%.1f\n" % ('RSAI4', 
            ts.gmtime().strftime("%Y/%m/%d %H:%M:%S"),
      drct, sknt, gust) )

csv.close()

cmd = "pqinsert -p 'data c 000000000000 csv/ctre.txt bogus txt' /tmp/ctre.txt"
os.system( cmd )
{'fraction_non_alphanumeric': 0.21643489254108722, 'fraction_numerical': 0.031352718078381796, 'mean_word_length': 3.4399551066217735, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 4, 'substrings_counts': 1, 'word_list_counts': {'cursed_substrings.json': 8, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '500451', 'n_tokens_mistral': 1684, 'n_tokens_neox': 1583, 'n_words': 306}