<commit_before>from twisted.words.protocols.jabber import client, jid, xmlstream
from twisted.words.xish import domish
from twisted.internet import reactor
from twisted.python import log
import os
log.startLogging(open('/mesonet/data/logs/%s/metarObFe.log' % (os.getenv("USER"),), 'a'))
log.FileLogObserver.timeFormat = "%Y/%m/%d %H:%M:%S %Z"


import  sys, re, string, traceback, StringIO
from pyIEM import iemAccess, iemAccessOb, mesonet, ldmbridge, iemdb
from twisted.enterprise import adbapi
from metar import Metar
from email.MIMEText import MIMEText
import smtplib, secret, mx.DateTime
from common import *


dbpool = adbapi.ConnectionPool("psycopg2", database='iem', host=secret.dbhost)

iemaccess = iemAccess.iemAccess()

windAlerts = {}

class myProductIngestor(ldmbridge.LDMProductReceiver):

    def connectionLost(self, reason):
        print 'connectionLost', reason
        reactor.callLater(2, self.shutdown)

    def shutdown(self):
        reactor.callWhenRunning(reactor.stop)

    def processData(self, buf):
        try:
            real_processor(buf)
        except:
            io = StringIO.StringIO()
            traceback.print_exc(file=io)
            log.msg( io.getvalue() )
            msg = MIMEText("%s\n\n>RAW DATA\n\n%s"%(io.getvalue(),buf.replace("\015\015\012", "\n") ))
            msg['subject'] = 'metarObFe.py Traceback'
            msg['From'] = "<EMAIL>"
            msg['To'] = "<EMAIL>"

            s = smtplib.SMTP()
            s.connect()
            s.sendmail(msg["From"], msg["To"], msg.as_string())
            s.close()



def real_processor(buf):
    tokens = re.split("METAR", buf)
    if (len(tokens) == 1):
        tokens = re.split("SPECI", buf)
        if (len(tokens) == 1):
            log.msg("Nothing found: %s" % (buf) )
            return
    dataSect = tokens[1]
    dataSect = re.sub("\015\015\012", " ", dataSect)
    dataSect = re.sub("\015", " ", dataSect)
    metars = re.split("=", dataSect)

    for metar in metars:
        if (len(metar) < 10):
            continue
        clean_metar = re.sub("\s+", " ", metar.strip())
        if (clean_metar[0] not in ("K","X")):
            continue
        try:
          mtr = Metar.Metar(clean_metar)
        except Metar.ParserError:
          io = StringIO.StringIO()
          traceback.print_exc(file=io)
          log.msg( io.getvalue() )
          log.msg(clean_metar)
          continue

        iemid = mtr.station_id[-3:]
        if (mtr.station_id[0] == "X"): # West Texas Mesonet
            iemid = mtr.station_id
        iem = iemAccessOb.iemAccessOb(iemid)
        gts = mx.DateTime.DateTime( mtr.time.year, mtr.time.month, mtr.time.day, mtr.time.hour, mtr.time.minute)

        iem.setObTimeGMT(gts)
        iem.load_and_compare(iemaccess.iemdb)
        if (mtr.temp):
            iem.data['tmpf'] = mtr.temp.value("F")
        if (mtr.dewpt):
            iem.data['dwpf'] = mtr.dewpt.value("F")
        if mtr.wind_speed:
            iem.data['sknt'] = mtr.wind_speed.value("KT")
        if mtr.wind_gust:
            iem.data['gust'] = mtr.wind_gust.value("KT")
        if mtr.wind_dir:
            iem.set_drct(mtr.wind_dir.value())
        if mtr.vis:
            iem.data['vsby'] = mtr.vis.value("SM")
        if mtr.press:
            iem.data['alti'] = mtr.press.value("IN")
        iem.data['phour'] = 0
        if mtr.precip_1hr:
            iem.data['phour'] = mtr.precip_1hr.value("CM") * 10.0
        iem.data['raw'] = clean_metar
        iem.updateDatabase(None, dbpool)

        # Search for Peak wind gust info....
        if (mtr.wind_gust or mtr.wind_speed_peak):
            d = 0
            if (mtr.wind_gust):
                v = mtr.wind_gust.value("KT")
                if (mtr.wind_dir):
                    d = mtr.wind_dir.value()
                t = mtr.time
            if (mtr.wind_speed_peak):
                v = mtr.wind_speed_peak.value("KT")
                d = mtr.wind_dir_peak.value()
                t = mtr.peak_wind_time<commit_msg>UNITS, rar<commit_after>from twisted.words.protocols.jabber import client, jid, xmlstream
from twisted.words.xish import domish
from twisted.internet import reactor
from twisted.python import log
import os
log.startLogging(open('/mesonet/data/logs/%s/metarObFe.log' % (os.getenv("USER"),), 'a'))
log.FileLogObserver.timeFormat = "%Y/%m/%d %H:%M:%S %Z"


import  sys, re, string, traceback, StringIO
from pyIEM import iemAccess, iemAccessOb, mesonet, ldmbridge, iemdb
from twisted.enterprise import adbapi
from metar import Metar
from email.MIMEText import MIMEText
import smtplib, secret, mx.DateTime
from common import *


dbpool = adbapi.ConnectionPool("psycopg2", database='iem', host=secret.dbhost)

iemaccess = iemAccess.iemAccess()

windAlerts = {}

class myProductIngestor(ldmbridge.LDMProductReceiver):

    def connectionLost(self, reason):
        print 'connectionLost', reason
        reactor.callLater(2, self.shutdown)

    def shutdown(self):
        reactor.callWhenRunning(reactor.stop)

    def processData(self, buf):
        try:
            real_processor(buf)
        except:
            io = StringIO.StringIO()
            traceback.print_exc(file=io)
            log.msg( io.getvalue() )
            msg = MIMEText("%s\n\n>RAW DATA\n\n%s"%(io.getvalue(),buf.replace("\015\015\012", "\n") ))
            msg['subject'] = 'metarObFe.py Traceback'
            msg['From'] = "<EMAIL>"
            msg['To'] = "<EMAIL>"

            s = smtplib.SMTP()
            s.connect()
            s.sendmail(msg["From"], msg["To"], msg.as_string())
            s.close()



def real_processor(buf):
    tokens = re.split("METAR", buf)
    if (len(tokens) == 1):
        tokens = re.split("SPECI", buf)
        if (len(tokens) == 1):
            log.msg("Nothing found: %s" % (buf) )
            return
    dataSect = tokens[1]
    dataSect = re.sub("\015\015\012", " ", dataSect)
    dataSect = re.sub("\015", " ", dataSect)
    metars = re.split("=", dataSect)

    for metar in metars:
        if (len(metar) < 10):
            continue
        clean_metar = re.sub("\s+", " ", metar.strip())
        if (clean_metar[0] not in ("K","X")):
            continue
        try:
          mtr = Metar.Metar(clean_metar)
        except Metar.ParserError:
          io = StringIO.StringIO()
          traceback.print_exc(file=io)
          log.msg( io.getvalue() )
          log.msg(clean_metar)
          continue

        iemid = mtr.station_id[-3:]
        if (mtr.station_id[0] == "X"): # West Texas Mesonet
            iemid = mtr.station_id
        iem = iemAccessOb.iemAccessOb(iemid)
        gts = mx.DateTime.DateTime( mtr.time.year, mtr.time.month, mtr.time.day, mtr.time.hour, mtr.time.minute)

        iem.setObTimeGMT(gts)
        iem.load_and_compare(iemaccess.iemdb)
        if (mtr.temp):
            iem.data['tmpf'] = mtr.temp.value("F")
        if (mtr.dewpt):
            iem.data['dwpf'] = mtr.dewpt.value("F")
        if mtr.wind_speed:
            iem.data['sknt'] = mtr.wind_speed.value("KT")
        if mtr.wind_gust:
            iem.data['gust'] = mtr.wind_gust.value("KT")
        if mtr.wind_dir:
            iem.set_drct(mtr.wind_dir.value())
        if mtr.vis:
            iem.data['vsby'] = mtr.vis.value("SM")
        if mtr.press:
            iem.data['alti'] = mtr.press.value("IN")
        iem.data['phour'] = 0
        if mtr.precip_1hr:
            iem.data['phour'] = mtr.precip_1hr.value("IN")
        iem.data['raw'] = clean_metar
        iem.updateDatabase(None, dbpool)

        # Search for Peak wind gust info....
        if (mtr.wind_gust or mtr.wind_speed_peak):
            d = 0
            if (mtr.wind_gust):
                v = mtr.wind_gust.value("KT")
                if (mtr.wind_dir):
                    d = mtr.wind_dir.value()
                t = mtr.time
            if (mtr.wind_speed_peak):
                v = mtr.wind_speed_peak.value("KT")
                d = mtr.wind_dir_peak.value()
                t = mtr.peak_wind_time
{'fraction_non_alphanumeric': 0.14466116529132284, 'fraction_numerical': 0.009127281820455115, 'mean_word_length': 2.125830402500977, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '288416', 'n_tokens_mistral': 2893, 'n_tokens_neox': 2752, 'n_words': 513}