<commit_before># Copyright (c) 2005 Iowa State University
# http://mesonet.agron.iastate.edu/ -- mailto:<EMAIL>
#
# This program is free software; you can redistribute it and/or modify it under
# the terms of the GNU General Public License as published by the Free Software
# Foundation; either version 2 of the License, or (at your option) any later
# version.
#
# This program is distributed in the hope that it will be useful, but WITHOUT
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License along with
# this program; if not, write to the Free Software Foundation, Inc.,
# 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
""" Aviation Product Parser! """

__revision__ = '$Id: spc_parser.py 4513 2009-01-06 16:57:49Z akrherz $'

# Twisted Python imports
from twisted.words.protocols.jabber import client, jid, xmlstream
from twisted.internet import reactor
from twisted.python import log
from twisted.enterprise import adbapi
from twisted.mail import smtp

# Standard Python modules
import re, os, math

# Python 3rd Party Add-Ons
import mx.DateTime, pg

# pyWWA stuff
from support import ldmbridge, TextProduct, utils
import secret
import common

os.environ["EMAILS"] = "10"


log.startLogging(open('logs/aviation.log','a'))
log.FileLogObserver.timeFormat = "%Y/%m/%d %H:%M:%S %Z"

POSTGIS = pg.connect(secret.dbname, secret.dbhost, user=secret.dbuser, 
                     passwd=secret.dbpass)
DBPOOL = adbapi.ConnectionPool("psycopg2", database=secret.dbname, 
                               host=secret.dbhost, password=<PASSWORD>)

# 
CS_RE = re.compile(r"""CONVECTIVE\sSIGMET\s(?P<label>[0-9A-Z]+)\s
VALID\sUNTIL\s(?P<hour>[0-2][0-9])(?P<minute>[0-5][0-9])Z\s
(?P<states>[A-Z ]+)\s
(?P<from>FROM)?\s?(?P<locs>[0-9A-Z \-]+)\s
(?P<dmshg>DMSHG|DVLPG|INTSF)?\s?(?P<geotype>AREA|LINE|ISOL)\s
(?P<cutype>EMBD|SEV|SEV\sEMBD|EMBD\sSEV)?\s?TS\s(?P<width>[0-9]+\sNM\sWIDE)?(?P<diameter>D[0-9]+)?
""", re.VERBOSE )

FROM_RE = re.compile(r"""
(?P<offset>[0-9]+)?(?P<drct>[A-Z]{1,3})?\s?(?P<loc>[A-Z0-9]{3})
""", re.VERBOSE)

OL_RE = re.compile(r"""
OUTLOOK\sVALID\s(?P<begin>[0-9]{6})-(?P<end>[0-9]{6})\n
""", re.VERBOSE)

AREA_RE = re.compile(r"""
AREA\s(?P<areanum>[0-9]+)\.\.\.FROM\s(?P<locs>[0-9A-Z \-]+)\n
""", re.VERBOSE)

# Load LOCS table
LOCS = {}
mesosite = pg.connect('mesosite', secret.dbhost)
rs = mesosite.query("""SELECT id, name, x(geom) as lon, y(geom) as lat from stations 
           WHERE network ~* 'ASOS' or network ~* 'AWOS'""").dictresult()
for i in range(len(rs)):
    LOCS[rs[i]['id']] = rs[i]
mesosite.close()

for line in open('/home/ldm/pyWWA/tables/vors.tbl'):
    if len(line) < 70 or line[0] == '!':
        continue
    id = line[:3]
    lat = float(line[56:60]) / 100.0
    lon = float(line[61:67]) / 100.0
    name = line[16:47].strip()
    LOCS[id] = {'lat': lat, 'lon': lon, 'name': name}

# Finally, GEMPAK!
for line in open('/home/ldm/pyWWA/tables/pirep_navaids.tbl'):
    id = line[:3]
    lat = float(line[56:60]) / 100.0
    lon = float(line[61:67]) / 100.0
    LOCS[id] = {'lat': lat, 'lon': lon}


# LDM Ingestor
class MyProductIngestor(ldmbridge.LDMProductReceiver):
    """ I receive products from ldmbridge and process them 1 by 1 :) """

    def connectionLost(self, reason):
        print 'connectionLost', reason
        reactor.callLater(5, self.shutdown)

    def shutdown(self):
        reactor.callWhenRunning(reactor.stop)


    def process_data(self, buf):
        """ Process the product """
        try:
            prod = TextProduct.TextProduct(buf)
            if prod.afos in ['SIGC','SIGW','SIGE']:
                process_SIGC(prod)
            elif prod.afos[:2] == 'WS':
                process_WS(prod)
        except Exception, myexp:
            common.email_error(myexp, buf)

def figure_expire(ptime, hour, minute):
    """
    Convert something like 0255Z into a full blown time
    """
    expire = ptime
    if hour < ptime.hour:
        expire += mx.DateTime.RelativeDateTime(days=1)
    expire += mx.DateTime.RelativeDateTime(hour=hour,minute=minute)
    return expire

def sanitize_angle(val):
    if val < 0:
        return 360 - math.fabs( val )
    if val > 360:
        return val - 360

def makebox(lons, lats):
    """
    Need to make a box from a line, sigh
    if we want a 10km box, we need to go 0.1 in the direction of choice
    hyp = 0.1
    cos(angle) = runx/0.1
    sin(angle) = runy/0.1
    """
    rlats = []
    rlons = []
    deltax = lons[1] - lons[0]
    deltay = lats[1] - lats[0]
    if deltax == 0:
        deltax = 0.001
    angle = math.atan(deltay/deltax)
    runx = 0.1 * math.cos(angle)
    runy = 0.1 * math.sin(angle)
    # UR
    rlons.append( lons[0] - runy )
    rlats.append( lats[0] + runx )
    # UL
    rlons.append( lons[1] - runy )
    rlats.append( lats[1] + runx )
    # LL
    rlons.append( lons[1] + runy )
    rlats.append( lats[1] - runx )
    # LR
    rlons.append( lons[0] + runy )
    rlats.append( lats[0] - runx )
    # UR
    rlons.append( lons[0] - runy )
    rlats.append( lats[0] + runx )
    return rlons, rlats

def locs2lonslats(locstr, geotype, widthstr, diameterstr):
    """
    Convert a locstring into a lon lat arrays
    """
    lats = []
    lons = []
    if geotype == 'LINE':
        width = float(widthstr.replace(" NM WIDE", ""))
        # Approximation
        widthdeg = width / 110.

    for l in locstr.split('-'):
        s = FROM_RE.search(l)
        if s:
            d = s.groupdict()
            if d['drct'] is not None:
                 (lon1, lat1) = utils.go2lonlat(LOCS[d['loc']]['lon'], LOCS[d['loc']]['lat'], 
                                                   d['drct'], float(d['offset']) )
            else:
                  (lon1, lat1) = (LOCS[d['loc']]['lon'], LOCS[d['loc']]['lat'])
            lats.append( lat1 )
            lons.append( lon1 )
    if geotype == 'ISOL':
        lats2 = []
        lons2 = []
        diameter = float(diameterstr.replace("D", ""))
        # Approximation
        diameterdeg = diameter / 110.
        # UR
        lons2.append( lons[0] - diameterdeg )
        lats2.append( lats[0] + diameterdeg )
        # UL
        lons2.append( lons[0] + diameterdeg )
        lats2.append( lats[0] + diameterdeg )<commit_msg>make sure offset is defined<commit_after># Copyright (c) 2005 Iowa State University
# http://mesonet.agron.iastate.edu/ -- mailto:<EMAIL>
#
# This program is free software; you can redistribute it and/or modify it under
# the terms of the GNU General Public License as published by the Free Software
# Foundation; either version 2 of the License, or (at your option) any later
# version.
#
# This program is distributed in the hope that it will be useful, but WITHOUT
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License along with
# this program; if not, write to the Free Software Foundation, Inc.,
# 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
""" Aviation Product Parser! """

__revision__ = '$Id: spc_parser.py 4513 2009-01-06 16:57:49Z akrherz $'

# Twisted Python imports
from twisted.words.protocols.jabber import client, jid, xmlstream
from twisted.internet import reactor
from twisted.python import log
from twisted.enterprise import adbapi
from twisted.mail import smtp

# Standard Python modules
import re, os, math

# Python 3rd Party Add-Ons
import mx.DateTime, pg

# pyWWA stuff
from support import ldmbridge, TextProduct, utils
import secret
import common

os.environ["EMAILS"] = "10"


log.startLogging(open('logs/aviation.log','a'))
log.FileLogObserver.timeFormat = "%Y/%m/%d %H:%M:%S %Z"

POSTGIS = pg.connect(secret.dbname, secret.dbhost, user=secret.dbuser, 
                     passwd=<PASSWORD>)
DBPOOL = adbapi.ConnectionPool("psycopg2", database=secret.dbname, 
                               host=secret.dbhost, password=<PASSWORD>)

# 
CS_RE = re.compile(r"""CONVECTIVE\sSIGMET\s(?P<label>[0-9A-Z]+)\s
VALID\sUNTIL\s(?P<hour>[0-2][0-9])(?P<minute>[0-5][0-9])Z\s
(?P<states>[A-Z ]+)\s
(?P<from>FROM)?\s?(?P<locs>[0-9A-Z \-]+)\s
(?P<dmshg>DMSHG|DVLPG|INTSF)?\s?(?P<geotype>AREA|LINE|ISOL)\s
(?P<cutype>EMBD|SEV|SEV\sEMBD|EMBD\sSEV)?\s?TS\s(?P<width>[0-9]+\sNM\sWIDE)?(?P<diameter>D[0-9]+)?
""", re.VERBOSE )

FROM_RE = re.compile(r"""
(?P<offset>[0-9]+)?(?P<drct>[A-Z]{1,3})?\s?(?P<loc>[A-Z0-9]{3})
""", re.VERBOSE)

OL_RE = re.compile(r"""
OUTLOOK\sVALID\s(?P<begin>[0-9]{6})-(?P<end>[0-9]{6})\n
""", re.VERBOSE)

AREA_RE = re.compile(r"""
AREA\s(?P<areanum>[0-9]+)\.\.\.FROM\s(?P<locs>[0-9A-Z \-]+)\n
""", re.VERBOSE)

# Load LOCS table
LOCS = {}
mesosite = pg.connect('mesosite', secret.dbhost, passwd=<PASSWORD>.dbpass)
rs = mesosite.query("""SELECT id, name, x(geom) as lon, y(geom) as lat from stations 
           WHERE network ~* 'ASOS' or network ~* 'AWOS'""").dictresult()
for i in range(len(rs)):
    LOCS[rs[i]['id']] = rs[i]
mesosite.close()

for line in open('/home/ldm/pyWWA/tables/vors.tbl'):
    if len(line) < 70 or line[0] == '!':
        continue
    id = line[:3]
    lat = float(line[56:60]) / 100.0
    lon = float(line[61:67]) / 100.0
    name = line[16:47].strip()
    LOCS[id] = {'lat': lat, 'lon': lon, 'name': name}

# Finally, GEMPAK!
for line in open('/home/ldm/pyWWA/tables/pirep_navaids.tbl'):
    id = line[:3]
    lat = float(line[56:60]) / 100.0
    lon = float(line[61:67]) / 100.0
    LOCS[id] = {'lat': lat, 'lon': lon}


# LDM Ingestor
class MyProductIngestor(ldmbridge.LDMProductReceiver):
    """ I receive products from ldmbridge and process them 1 by 1 :) """

    def connectionLost(self, reason):
        print 'connectionLost', reason
        reactor.callLater(5, self.shutdown)

    def shutdown(self):
        reactor.callWhenRunning(reactor.stop)


    def process_data(self, buf):
        """ Process the product """
        try:
            prod = TextProduct.TextProduct(buf)
            if prod.afos in ['SIGC','SIGW','SIGE']:
                process_SIGC(prod)
            elif prod.afos[:2] == 'WS':
                process_WS(prod)
        except Exception, myexp:
            common.email_error(myexp, buf)

def figure_expire(ptime, hour, minute):
    """
    Convert something like 0255Z into a full blown time
    """
    expire = ptime
    if hour < ptime.hour:
        expire += mx.DateTime.RelativeDateTime(days=1)
    expire += mx.DateTime.RelativeDateTime(hour=hour,minute=minute)
    return expire

def sanitize_angle(val):
    if val < 0:
        return 360 - math.fabs( val )
    if val > 360:
        return val - 360

def makebox(lons, lats):
    """
    Need to make a box from a line, sigh
    if we want a 10km box, we need to go 0.1 in the direction of choice
    hyp = 0.1
    cos(angle) = runx/0.1
    sin(angle) = runy/0.1
    """
    rlats = []
    rlons = []
    deltax = lons[1] - lons[0]
    deltay = lats[1] - lats[0]
    if deltax == 0:
        deltax = 0.001
    angle = math.atan(deltay/deltax)
    runx = 0.1 * math.cos(angle)
    runy = 0.1 * math.sin(angle)
    # UR
    rlons.append( lons[0] - runy )
    rlats.append( lats[0] + runx )
    # UL
    rlons.append( lons[1] - runy )
    rlats.append( lats[1] + runx )
    # LL
    rlons.append( lons[1] + runy )
    rlats.append( lats[1] - runx )
    # LR
    rlons.append( lons[0] + runy )
    rlats.append( lats[0] - runx )
    # UR
    rlons.append( lons[0] - runy )
    rlats.append( lats[0] + runx )
    return rlons, rlats

def locs2lonslats(locstr, geotype, widthstr, diameterstr):
    """
    Convert a locstring into a lon lat arrays
    """
    lats = []
    lons = []
    if geotype == 'LINE':
        width = float(widthstr.replace(" NM WIDE", ""))
        # Approximation
        widthdeg = width / 110.

    for l in locstr.split('-'):
        s = FROM_RE.search(l)
        if s:
            d = s.groupdict()
            if d['offset'] is not None:
                (lon1, lat1) = utils.go2lonlat(LOCS[d['loc']]['lon'], LOCS[d['loc']]['lat'], 
                                                   d['drct'], float(d['offset']) )
            else:
                  (lon1, lat1) = (LOCS[d['loc']]['lon'], LOCS[d['loc']]['lat'])
            lats.append( lat1 )
            lons.append( lon1 )
    if geotype == 'ISOL':
        lats2 = []
        lons2 = []
        diameter = float(diameterstr.replace("D", ""))
        # Approximation
        diameterdeg = diameter / 110.
        # UR
        lons2.append( lons[0] - diameterdeg )
        lats2.append( lats[0] + diameterdeg )
        # UL
        lons2.append( lons[0] + diameterdeg )
        lats2.append( lats[0] + diameterdeg )
{'fraction_non_alphanumeric': 0.15995021005134588, 'fraction_numerical': 0.02971837560292516, 'mean_word_length': 2.740180389874891, 'pattern_counts': {'":': 0, '<': 51, '<?xml version=': 0, '>': 47, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5670866', 'n_tokens_mistral': 5008, 'n_tokens_neox': 4676, 'n_words': 1131}