<commit_before>#!/usr/bin/env python
#
# Objective:
# 	Fetch open data from Taiwan EPA, and publish the measurement data
#	to the MQTT of LASS
#
# Note:
#	This source code is for reference only. Please do NOT run it unless
#	you change the MQTT server settings (we don't want to have too many
#	duplicate messages on LASS server :p)
#
# Requirements:
# 	Paho: The Paho Python Client provides a client class with support 
#	      for both MQTT v3.1 and v3.1.1 on Python 2.7 or 3.x. It also 
#	      provides some helper functions to make publishing one off 
#	      messages to an MQTT server very straightforward.
#	      URL: https://eclipse.org/paho/clients/python/

import paho.mqtt.client as mqtt
import re
import json
import requests
import pytz, datetime
import sys

################################################################
# Please configure the following settings for your environment
#MQTT_SERVER = "your mqtt server"
MQTT_PORT = 1883
MQTT_ALIVE = 60
MQTT_TOPIC = "LASS/Test/OpenData"

URL_DATA = "http://opendata.epa.gov.tw/ws/Data/REWXQA/"
PARA_DATA = dict(
	orderby = 'SiteName',
	skip = '0',
	top = '1000',
	format = 'json'
)
URL_SITE = "http://opendata.epa.gov.tw/ws/Data/AQXSite/"
PARA_SITE = dict(
	orderby = 'SiteName',
	skip = '0',
	top = '1000',
	format = 'json'
)

################################################################

def on_connect(client, userdata, flags, rc):
    print("MQTT Connected with result code "+str(rc))

def on_publish(mosq, obj, mid):
    print("mid: " + str(mid))


mqtt_client = mqtt.Client()
mqtt_client.on_connect = on_connect
mqtt_client.on_publish = on_publish
mqtt_client.connect(MQTT_SERVER, MQTT_PORT, MQTT_ALIVE)

resp = requests.get(url=URL_SITE, params=PARA_SITE)
array_site = json.loads(resp.text)

resp = requests.get(url=URL_DATA, params=PARA_DATA)
array_data = json.loads(resp.text)

for item in array_data:
	lat = 0
	lon = 0
	msg = "|ver_format=3|fmt_opt=1|app=EPA_COPY"
	publish_time = item['PublishTime']
	local = pytz.timezone("Asia/Taipei")
	naive = datetime.datetime.strptime(publish_time, "%Y-%m-%d %H:%M")
	local_dt = local.localize(naive, is_dst=None)
	publish_time = local_dt.astimezone(pytz.utc).strftime("%Y-%m-%d %H:%M:%S")
	pairs = publish_time.split(" ")
	msg = msg + "|date=" + pairs[0]
	msg = msg + "|time=" + pairs[1]
	for site in array_site:
		if (item['SiteName']==site['SiteName']):
			lat = site['TWD97Lat']
			lon = site['TWD97Lon']
			item['gps_lat'] = site['TWD97Lat']
			item['gps_lon'] = site['TWD97Lon']
			item['SiteEngName'] = site['SiteEngName']
			item['SiteType'] = site['SiteType']
			break;
	for key in item.iterkeys():
		msg = msg + "|" + key.replace(".","_") + "=" + item[key]

	mqtt_client.publish(MQTT_TOPIC, msg)

mqtt_client.disconnect()	
	<commit_msg>change the gps coordinate from DD to DMS format<commit_after>#!/usr/bin/env python
#
# Objective:
# 	Fetch open data from Taiwan EPA, and publish the measurement data
#	to the MQTT of LASS
#
# Note:
#	This source code is for reference only. Please do NOT run it unless
#	you change the MQTT server settings (we don't want to have too many
#	duplicate messages on LASS server :p)
#
# Requirements:
# 	Paho: The Paho Python Client provides a client class with support 
#	      for both MQTT v3.1 and v3.1.1 on Python 2.7 or 3.x. It also 
#	      provides some helper functions to make publishing one off 
#	      messages to an MQTT server very straightforward.
#	      URL: https://eclipse.org/paho/clients/python/

import paho.mqtt.client as mqtt
import re
import json
import requests
import pytz, datetime
import sys

################################################################
# Please configure the following settings for your environment
#MQTT_SERVER = "your mqtt server"
MQTT_PORT = 1883
MQTT_ALIVE = 60
MQTT_TOPIC = "LASS/Test/OpenData"

URL_DATA = "http://opendata.epa.gov.tw/ws/Data/REWXQA/"
PARA_DATA = dict(
	orderby = 'SiteName',
	skip = '0',
	top = '1000',
	format = 'json'
)
URL_SITE = "http://opendata.epa.gov.tw/ws/Data/AQXSite/"
PARA_SITE = dict(
	orderby = 'SiteName',
	skip = '0',
	top = '1000',
	format = 'json'
)

################################################################

def on_connect(client, userdata, flags, rc):
    print("MQTT Connected with result code "+str(rc))

def on_publish(mosq, obj, mid):
    print("mid: " + str(mid))

def dd2dms(dd):
    dd = float(dd)
    negative = dd < 0
    dd = abs(dd)
    minutes,seconds = divmod(dd*3600,60)
    degrees,minutes = divmod(minutes,60)
    if negative:
        if degrees > 0:
            degrees = -degrees
        elif minutes > 0:
            minutes = -minutes
        else:
            seconds = -seconds
    return str(degrees + 0.01 * minutes + 0.0001 * seconds)


mqtt_client = mqtt.Client()
mqtt_client.on_connect = on_connect
mqtt_client.on_publish = on_publish
mqtt_client.connect(MQTT_SERVER, MQTT_PORT, MQTT_ALIVE)

resp = requests.get(url=URL_SITE, params=PARA_SITE)
array_site = json.loads(resp.text)

resp = requests.get(url=URL_DATA, params=PARA_DATA)
array_data = json.loads(resp.text)

for item in array_data:
	lat = 0
	lon = 0
	msg = "|ver_format=3|fmt_opt=1|app=EPA_COPY"
	publish_time = item['PublishTime']
	local = pytz.timezone("Asia/Taipei")
	naive = datetime.datetime.strptime(publish_time, "%Y-%m-%d %H:%M")
	local_dt = local.localize(naive, is_dst=None)
	publish_time = local_dt.astimezone(pytz.utc).strftime("%Y-%m-%d %H:%M:%S")
	pairs = publish_time.split(" ")
	msg = msg + "|date=" + pairs[0]
	msg = msg + "|time=" + pairs[1]
	for site in array_site:
		if (item['SiteName']==site['SiteName']):
			lat = site['TWD97Lat']
			lon = site['TWD97Lon']
			item['gps_lat'] = dd2dms(str(site['TWD97Lat']))
			item['gps_lon'] = dd2dms(str(site['TWD97Lon']))
			item['SiteEngName'] = site['SiteEngName']
			item['SiteType'] = site['SiteType']
			break;
	for key in item.iterkeys():
		msg = msg + "|" + key.replace(".","_") + "=" + item[key]

	mqtt_client.publish(MQTT_TOPIC, msg)

mqtt_client.disconnect()	
	
{'fraction_non_alphanumeric': 0.1762247115866912, 'fraction_numerical': 0.016385219862899182, 'mean_word_length': 4.247368421052632, 'pattern_counts': {'":': 0, '<': 4, '<?xml version=': 0, '>': 5, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2231222', 'n_tokens_mistral': 2326, 'n_tokens_neox': 2193, 'n_words': 546}