<jupyter_start><jupyter_text>Official documentation:http://powietrze.gios.gov.pl/pjp/content/api<jupyter_code>%matplotlib inline
import requests
from pandas.io.json import json_normalize
import pandas as pd
import numpy as np<jupyter_output><empty_output><jupyter_text>Getting all stations:<jupyter_code>r = requests.get('http://api.gios.gov.pl/pjp-api/rest/station/findAll')
allStations = json_normalize(r.json())
print(allStations[allStations["city.name"] == u"Gdańsk"])<jupyter_output>addressStreet  city city.commune.communeName  \
27   ul. Powstańców Warszawskich   NaN                   Gdańsk   
46                  ul. Leczkowa   NaN                   Gdańsk   
47                 ul. Ostrzycka   NaN                   Gdańsk   
120                 ul. Kaczeńce   NaN                   Gdańsk   
143               ul. Wyzwolenia   NaN                   Gdańsk   

    city.commune.districtName city.commune.provinceName  city.id city.name  \
27                     Gdańsk                 POMORSKIE    218.0    Gdańsk   
46                     Gdańsk                 POMORSKIE    218.0    Gdańsk   
47                     Gdańsk                 POMORSKIE    218.0    Gdańsk   
120                    Gdańsk                 POMORSKIE    218.0    Gdańsk   
143                    Gdańsk                 POMORSKIE    218.0    Gdańsk   

    dateEnd            dateStart    gegrLat    gegrLon   id  \
27     None  1996-10-01 00:00:00  54.353336  18.635283  729   
46[...]<jupyter_text>Lets see what we have in "AM5 Gdańsk Szadółki" which has id: 733<jupyter_code>stationId = 733
r = requests.get('http://api.gios.gov.pl/pjp-api/rest/station/sensors/' + str(stationId))
sensors = json_normalize(r.json())
print(sensors)<jupyter_output>id  param.idParam param.paramCode param.paramFormula  \
0  4720              8              CO                 CO   
1  4727              3            PM10               PM10   
2  4723              6             NO2                NO2   
3  4725              5              O3                 O3   
4  4730              1             SO2                SO2   

       param.paramName sensorDateEnd      sensorDateStart  stationId  
0         tlenek węgla          None  1998-05-01 00:00:00        733  
1  pył zawieszony PM10          None  1998-05-01 00:00:00        733  
2      dwutlenek azotu          None  1998-05-01 00:00:00        733  
3                 ozon          None  1998-05-01 00:00:00        733  
4     dwutlenek siarki          None  1998-05-01 00:00:00        733<jupyter_text>Lets now see data about O3 concentration - sensorId = 4725<jupyter_code>sensorId = 4725
r = requests.get('http://api.gios.gov.pl/pjp-api/rest/data/getData/' + str(sensorId))
concentration = json_normalize(r.json())
concentrationFrame = pd.DataFrame()
concentrationFrame["dates"] = [d[u'date'] for d in concentration["values"].values.item()]
concentrationFrame["values"] = [d[u'value'] for d in concentration["values"].values.item()]
concentrationFrame.set_index(["dates"], inplace=True)
#concentrationFrame.sort_index(inplace=True)
# We cannot sort index, because it is not unique. There is 12 hours notation used, but without AM/PM distinction ;(
# But we can just reverse it until API will be fixed
concentrationFrame = concentrationFrame.iloc[::-1]
print(concentrationFrame)
concentrationFrame.plot(figsize=(15,5), grid=True)<jupyter_output><empty_output><jupyter_text>And overall air quality index for the same station<jupyter_code>r = requests.get('http://api.gios.gov.pl/pjp-api/rest/aqindex/getIndex/' + str(stationId))
r.json()
allStations
stationsId = allStations["id"]
sid = list(stationsId)
allsensors = pd.DataFrame()
for station in sid:
    print(station)
    sensorlist = json_normalize(requests.get('http://api.gios.gov.pl/pjp-api/rest/station/sensors/' + str(station)).json())
    print(sensorlist)
    print("--------------------------")
    allsensors = allsensors.append(sensorlist)
    #if station == 9000:
    #    break
allsensors
allStations[["id", "gegrLat", "gegrLon"]]
finalData = pd.merge(allsensors, allStations[["id", "gegrLat", "gegrLon"]], how='inner', left_on="stationId", right_on="id")

def get_latest_measurement(sensorId):
    concentration = json_normalize(requests.get('http://api.gios.gov.pl/pjp-api/rest/data/getData/' + str(sensorId)).json())
    concentrationFrame = pd.DataFrame()
    concentrationFrame["dates"] = [d[u'date'] for d in concentration["values"].values.item()]
    concentrationFrame["values"] = [d[u'value'] for d in concentration["values"].values.item()]
    concentrationFrame["dates"] = pd.to_datetime(concentrationFrame["dates"])
    #print(concentrationFrame[concentrationFrame["dates"] == previousHourStr]["values"])
    try:
        return_value = concentrationFrame[concentrationFrame["dates"] == previousHourStr]["values"].item()
    except ValueError:
        return_value = np.NaN
    #print(return_value)
    return return_value
get_latest_measurement(sensorId=sensorId)
import datetime
previousHour = datetime.datetime.now() - datetime.timedelta(hours = 1)

previousHourStr = previousHour.strftime('%Y-%m-%d %H:00:00')
sdfasd[sdfasd["dates"] == previousHourStr]["values"].item()
from tqdm import tqdm, tqdm_pandas
tqdm_pandas(tqdm())
finalData["value"] = finalData["id_x"].progress_map(get_latest_measurement)
finalData<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.11519790888722928, 'fraction_numerical': 0.04555638536221061, 'mean_word_length': 2.5904825737265416, 'pattern_counts': {'":': 0, '<': 19, '<?xml version=': 0, '>': 19, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 9, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '5', 'dir': 'jupyter-structured-clean-dedup', 'id': '437102', 'license': 'Apache-2.0', 'n_tokens_mistral': 1912, 'n_tokens_neox': 1747, 'n_words': 348, 'path': 'workspace/realtime.ipynb', 'repo_name': 'QuantumDamage/AQIP', 'size': '5356'}