<jupyter_start><jupyter_text>Get sources (stations) in Bergen<jupyter_code># issue an HTTP GET request
r = requests.get(
    'https://frost.met.no/sources/v0.jsonld',
    {'ids': None},
    auth=(client_id, '')
)

def codec_utf8(s):
    return s.encode('utf-8').decode('utf-8') # should be used for Python 3

# extract some data from the response
if r.status_code == 200:
    for item in r.json()['data']:
        try:
            if 'BERGEN' in item['name']:
                sys.stdout.write('ID: {}\n'.format(item['id']))
                sys.stdout.write('Name: {}\n'.format(codec_utf8(item['name'])))
                if 'geometry' in item:
                    sys.stdout.write('longitude: {}\n'.format(item['geometry']['coordinates'][0]))
                    sys.stdout.write('latitude: {}\n'.format(item['geometry']['coordinates'][1]))
                if 'municipality' in item:
                    sys.stdout.write('Municipality: {}\n'.format(codec_utf8(item['municipality'])))
                if 'county' in item:
                    sys.stdout.write('County: {}\n'.format(codec_utf8(item['county'])))
                sys.stdout.write('Country: {}\n'.format(codec_utf8(item['country'])))
                if 'externalIds' in item:
                    for ext_id in item['externalIds']:
                        sys.stdout.write('external ID: {}\n'.format(ext_id))
                else:
                    sys.stdout.write('no external IDs found\n')
                    
                print('\n')
        except:
            print('\n')
else:
    sys.stdout.write('error:\n')
    sys.stdout.write('\tstatus code: {}\n'.format(r.status_code))
    if 'error' in r.json():
        assert(r.json()['error']['code'] == r.status_code)
        sys.stdout.write('\tmessage: {}\n'.format(r.json()['error']['message']))
        sys.stdout.write('\treason: {}\n'.format(r.json()['error']['reason']))
    else:
        sys.stdout.write('\tother error\n')<jupyter_output>ID: SN76944
Name: <NAME>
Municipality: KONTINENTALSOKKELEN
County: KONTINENTALSOKKELEN
Country: Norge
external ID: LFSR


ID: SN50540
Name: <NAME>
longitude: 5.3327
latitude: 60.383
Municipality: BERGEN
County: VESTLAND
Country: Norge
external ID: 01317
external ID: 10.249.0.159


ID: SN50480
Name: <NAME>
longitude: 5.2777
latitude: 60.2913
Municipality: BERGEN
County: VESTLAND
Country: Norge
external ID: 56.1


ID: SN50539
Name: BERGEN - <NAME>
longitude: 5.332
latitude: 60.3837
Municipality: BERGEN
County: VESTLAND
Country: Norge
external ID: 10.249.1.50


ID: SN245200
Name: KILSBERGEN-SUTTARBODA
longitude: 14.8986111111111
latitude: 59.2994444444444
Municipality: ÖREBRO LÄN
Country: Sverige
external ID: 02452
external ID: 94190<jupyter_text>Get temperature data<jupyter_code>endpoint = 'https://frost.met.no/observations/v0.jsonld'

for year in range(2013, 2020):

    parameters = {
        'sources': 'SN50540',
        'elements': 'air_temperature',
        'referencetime': str(year) + '-01-01/' + str(year) + '-12-31',
    }

    r = requests.get(
        'https://frost.met.no/observations/v0.jsonld',
        parameters,
        auth=(client_id, '')
    )
    
    # Check if the request worked, print out any errors
    json_data = r.json()
    if r.status_code == 200:
        data = json_data['data']
        print('Data retrieved from frost.met.no!')
    else:
        print('Error! Returned status code %s' % r.status_code)
        print('Message: %s' % json_data['error']['message'])
        print('Reason: %s' % json_data['error']['reason'])
        
    with open('../../data/weather/florida/json/weather_florida_' + str(year) + '.json', 'w') as f:
        json.dump(json_data, f)
parameters = {
    'sources': 'SN50540',
    'elements': 'air_temperature',
    'referencetime': '2020-01-01/2020-09-08',
}

r = requests.get(
    'https://frost.met.no/observations/v0.jsonld',
    parameters,
    auth=(client_id, '')
)

# Check if the request worked, print out any errors
json_data = r.json()
if r.status_code == 200:
    data = json_data['data']
    print('Data retrieved from frost.met.no!')
else:
    print('Error! Returned status code %s' % r.status_code)
    print('Message: %s' % json_data['error']['message'])
    print('Reason: %s' % json_data['error']['reason'])

with open('../../data/weather/florida/json/weather_florida_2020.json', 'w') as f:
    json.dump(json_data, f)<jupyter_output>Data retrieved from frost.met.no!<jupyter_text>Convert to csv<jupyter_code>json_data_path = '../../data/weather/florida/json/'

for path in os.listdir(json_data_path):
    
    if path[-5:] == '.json':

        df_current = json_normalize(pd.read_json(json_data_path + path)['data'])
        df_current['observations'] = df_current.observations.map(lambda x: x[0])

        df_current[['elementId', 'exposureCategory', 'level.levelType', 'level.unit',
           'level.value', 'performanceCategory', 'qualityCode', 'timeOffset',
           'timeResolution', 'timeSeriesId', 'unit', 'value']] = json_normalize(df_current['observations'])

        df_current.drop('observations', axis=1, inplace=True)

        print('Unique level values for ' + path + ':', df_current['level.value'].unique().item())

        df_current.to_csv('../../data/weather/florida/csv/' + path[:-5] + '.csv', index=False)<jupyter_output>Unique level values for weather_florida_2013.json: 2
Unique level values for weather_florida_2014.json: 2
Unique level values for weather_florida_2015.json: 2
Unique level values for weather_florida_2016.json: 2
Unique level values for weather_florida_2017.json: 2
Unique level values for weather_florida_2018.json: 2
Unique level values for weather_florida_2019.json: 2
Unique level values for weather_florida_2020.json: 2
{'fraction_non_alphanumeric': 0.159356980604578, 'fraction_numerical': 0.041062379870697184, 'mean_word_length': 3.1598837209302326, 'pattern_counts': {'":': 0, '<': 14, '<?xml version=': 0, '>': 14, 'https://': 4, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 2, 'substrings_counts': 2, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '142976', 'license': 'MIT', 'n_tokens_mistral': 2125, 'n_tokens_neox': 1922, 'n_words': 403, 'path': 'notebooks/get_weather_data/get_weather_data_bergen.ipynb', 'repo_name': 'Itera/load-forecasting', 'size': '5756'}