<jupyter_start><jupyter_text>Weather Data: Renewables.ninja processing notebook              Main notebook        Download weather data from Renewables.ninja        Process weather data from Renewables.ninja        Example script to download arbitrary MERRA-2 data            This Notebook is part of the Weather Data Package of Open Power System Data.<jupyter_code>%load_ext autoreload
%autoreload 2

import glob
import os
import sqlite3
import hashlib
import shutil

import pandas as pd
import geopandas as gp
import gsee
import tqdm
from joblib import Parallel, delayed

import generate_metadata
version = '2019-04-09'
changes = 'All European countries'
dir_shapefiles = os.path.join('downloads', 'shapefiles')
dir_countries = os.path.join('downloads', 'countries')
dir_nuts = os.path.join('downloads', 'nuts')<jupyter_output><empty_output><jupyter_text>Read and process data<jupyter_code>if os.path.exists('cached_dataframe.csv'):
    df_cached = pd.read_csv('cached_dataframe.csv', index_col=0, header=[0, 1], parse_dates=True)
else:
    df_cached = df_cached = pd.DataFrame({})  # Empty dataframe, to permit 'x in df_cached' tests
dataframes = {}
parse_kwargs = dict(skiprows=2, index_col=0, parse_dates=True)

# Files of form `ninja_weather_country_AT_merra-2_population_weighted.csv`
for f in glob.glob(os.path.join(dir_countries, '*.csv')):

    country_code = f.split('_')[3]
    
    if country_code not in df_cached:
        df = pd.read_csv(f, **parse_kwargs)
        dataframes[country_code] = df
    
# Files of form `ninja_weather_irradiance_surface_country_DE_merra-2_nuts-2_population_weighted.csv`
for f in glob.glob(os.path.join(dir_nuts, '*.csv')):

    country_code = f.split('country_')[1][0:2]
    variable = f.split('weather_')[1].split('_country')[0]

    df = pd.read_csv(f, **parse_kwargs)
    df = df.rename(columns={country_code + '_TOTAL': country_code})

    for c in df.columns:
        
        if c not in df_cached:
        
            if c not in dataframes:
                dataframes[c] = pd.DataFrame({variable: df[c]})
            else:
                dataframes[c].loc[:, variable] = df[c]

# Estimate direct and diffuse radiation using the BRL model as implented in GSEE.
# https://github.com/renewables-ninja/gsee

# Warning: this code takes a while to execute (easily 1-2mins CPU time per location).

nuts_centroids = gp.GeoDataFrame.from_file(os.path.join(dir_shapefiles, 'NUTS_LB_2016_4326.shp'))
nuts_centroids.set_index('NUTS_ID', inplace=True)

# Map GB to UK, GR to EL
nuts_centroids.loc['GB', : ] = nuts_centroids.loc['UK', :]
nuts_centroids.loc['GR', : ] = nuts_centroids.loc['EL', :]

data = {k: {
    'clearness': dataframes[k]['irradiance_surface'] / dataframes[k]['irradiance_toa'],
    'centroid': list(nuts_centroids.loc[k, 'geometry'].coords)[0][::-1],
    } for k in dataframes.keys()}


def process_item(item):
    return gsee.brl_model.run(hourly_clearness=item['clearness'], coords=item['centroid'])


result = Parallel(n_jobs=-1)(delayed(process_item)(item) for item in tqdm.tqdm(data.values()))
for i, k in enumerate(data.keys()):
    df = dataframes[k]
    diffuse_fraction = result[i]
    df['radiation_direct_horizontal'] = ((1 - diffuse_fraction) * df['irradiance_surface']).fillna(0)
    df['radiation_diffuse_horizontal'] = (diffuse_fraction * df['irradiance_surface']).fillna(0)
    dataframes[k] = df
variables = ['windspeed_10m', 'temperature', 'radiation_direct_horizontal', 'radiation_diffuse_horizontal']

for k in dataframes.keys():
    dataframes[k] = dataframes[k].loc[:, [v for v in variables if v in dataframes[k].columns]]

if len(dataframes) > 0:
    
    complete_data = pd.concat(dataframes, axis=1, join='inner')

    df = pd.concat([complete_data, df_cached], axis=1)

    df.columns = pd.MultiIndex.from_tuples(
        [(i[0], i[1]) for i in df.columns],
        names=['geography', 'variable']
    )

    df.index.name = 'utc_timestamp'

    df.to_csv('cached_dataframe.csv')
    
else:
    
    df = df_cached<jupyter_output><empty_output><jupyter_text>Write data to disk<jupyter_code>os.makedirs(version, exist_ok=True)<jupyter_output><empty_output><jupyter_text>Reshape data Data are provided in two different "shapes": - SingleIndex (easy to read for humans, compatible with datapackage standard, small file size)  - File formats: CSV, SQLite- MultiIndex (easy to read into GAMS, not compatible with datapackage standard, small file size)  - File formats: CSV, Excel<jupyter_code>df_multiindex = df

df_singleindex = df.copy()

df_singleindex.columns = [
    '_'.join([level for level in list(col)])
    for col in df.columns.values
]

df_stacked = df.copy()
df_stacked = df_stacked.transpose().stack(dropna=True).to_frame(name='data')
datetime_format = '%Y-%m-%dT%H:%M:%SZ'<jupyter_output><empty_output><jupyter_text>Write to SQLite database<jupyter_code># SQLite is required for the filtering function on the OPSD website

df_sqlite = df_singleindex.copy()
df_sqlite.index = df_sqlite.index.strftime(datetime_format)
filepath = os.path.join(version, 'weather_data.sqlite')

if os.path.exists(filepath):
    os.unlink(filepath)

df_sqlite.to_sql(
    'weather_data',
    sqlite3.connect(filepath),
    if_exists='replace',
    index_label='utc_timestamp'
)<jupyter_output><empty_output><jupyter_text>Write to CSV<jupyter_code>csv_kwargs = dict(
    float_format='%.4f',
    date_format=datetime_format
)

df_singleindex.to_csv(
    os.path.join(version, 'weather_data.csv'),
    **csv_kwargs
)

df_multiindex.to_csv(
    os.path.join(version, 'weather_data_multiindex.csv'),
    **csv_kwargs
)<jupyter_output><empty_output><jupyter_text>Write metadata<jupyter_code># See generate_metadata.py for details
generate_metadata.generate_json(df_multiindex, version, changes)<jupyter_output><empty_output><jupyter_text>Write checksums.txt We publish SHA checksums for the output files on GitHub to allow verifying their integrity on the OPSD server.<jupyter_code>def get_sha_hash(path, blocksize=65536):
    sha_hasher = hashlib.sha256()
    with open(path, 'rb') as f:
        buffer = f.read(blocksize)
        while len(buffer) > 0:
            sha_hasher.update(buffer)
            buffer = f.read(blocksize)
        return sha_hasher.hexdigest()


checksum_file_path = os.path.join(version, 'checksums.txt')
files = glob.glob(os.path.join(version, 'weather_data*'))

# Create checksums.txt in the version directory
with open(checksum_file_path, 'w') as f:
    for this_file in files:
        file_hash = get_sha_hash(this_file)
        f.write('{},{}\n'.format(os.path.basename(this_file), file_hash))

# Copy the file to root directory from where it will be pushed to GitHub
shutil.copyfile(checksum_file_path, 'checksums.txt')<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.12405625462620282, 'fraction_numerical': 0.008734270910436713, 'mean_word_length': 4.672544080604534, 'pattern_counts': {'":': 1, '<': 33, '<?xml version=': 0, '>': 35, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 5, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '8', 'dir': 'jupyter-structured-clean-dedup', 'id': '125074', 'license': 'MIT', 'n_tokens_mistral': 2426, 'n_tokens_neox': 2315, 'n_words': 504, 'path': 'processing.ipynb', 'repo_name': 'Open-Power-System-Data/weather_data', 'size': '6755'}