<jupyter_start><jupyter_text>Cordex preprocessing<jupyter_code>%load_ext autoreload
%autoreload 2
import xclim
xclim.__version__
import os
import intake
import xarray as xr
import numpy as np
from tqdm.notebook import tqdm

os.environ["HDF5_USE_FILE_LOCKING"] = "FALSE"

xr.set_options(keep_attrs=True)
print(np.__version__)
print(xr.__version__)

import intake_esm
print(intake_esm.__version__)

import cordex as cx
from cordex.preprocessing import preprocessing as preproc
eur11 = cx.cordex_domain('EUR-11')
!echo $HDF5_USE_FILE_LOCKING
from dask.distributed import Client, progress
client = Client()
client
#client.get_versions(check=True)<jupyter_output><empty_output><jupyter_text>Data access<jupyter_code>url = "/work/kd0956/Catalogs/mistral-cordex.json" 
cat = intake.open_esm_datastore(url)
cat
models = ['CLMcom-CCLM4-8-17',
     'CLMcom-BTU-CCLM4-8-17',
    'CLMcom-ETH-COSMO-crCLIM-v1-1',
     'CNRM-ALADIN53',
     'CNRM-ALADIN63',
     'DMI-HIRHAM5',
     'GERICS-REMO2015',
     'ICTP-RegCM4-6',  # regcm seems to make trouble when opening with to_dataset_dict..
     'IPSL-INERIS-WRF331F',
     'KNMI-RACMO22E',
    'MOHC-HadREM3-GA7-05',
     'MPI-CSC-REMO2009',
     'RMIB-UGent-ALARO-0',
     'SMHI-RCA4', 
     'UHOH-WRF361H']
models = ['UHOH-WRF361H']
#models = 'ICTP-RegCM4-6'
#institute_ids = ['CLMcom', 'CLMcom-BTU', 'CLMcom-ETH', 'CNRM', 'DMI', 'GERICS',
#       'IPSL-INERIS', 'KNMI', 'MOHC', 'MPI-CSC', 'RMIB-UGent', 'SMHI',
#       'UHOH', 'z_GERICS_KB']
# there are some missleading entries in the catalog, so we state all institute ids explicitly here
# to avoid wrong ones...
institute_ids =  ['CLMcom', 'CLMcom-BTU', 'CLMcom-ETH', 'CNRM', 'DMI', 'GERICS',
        'ICTP', 'IPSL-INERIS', 'KNMI', 'MOHC', 'MPI-CSC', 'RMIB-UGent',
        'SMHI', 'UHOH', 'z_GERICS_KB']
scens = ['rcp26', 'rcp45', 'rcp85']
expts = ['historical'] + scens
attrs = {'variable_id': ['tas'] , 'frequency': 'mon', 'CORDEX_domain': 'EUR-11', 
         'experiment_id': expts, 'rcm_version_id': 'v1', 'model_id': models,}
       #  'institute_id': institute_ids}
import pandas as pd
pd.set_option('display.max_rows', None)
selection = cat.search(**attrs)
selection.df.groupby(['model_id', 'institute_id', 'experiment_id', 'driving_model_id', 'member', 'frequency', 'rcm_version_id', 'version'])['variable_id'].unique().apply(list).to_frame()
selection.df.institute_id.unique()
#selection['EUR-11.MIROC-MIROC5.UHOH.UHOH-WRF361H.rcp85.mon'].df
preproc.rename_cordex
dset_dict = selection.to_dataset_dict(cdf_kwargs = {"use_cftime": True, "chunks": {}}, preprocess=preproc.rename_cordex)
dset_dict_flatten = preproc.member_id_to_dset_id(dset_dict)
dset_dict.keys()
sort = preproc.sort_ds_dict_by_attr(dset_dict, 'model_id')
for ds_id, ds in dset_dict.items():
    print(ds_id)
    print(ds.tas.dims)
dset_sorted = preproc.sort_ds_dict_by_attr(dset_dict_flatten, 'experiment_id')
dset_sorted['rcp45'].keys()
ds_list = []
for ds in dset_sorted['rcp85'].values():
    ds = preproc.replace_rlon_rlat(ds)
    ds = preproc.replace_lon_lat(ds)
    ds_list.append(ds)
from xclim.ensembles import create_ensemble
rcp85 = create_ensemble(ds_list, resample_freq='MS')
for key, ds in dset_dict.items():
    print(key)
    print(list(ds.dims))
    ds = preproc.rename_cordex(ds)
    preproc.check_domain(ds)
ds = dset_dict['EUR-11.NCC-NorESM1-M.z_GERICS_KB.GERICS-REMO2015.historical.mon']
ds = preproc.rename_cordex(ds)
ds = preproc.promote_empty_dims(ds)
ds = preproc.replace_rlon_rlat(ds)
ds
ds = dset_dict['EUR-11.NCC-NorESM1-M.CNRM.CNRM-ALADIN63.historical.mon']
ds
preproc.get_grid_mapping(ds)
ds = dset_dict['EUR-11.MPI-M-MPI-ESM-LR.GERICS.GERICS-REMO2015.historical.mon']
ds
preproc.get_grid_mapping(ds)
preproc.get_grid_mapping(ds)
preproc.remap_lambert_conformal(ds)
preproc.regridder is None
preproc._init_regridder(ds, eur11)
preproc.regridder
preproc.cordex_dataset_id(ds)
ds = dset_dict['EUR-11.IPSL-IPSL-CM5A-LR.GERICS.GERICS-REMO2015.historical.mon']
ds<jupyter_output><empty_output><jupyter_text>We move the member id from a coordinate to the dsets key so that all datasets have the same structure... we concentrate on rotated pole grids for now. there are just a handful of lambert conformal projection models that we can deal later with.<jupyter_code>from cordex.preprocessing import preprocessing as preproc
dset_dict['EUR-11.ICHEC-EC-EARTH.DMI.DMI-HIRHAM5.historical.mon']<jupyter_output><empty_output><jupyter_text>Now, we concatenate scenarion data with historical data for easier comparisons: Now, we have to align the different coordinats. The ensemble members might have slightly different rotated coordinates and also the projection to the global coordinates might be different. We use a reference dataset here, from wich we copy the coordinates to all datasets to make them comparable. The idea to make the whole dataset id a coordinate, so that xarray will automatically do all stuff for all datasets...<jupyter_code># make the dataset id a coordinate for easier access
dims = {}
for rcp, dsets in dsets_sorted.items():
    dset_ids = list(dsets.keys())
    dim = xr.DataArray(dset_ids, dims='dset_id', name='dset_id',
                      coords={'dset_id': dset_ids})
    dims[rcp] = dim
preproc.dset_ids_to_coord(dset_dict)<jupyter_output><empty_output><jupyter_text>we create three big datasets for each senario one:<jupyter_code>from cordex import cordex_domain


def create_test_ds(name, pol_name='rotated_latitude_longitude'):
    domain = cordex_domain(name, mapping_name=pol_name, dummy=True, add_vertices=True)
    domain.dummy
    return domain
    

dm = create_test_ds('EUR-11', 'rotated_pole')
#dm = dm.drop(('lon', 'lat'))
#dm.rename({'rlon': 'lon', 'rlat': 'lat'})
dm['dummy'] = xr.DataArray(dm.dummy.values, dims=('lat', 'lon'), attrs=dm.dummy.attrs)
dm
preproc.rename_cordex(dm)
preproc.rename_cordex(dm).equals(create_test_ds('EUR-11'))
dm = create_test_ds('EUR-11', 'rotated_pole')
#dm = dm.rename({'lon_vertices': 'longitude_vertices', 'lat_vertices': 'latitude_vertices'})
dm
dm.drop_vars(('rlon', 'rlat'))
preproc.rename_cordex(dm)
preproc.rename_cordex(ds)
xr.Dataset(dict)
preproc.rename_cordex(dm)
dm.dummy.reset_coords().assign_coords({'lat': dm.rlat.values, 'lon': dm.rlon.values})
dm.dummy.assign_coords({'rlat': dm.rlat.values, 'rlon': dm.rlon.values})<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.13999051982935692, 'fraction_numerical': 0.018012324221835994, 'mean_word_length': 5.614420062695925, 'pattern_counts': {'":': 2, '<': 21, '<?xml version=': 0, '>': 21, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '5', 'dir': 'jupyter-structured-clean-dedup', 'id': '8833', 'license': 'MIT', 'n_tokens_mistral': 2570, 'n_tokens_neox': 2475, 'n_words': 476, 'path': 'notebooks/cordex-preprocessing.ipynb', 'repo_name': 'ludwiglierhammer/py-cordex', 'size': '6329'}