<jupyter_start><jupyter_text>Data availability in cloud<jupyter_code># NB: This does not work on Cheyenne, since we need internet connection
import warnings
warnings.filterwarnings('ignore')
import numpy as np
from scipy.signal import detrend
from matplotlib import pyplot as plt
from scipy import signal
import pandas as pd
import xarray as xr
import intake
import pprint 
import util 
col = intake.open_esm_datastore("../catalogs/pangeo-cmip6.json")
exp_list = ['piControl', 'historical', 'ssp126', 'ssp245', 'ssp370', 'ssp585', 'abrupt-4xCO2']

df = pd.DataFrame()
for chosen_exp in exp_list:
    print(chosen_exp)
    cat = col.search(experiment_id = chosen_exp, variable_id='ts', table_id='Amon')
    
    # the following code removes models that causes errors in .to_dataset_dict:
    uni_dict = cat.unique(['source_id'])
    models = uni_dict['source_id']['values']; 
    source_ids = models[:]
    
    if chosen_exp == 'piControl':
            remove_models = ['IPSL-CM6A-LR']
    else:
        remove_models = []
        
    for mod in remove_models:
        source_ids.remove(mod) # remove models
        df.loc[mod, chosen_exp + ' (ens.mem.)'] = 'data problem'

    # load new table without the problematic models:
    cat = col.search(source_id = source_ids, experiment_id = chosen_exp, variable_id='ts', table_id='Amon')
    
    dset_dict = cat.to_dataset_dict(zarr_kwargs={'consolidated': True}, cdf_kwargs={'chunks': {}})
    keys = []
    for key in dset_dict.keys():
        keys.append(key)

    for key in keys:
        ds = dset_dict[key]
        model = ds.source_id
        start_time = str(ds['time'][0].values)[:7]
        end_time = str(ds['time'][-1].values)[:7]
        run_length = int(end_time[:4]) + 1 - int(start_time[:4])
        df.loc[model,chosen_exp + ' (yrs)'] = run_length
        df.loc[model,chosen_exp + ' (ens.mem.)'] = ds.member_id.size # number of ensemble members

    
column_names = list(df.columns);
yr_cols = [col for col in column_names if col[-5:] == '(yrs)']
ens_cols = [col for col in column_names if col[-10:] == '(ens.mem.)']
df2 = df[ens_cols + yr_cols]
df2
#df2.to_csv('available_data_cloud.txt', sep='\t')<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.13706739526411657, 'fraction_numerical': 0.013205828779599272, 'mean_word_length': 3.6844349680170576, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '643030', 'license': 'MIT', 'n_tokens_mistral': 788, 'n_tokens_neox': 761, 'n_words': 184, 'path': 'notebooks/Check_data_availability-cloud.ipynb', 'repo_name': 'Hegebf/enso_paper', 'size': '2196'}