<jupyter_start><jupyter_text>this should go to the docs/test for metric module)See [this](https://github.com/jbusecke/cmip6_preprocessing/issues/17) issue for details.<jupyter_code>import pandas as pd
# get all available ocean models from the cloud. 
url = 'https://storage.googleapis.com/cmip6/pangeo-cmip6.csv'
df = pd.read_csv(url)
df_ocean =df[(df.table_id=='Omon') + (df.table_id=='Oyr')]
ocean_models = df_ocean.source_id.unique()
print(ocean_models)
print(len(ocean_models))

exclude_variables = ['mlotst', 'msftmz', 'intpp'] # at the moment cmip6_pp does not like fields that have only one spatial dimension


# get one dataset from each model, shouldnt matter which
col = intake.open_esm_datastore("https://raw.githubusercontent.com/NCAR/intake-esm-datastore/master/catalogs/pangeo-cmip6.json")
# cat = col.search(source_id=ocean_models)
cat = col.search(table_id=['Omon', 'Oyr'])
exclude_index = [vid not in exclude_variables for vid in cat.df['variable_id']]
cat.df = cat.df[exclude_index]
cat.df = cat.df.drop_duplicates(subset='source_id')
print(len(cat.df))
data_dict_raw = cat.to_dataset_dict(zarr_kwargs={'consolidated': True, 'decode_times':False},
                                preprocess=combined_preprocessing)
data_dict = parse_metrics(data_dict_raw, col, preprocess=combined_preprocessing)<jupyter_output>/srv/conda/envs/notebook/lib/python3.7/site-packages/pandas/core/computation/expressions.py:194: UserWarning: evaluating in Python space because the '+' operator is not supported by numexpr for the bool dtype, use '|' instead
  op=op_str, alt_op=unsupported[op_str]<jupyter_text>> UMMM why does it say there is no area detected?Ok see below. The parse metric function does only return the models that have an area in the cataloge<jupyter_code>print(len(data_dict_raw.keys()))
print(len(data_dict.keys()))
import matplotlib.pyplot as plt
fig, axarr = plt.subplots(ncols=6, nrows=len(data_dict.keys())//6 +1, figsize=[30,30], sharex=True, sharey=True)
for ax, (k, ds) in zip(axarr.flat, data_dict.items()):
    ds.areacello.plot(ax=ax, add_labels=False)
    ax.set_title(k)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.12816901408450704, 'fraction_numerical': 0.00892018779342723, 'mean_word_length': 6.009868421052632, 'pattern_counts': {'":': 0, '<': 8, '<?xml version=': 0, '>': 9, 'https://': 3, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '603046', 'license': 'MIT', 'n_tokens_mistral': 771, 'n_tokens_neox': 752, 'n_words': 164, 'path': 'notebooks/metric_parse_improvement.ipynb', 'repo_name': 'jetesdal/cmip6_preprocessing', 'size': '2130'}