<jupyter_start><jupyter_text>Part 2.  We collect the CMIP6 Data Issues with the Google Form[Collection Form](https://tinyurl.com/y5cw76at)- this notebook reads the corresponding Google Sheet as a csv file- currently all the issues filed by '<EMAIL>' are the issues used to concatenate the netcdf files in time to create zarr files in Google Cloud Storage (GCS)<jupyter_code>import os
import pandas as pd
import numpy as np
from my_search import esgf_search
import xarray as xr
import cftime
#import qgrid
# Link to the current list csv file of CMIP6 data issues collected through 
df = pd.read_csv('https://tinyurl.com/y3wmptly')
df.head()
cat = pd.read_csv('https://cmip6.storage.googleapis.com/cmip6-zarr-consolidated-stores.csv')
cat.head()
# define a simple search on keywords
def search_df(df, **search):
    d = df
    for skey in search.keys():
        print(skey,search[skey])
        d = d [ d[skey] == search[skey] ]
    return d
df.keys()
# ALL SUBSEQUENT CELLS ARE FOR PLAYING - REQUIRE MUCH MORE WORK - beware from here on ...

# Find files in our existing collection which are in the exception list (most have already been fixed)
dfs = df[['how_to_fix','source_id','experiment_id','member_id', 'table_id', 'variable_id', 'grid_label']]
dfall = []
for index, drow in dfs.iterrows():
    issue_fix = drow['how_to_fix']
    del drow['how_to_fix']
    dfthis = search_df(cat,**drow)
    dfthis['fix_code'] = issue_fix
    print(index,len(dfthis))
    dfall += [dfthis]
dfcat = pd.concat(dfall,sort=False)
#qgrid.show_grid(dfcat)
dfcat.head()
# grab tracking id from a zarr:
tracking_id = "hdl:21.14100/dca6a9f1-0ab9-4ab4-a661-9c49955bbd99\nhdl:21.14100/c13d7735-561e-4d34-9a9e-ec587ad18a2f\nhdl:21.14100/27d2a2f1-4ef7-4dfd-b1aa-c912ce569054\nhdl:21.14100/b36801f6-359a-4ce4-a423-f89676444659\nhdl:21.14100/7d61d119-9c99-4afa-9376-cd016c6520b6\nhdl:21.14100/2ebfd9a7-dea1-4bb6-97c1-a636901ed2e1\nhdl:21.14100/c5563f39-a6c7-41c8-a6e4-9221859019c9\nhdl:21.14100/f55c6182-0bfb-45c5-bebb-98af2465f6f5\nhdl:21.14100/e5dd99a9-df00-4c9b-93b2-e3d831a16d4e"
tracks = tracking_id.split('\n')
import pandas as pd

df_list = []
for track in tracks:
    files= esgf_search(server="https://esgf-node.llnl.gov/esg-search/search", mip_era='CMIP6', tracking_id=track, 
                                     page_size=500, verbose=False)
    files.loc[:,'version'] = [str.split('/')[-2] for str in files['HTTPServer_url']]
    files.loc[:,'file_name'] = [str.split('/')[-1] for str in files['HTTPServer_url']]
    # might need to set activity_id to activity_drs for some files (see old versions)
    files.loc[:,'activity_id'] = files.activity_drs
    files.loc[:,'time_range'] = [str[-28:-3] for str in files['file_name']]
    files.loc[:,'century'] = [str[-28:-26] for str in files['file_name']]
    #if (experiment_id=='highresSST-present')&(('3hr' in table_id)|('6hr' in table_id)):
    #    files = files[files.century=='20'] 

    df_list += [files.drop_duplicates(subset =["file_name","version","checksum"]) ]
dESGF = pd.concat(df_list,sort=False)
dESGF = dESGF.drop_duplicates(subset =["file_name","version","checksum"])
dESGF.keys()
dESGF.dataset_id.values
dESGF.time_range
dESGF.HTTPServer_url.values
dESGF.retracted
dESGF.replica<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1423312883435583, 'fraction_numerical': 0.08251533742331288, 'mean_word_length': 4.701048951048951, 'pattern_counts': {'":': 0, '<': 6, '<?xml version=': 0, '>': 6, 'https://': 4, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '278067', 'license': 'MIT', 'n_tokens_mistral': 1430, 'n_tokens_neox': 1282, 'n_words': 257, 'path': 'notebooks/Exceptions.ipynb', 'repo_name': 'naomi-henderson/DataCleaning', 'size': '3270'}