<jupyter_start><jupyter_text>Look at ESGF data availability<jupyter_code>#!/usr/bin/env python
from __future__ import print_function
import requests
import xml.etree.ElementTree as ET
import numpy

# Author: Unknown
# I got the original version from a word document published by ESGF
# https://docs.google.com/document/d/1pxz1Kd3JHfFp8vR2JCVBfApbsHmbUQQstifhGNdc6U0/edit?usp=sharing

# API AT: https://github.com/ESGF/esgf.github.io/wiki/ESGF_Search_REST_API#results-pagination

def esgf_search(server="https://esgf-node.llnl.gov/esg-search/search",
                files_type="OPENDAP", local_node=True, project="CMIP6",
                verbose=False, format="application%2Fsolr%2Bjson",
                use_csrf=False, **search):
    client = requests.session()
    payload = search
    payload["project"] = project
    payload["type"]= "File"
    if local_node:
        payload["distrib"] = "false"
    if use_csrf:
        client.get(server)
        if 'csrftoken' in client.cookies:
            # Django 1.6 and up
            csrftoken = client.cookies['csrftoken']
        else:
            # older versions
            csrftoken = client.cookies['csrf']
        payload["csrfmiddlewaretoken"] = csrftoken

    payload["format"] = format

    offset = 0
    numFound = 10000
    all_files = []
    files_type = files_type.upper()
    while offset < numFound:
        payload["offset"] = offset
        url_keys = [] 
        for k in payload:
            url_keys += ["{}={}".format(k, payload[k])]

        url = "{}/?{}".format(server, "&".join(url_keys))
        print(url)
        r = client.get(url)
        r.raise_for_status()
        resp = r.json()["response"]
        numFound = int(resp["numFound"])
        resp = resp["docs"]
        offset += len(resp)
        for d in resp:
            if verbose:
                for k in d:
                    print("{}: {}".format(k,d[k]))
            url = d["url"]
            for f in d["url"]:
                sp = f.split("|")
                if sp[-1] == files_type:
                    all_files.append(sp[0].split(".html")[0])
    return sorted(all_files)
'''df_ta = df.query("activity_id=='ScenarioMIP' & member_id == 'r1i1p1f1' & table_id == 'day' & variable_id == 'pr' & experiment_id == 'ssp370'" )
df_ta'''

result = esgf_search(server="https://esgf-node.llnl.gov/esg-search/search", activity_id='ScenarioMIP', table_id='day', variable_id='pr', experiment_id='ssp370',
                  member_id="r1i1p1f1")
result<jupyter_output>https://esgf-node.llnl.gov/esg-search/search/?activity_id=ScenarioMIP&table_id=day&variable_id=pr&experiment_id=ssp370&member_id=r1i1p1f1&project=CMIP6&type=File&distrib=false&format=application%2Fsolr%2Bjson&offset=0
https://esgf-node.llnl.gov/esg-search/search/?activity_id=ScenarioMIP&table_id=day&variable_id=pr&experiment_id=ssp370&member_id=r1i1p1f1&project=CMIP6&type=File&distrib=false&format=application%2Fsolr%2Bjson&offset=10
https://esgf-node.llnl.gov/esg-search/search/?activity_id=ScenarioMIP&table_id=day&variable_id=pr&experiment_id=ssp370&member_id=r1i1p1f1&project=CMIP6&type=File&distrib=false&format=application%2Fsolr%2Bjson&offset=20
https://esgf-node.llnl.gov/esg-search/search/?activity_id=ScenarioMIP&table_id=day&variable_id=pr&experiment_id=ssp370&member_id=r1i1p1f1&project=CMIP6&type=File&distrib=false&format=application%2Fsolr%2Bjson&offset=30
https://esgf-node.llnl.gov/esg-search/search/?activity_id=ScenarioMIP&table_id=day&variable_id=pr&experiment_id=ssp370&member[...]
{'fraction_non_alphanumeric': 0.1450555713878598, 'fraction_numerical': 0.024223425477343973, 'mean_word_length': 3.6123521681997373, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 4, 'https://': 9, 'lorem ipsum': 0, 'www.': 0, 'xml': 1}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '145616', 'license': 'MIT', 'n_tokens_mistral': 1324, 'n_tokens_neox': 1258, 'n_words': 163, 'path': 'notebooks/downscaling_pipeline/search_cmip6_gcs_esgf.ipynb', 'repo_name': 'brews/downscaleCMIP6', 'size': '3509'}