<jupyter_start><jupyter_text>Get list of available Zarr Stores<jupyter_code>def get_file_list():
    root = "s3://ncar-cesm-lens"
    fs = s3fs.S3FileSystem(anon=True)
    components = fs.ls(root)
    stores = []
    for component in tqdm(components):
        freq = fs.ls(component)[1:]
        for f in freq:
            print(f)
            objects = fs.glob(f"{f}/*.zarr")
            stores.extend(objects)
    return stores
stores = get_file_list()
stores[0:2]
stores = [f"s3://{store}" for store in stores]
stores[0:2]<jupyter_output><empty_output><jupyter_text>Extract attributes of zarr stores<jupyter_code>stores[1]
template = "s3://ncar-cesm-lens/{component}/{frequency}/cesmLE-{experiment}-{variable}.zarr"

def get_attrs(store):
    f = reverse_format(template, store)
    f['path'] = store
    return f
get_attrs(stores[1])
%%time
entries = list(map(get_attrs, stores))
entries[0]
df = pd.DataFrame(entries)
df = df.drop_duplicates(subset=['path'], keep='last').reset_index(drop=True)
df.head()
len(df)
df.component.unique()
df.to_csv("../catalogs/aws-cesm1-le.csv.gz", compression="gzip", index=False)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.16230366492146597, 'fraction_numerical': 0.012216404886561954, 'mean_word_length': 4.359813084112149, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '339429', 'license': 'Apache-2.0', 'n_tokens_mistral': 436, 'n_tokens_neox': 431, 'n_words': 72, 'path': 'builders/notebooks/aws-cesm1-le_catalog_builder.ipynb', 'repo_name': 'andersy005/intake-esm-datastore', 'size': '1146'}