<jupyter_start><jupyter_text>Advantage of using these tools- Create a 32 year timesseries at a single point in < 1 min, because only the data chunks that are needed are loaded  Import libraries<jupyter_code>import xarray as xr
from glob import glob
import numpy as np
import os.path


import sys<jupyter_output><empty_output><jupyter_text>Old School- Create a function to create filename and test if it exists<jupyter_code>def get_filename(lyr,idyjl):
    import datetime as dt
    import os.path
    data_dir = 'F:/data/sat_data/ccmp/v02.0/'
    d = dt.date(lyr,1,1) + dt.timedelta(idyjl)
    syr=str(d.year).zfill(4)
    smon=str(d.month).zfill(2)
    sdym=str(d.day).zfill(2)
    sjdy=str(idyjl).zfill(3)
    if (lyr>=2020) or ((lyr==2019) and (idyjl>119)):
        filename=data_dir + 'Y' + syr + '/M' + smon +  '/CCMP_RT_Wind_Analysis_'+ syr + smon + sdym +'_V02.0_L3.0_RSS.nc'
    else:
        filename=data_dir + 'Y' + syr + '/M' + smon +  '/CCMP_Wind_Analysis_'+ syr + smon + sdym +'_V02.0_L3.0_RSS.nc'
    iexist = os.path.isfile(filename) 
    return filename,iexist
%%time
icnt = 0 
for lyr in range(1987,2021):
    for idyjl in range(1,367):
        fname,iexist = get_filename(lyr,idyjl)
        if iexist:
            ds_day=xr.open_dataset(fname)
            if icnt==0:
                ds = ds_day
                icnt=1
            else:
                ds = xr.concat([ds,ds_day],dim='time')
ds<jupyter_output><empty_output><jupyter_text>Using Xarray and glob methods on netcdf files<jupyter_code>%%time
#directory structure
pattern_netcdf = 'F:/data/sat_data/ccmp/v02.0/*/*/*_V02.0_L3.0_RSS.nc'
#list files
netcdf_file_list = [x for x in glob(pattern_netcdf)]
%%time
#open dataset
ds=xr.open_mfdataset(files,combine='nested',concat_dim='time')
%%time

ds['wspd'] = np.sqrt(ds.uwnd**2+ds.vwnd**2)

ts = ds.wspd.sel(latitude=slice(-10,0),longitude=slice(170,180)).mean({'latitude','longitude'}).plot()<jupyter_output><empty_output><jupyter_text>The same workflow using Zarr<jupyter_code>%%time
pattern_zarr = 'F:/data/sat_data/ccmp/zarr2/'
ds= xr.open_zarr(pattern_zarr)
ds
%%time

ds['wspd'] = np.sqrt(ds.uwnd**2+ds.vwnd**2)

ts = ds.wspd.sel(latitude=slice(-10,0),longitude=slice(170,180)).mean({'latitude','longitude'}).plot()<jupyter_output>Wall time: 1min 25s
{'fraction_non_alphanumeric': 0.14986853637160386, 'fraction_numerical': 0.03593339176161262, 'mean_word_length': 4.0846325167037865, 'pattern_counts': {'":': 0, '<': 17, '<?xml version=': 0, '>': 18, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '299086', 'license': 'Apache-2.0', 'n_tokens_mistral': 982, 'n_tokens_neox': 930, 'n_words': 173, 'path': 'Cloud_testing/Test_CCMP_zarr_cmp_3ways_version.ipynb', 'repo_name': 'caitlinkroeger/cloud_science', 'size': '2282'}