<jupyter_start><jupyter_text>Making Zarr data from NetCDF files- Funding: Interagency Implementation and Advanced Concepts Team [IMPACT](https://earthdata.nasa.gov/esds/impact) for the Earth Science Data Systems (ESDS) program and AWS Public Dataset Program- Software developed during [OceanHackWeek 2020](https://github.com/oceanhackweek)    Credits: Tutorial development* [Dr. <NAME>](mailto:<EMAIL>) -  [Twitter](https://twitter.com/ChelleGentemann)   - Farallon Institute* [<NAME>](mailto:<EMAIL>) - [Twitter](https://twitter.com/clifgray) - Duke University* [<NAME>](mailto:<EMAIL>) - [Twitter](https://twitter.com/pahdsn) - University of Southampton Why data format matters- NetCDF sprinkles metadata throughout files, making them slow to access and read data- Zarr consolidates the metadata, making them FAST for access and reading<jupyter_code># filter some warning messages
import warnings 
warnings.filterwarnings("ignore") 

#libraries
import datetime as dt
import xarray as xr
import fsspec
import s3fs
from matplotlib import pyplot as plt
import numpy as np
import pandas as pd
# make datasets display nicely
xr.set_options(display_style="html")  
import os.path
from os import path

#magic fncts #put static images of your plot embedded in the notebook
%matplotlib inline  
plt.rcParams['figure.figsize'] = 12, 6
%config InlineBackend.figure_format = 'retina' 

def get_geo_data(sat,lyr,idyjl):
    # arguments
    # sat   goes-east,goes-west,himawari
    # lyr   year
    # idyjl day of year
    
    ds,iexist=[],False
    
    d = dt.datetime(lyr,1,1) + dt.timedelta(days=idyjl)
    fs = s3fs.S3FileSystem(anon=True) #connect to s3 bucket!

    #create strings for the year and julian day
    imon,idym=d.month,d.day
    syr,sjdy,smon,sdym = str(lyr).zfill(4),str(idyjl).zfill(3),str(imon).zfill(2),str(idym).zfill(2)
    
    #use glob to list all the files in the directory
    if sat=='goes-east':
        file_location,var = fs.glob('s3://noaa-goes16/ABI-L2-SSTF/'+syr+'/'+sjdy+'/*/*.nc'),'SST'
    if sat=='goes-west':
        file_location,var = fs.glob('s3://noaa-goes17/ABI-L2-SSTF/'+syr+'/'+sjdy+'/*/*.nc'),'SST'
    if sat=='himawari':
        file_location,var = fs.glob('s3://noaa-himawari8/AHI-L2-FLDK-SST/'+syr+'/'+smon+'/'+sdym+'/*/*L2P*.nc'),'sea_surface_temperature'
    print(file_location)
    #make a list of links to the file keys
    if len(file_location)<1:
        return file_ob
    file_ob = [fs.open(file) for file in file_location]        #open connection to files
    
    print('all files open')
    #open all the day's data
    with xr.open_mfdataset(file_ob[0:2], combine='nested',concat_dim='time') as ds: #note file is super messed up formatting
        iexist = True  #file exists

        #clean up coordinates which are a MESS in GOES
        #rename one of the coordinates that doesn't match a dim & should
        if not sat=='himawari':
            ds = ds.rename({'t':'time'})
            ds = ds.reset_coords()
            #clean up variables that should be attributes
            varlist = ['retrieval_solar_zenith_angle', 'day_solar_zenith_angle', 'y_image', 'retrieval_local_zenith_angle', 'x_image', 'night_solar_zenith_angle', 'quantitative_local_zenith_angle',
               'SST_night_only_emissive_wavelength', 'SST_night_only_emissive_band_id', 'SST_day_night_emissive_band_ids', 'SST_day_night_emissive_wavelengths']
            for var in varlist:
                try : 
                    ds[var]
                except KeyError: 
                    pass
                else: 
                    ds.attrs[var]=str(ds.variables[var].attrs['long_name']) + ' (' + str(ds.variables[var].attrs['units']) + ') = ' + str(ds.variables[var].values)
                    ds = ds.drop(var, dim=None)
        else:
            ds = ds.rename({'ni':'x','nj':'y'})    
    
    #put in to Celsius
    #ds[var] -= 273.15   #nice python shortcut to +- from itself a-=273.15 is the same as a=a-273.15
    #ds[var].attrs['units'] = '$^\circ$C'
   
    return ds, iexist<jupyter_output><empty_output><jupyter_text>Open GOES-16 (East Coast) Data- Careful of what you ask for.... each day is about 3 min to access<jupyter_code>ds,iexist = get_geo_data('g',lyr,idyjl)
%%time
lyr = 2020

#satlist = ['goes-east','goes-west','himawari']
satlist = ['goes-east']

for sat in satlist:

    init = 0  #reset new data store

    for idyjl in range(210,222,1): #180:201 #6/28/2020-7/18/2020

        print('starting ', idyjl)

        ds,iexist = get_geo_data(sat,lyr,idyjl)
        
        if not iexist:
            continue

        print('writing zarr store')

        if init == 0:
            ds.to_zarr(sat) #, mode='w'
            init = 1
        else:
            ds.to_zarr(sat,append_dim='time')

%%time 
lyr = 2020

#satlist = ['goes-east','goes-west','himawari']
sat = 'himawari'
idyjl = 210

ds,iexist = get_geo_data(sat,lyr,idyjl)
ds<jupyter_output>['noaa-himawari8/AHI-L2-FLDK-SST/2020/07/29/0000/20200729000000-STAR-L2P_GHRSST-SSTsubskin-AHI_H08-ACSPO_V2.71-v02.0-fv01.0.nc', 'noaa-himawari8/AHI-L2-FLDK-SST/2020/07/29/0100/20200729010000-STAR-L2P_GHRSST-SSTsubskin-AHI_H08-ACSPO_V2.71-v02.0-fv01.0.nc', 'noaa-himawari8/AHI-L2-FLDK-SST/2020/07/29/0200/20200729020000-STAR-L2P_GHRSST-SSTsubskin-AHI_H08-ACSPO_V2.71-v02.0-fv01.0.nc', 'noaa-himawari8/AHI-L2-FLDK-SST/2020/07/29/0300/20200729030000-STAR-L2P_GHRSST-SSTsubskin-AHI_H08-ACSPO_V2.71-v02.0-fv01.0.nc', 'noaa-himawari8/AHI-L2-FLDK-SST/2020/07/29/0400/20200729040000-STAR-L2P_GHRSST-SSTsubskin-AHI_H08-ACSPO_V2.71-v02.0-fv01.0.nc', 'noaa-himawari8/AHI-L2-FLDK-SST/2020/07/29/0500/20200729050000-STAR-L2P_GHRSST-SSTsubskin-AHI_H08-ACSPO_V2.71-v02.0-fv01.0.nc', 'noaa-himawari8/AHI-L2-FLDK-SST/2020/07/29/0600/20200729060000-STAR-L2P_GHRSST-SSTsubskin-AHI_H08-ACSPO_V2.71-v02.0-fv01.0.nc', 'noaa-himawari8/AHI-L2-FLDK-SST/2020/07/29/0700/20200729070000-STAR-L2P_GHRSST-SSTsubskin-AHI_H08-ACSPO[...]<jupyter_text>Now write this to our shared AWS S3 bucketNote that in order to do this you need the aws command line tools which can be installed by running from the command line `pip install awscli``aws s3 sync ./goes_east s3://ohw-bucket/goes_east``aws s3 sync ./goes_west s3://ohw-bucket/goes_west``aws s3 sync ./goes_west s3://ohw-bucket/himawari` note that putting the ! in front of a command in jupyter send it to the terminal so you could run it here with<jupyter_code>! pip install awscli
#! aws s3 sync ./goes_east s3://ohw-bucket/goes_east
#! aws s3 sync ./goes_west s3://ohw-bucket/goes_west
! aws s3 sync ./himawari s3://ohw-bucket/himawari<jupyter_output><empty_output><jupyter_text>Test reading the data<jupyter_code>%%time

sat = 'himawari'
file_location = 's3://ohw-bucket/'+sat

ds = xr.open_zarr(fsspec.get_mapper(file_location,anon=False))

ds<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1375055090348171, 'fraction_numerical': 0.061113559571029825, 'mean_word_length': 3.5568942436412314, 'pattern_counts': {'":': 0, '<': 23, '<?xml version=': 0, '>': 22, 'https://': 5, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '86057', 'license': 'Apache-2.0', 'n_tokens_mistral': 2938, 'n_tokens_neox': 2640, 'n_words': 510, 'path': 'make_zarr/netcdf_to_zarr_GEO_data.ipynb', 'repo_name': 'oceanhackweek/ohw20-proj-gapfree-sst', 'size': '6881'}