<jupyter_start><jupyter_text>S3 Access for cloud enabled datasetsIf a given dataset is in the cloud we have a set of libraries to access them. If the data is in a cloud friendly format we can efficiently load only what we need. If not we may need to read entire files.Dependencies:* [Valid `.netrc`](https://github.com/NASA-Openscapes/2021-Cloud-Hackathon/blob/main/tutorials/04_NASA_Earthdata_Authentication.ipynb) file in home directory* Running this code in AWS **us-west-2** (Like we are in the Openscapes hub)Glossary* Collection:* Granule:* S3: * S3 bucket:<jupyter_code>import requests
from pprint import pprint
from pathlib import Path
import s3fs
import fsspec

import xarray as xr

import matplotlib.pyplot as plt
import cartopy.crs as ccrs
# setting the endpoints and credentials

# This endpoint is specific to PODAAC, if we want cloud data from a different DAAC
# we need to change it. See: https://raw.githubusercontent.com/betolink/earthdata/main/earthdata/daac.py 
s3_cred_endpoint = 'https://archive.podaac.earthdata.nasa.gov/s3credentials'

cmr_search_url = 'https://cmr.earthdata.nasa.gov/search'
cmr_collection_url = f'{cmr_search_url}/{"collections"}'

cmr_search_url = 'https://cmr.earthdata.nasa.gov/search'
cmr_collection_url = f'{cmr_search_url}/{"collections"}'


def get_temp_creds():
    temp_creds_url = s3_cred_endpoint
    return requests.get(temp_creds_url).json()
temp_creds_req = get_temp_creds()

s3_fs = s3fs.S3FileSystem(
    key=temp_creds_req['accessKeyId'],
    secret=temp_creds_req['secretAccessKey'],
    token=temp_creds_req['sessionToken'],
    client_kwargs={'region_name':'us-west-2'},
)
# Search parameters

# In this case we know the dataset id in advance but we could use CMR to look for one
data_name = 'AVHRR_OI-NCEI-L4-GLOB-v2.1'
# Bounding Box spatial parameter in decimal degree 'W,S,E,N' format.
# If the dataset is global the bbox is just ignored because it will match anything.
bounding_box = '-105,21,-125,32'
# Each date in yyyy-MM-ddTHH:mm:ssZ format; date range in start,end format
temporal = '2019-06-22T00:00:00Z,2019-06-23T23:59:59Z'
response = requests.get(cmr_collection_url, 
                        params={
                            'short_name': data_name,
                            'cloud_hosted': 'True',
                            },
                        headers={
                            'Accept': 'application/json'
                            }
                       )
response = response.json()
collections = response['feed']['entry']

for collection in collections:
    print(f'{collection["id"]} {"version:"}{collection["version_id"]}')

data_concept_id = collections[0]["id"]

granule_url = f'{cmr_search_url}/{"granules"}'
response = requests.get(granule_url, 
                        params={
                            'concept_id': data_concept_id,
                            'temporal': temporal,
                            'bounding_box': bounding_box,
                            'page_size': 200,
                            },
                        headers={
                            'Accept': 'application/json'
                            }
                       )

# These are the metadata records for each granule, this is where we get our links to the data
# If the data is in the cloud the link prefix will start with s3://
granules = response.json()['feed']['entry']
urls = []

for granule in granules:
    print(granule['boxes'], granule['links'][0]['href'])
    urls.append(granule['links'][0]['href'])
# If a bucket is listable we can inspect whole directories
# The list of operations for s3fs is in here https://s3fs.readthedocs.io/en/latest/api.html
try:
    bucket = 's3://podaac-ops-cumulus-protected/AVHRR_OI-NCEI-L4-GLOB-v2.1/'
    list = s3_fs.ls(bucket)
    # Let's print the first 5 items
    for item in list[0:5]:
        print(item)
except Exception as e:
    print(e)
# Getting data from 1 granule (file) on S3
%time
file = s3_fs.open(urls[0])
ds = xr.open_dataset(file)
ds
# If we are opening multiple files that are on the same projection we can stack them
%time
fileset = [s3_fs.open(file) for file in urls] 
ds_stacked = xr.open_mfdataset(fileset, combine='by_coords')
ds_stacked
ds_stacked.sea_ice_fraction.sel(time='2019-06-21').plot(cmap="inferno")
# simple monthly mean (with only 3 days) for a given variable
ds_stacked.sea_ice_fraction.resample(time='M').mean().plot(cmap="inferno")<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1094661283086586, 'fraction_numerical': 0.020188425302826378, 'mean_word_length': 3.0135013501350136, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 6, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '324632', 'license': 'MIT', 'n_tokens_mistral': 1493, 'n_tokens_neox': 1417, 'n_words': 384, 'path': 'S3-PODAAC.ipynb', 'repo_name': 'NASA-Openscapes/nch2021-projects-pypsicle', 'size': '4458'}