<jupyter_start><jupyter_text>[36](https://github.com/blaylockbk/Herbie/issues/36) Adding access to ECMWF fields<jupyter_code>from herbie.archive import Herbie
import xarray as xr
import cfgrib
import matplotlib.pyplot as plt
import pandas as pd

import requests
import json

from toolbox.cartopy_tools import common_features, pc
H = Herbie('2022-01-26', model='ecmwf')
H
H.grib
H.idx
H.download(":t:")
H.xarray(':t:')
_.gribfile_projection
df.index += 1
df
grib_source = H.grib
outFile = H.get_localFilePath(searchString=":10u:")

for i, (grbmsg, row) in enumerate(H.idx_df.iterrows()):
    if i == 0:
        curl = f"curl -s --range {row.range} {grib_source} > {outFile}"
        print(curl)

        print(f"{i=}")
        print(f"{grbmsg=}")
        print(f"{row.search_this}")
        #print(f"{}")

        print(f"{i+1:>4g}: GRIB_message={grbmsg:<3g} \033[34m{':'.join(row.values[5:]).rstrip(':')}\033[m")

H.get_localFilePath(':t:')
H.xarray(":10u:")
from herbie.archive import _searchString_help
print(_searchString_help('grib_ls'))
ds = cfgrib.open_datasets(
    "/p/cwfs/blaylock/data/ecmwf/20220126/20220126000000-0h-oper-fc.grib2",
    backend_kwargs=dict(indexpath="")
)
len(ds)
ds[0]
n = 0
ax = common_features(dpi=150).ax
ax.pcolormesh(ds[n].longitude, ds[n].latitude, ds[n].st)
ax.set_title(ds[n].st.attrs['GRIB_name'])<jupyter_output><empty_output><jupyter_text>Create Index DataFrame from grib_ls output<jupyter_code>H2 = Herbie('2022-01-26', model='hrrr')
H2.read_idx()
H.idx
# grib_ls keywords explained here:
# https://confluence.ecmwf.int/display/UDOC/Identification+keywords

r = requests.get(H.idx)
idxs = [
    json.loads(x) for x in r.text.split("\n") if x
]
df = pd.DataFrame(idxs)
df['start_byte'] = df['_offset']
df['end_byte'] = df['_offset'] + df['_length']
df['range'] = df.start_byte.astype(str) + "-" + df.end_byte.astype(str)
df['reference_time'] = pd.to_datetime(df.date+df.time, format="%Y%m%d%H%M")
df['step'] = pd.to_timedelta(df.step.astype(int), unit='H')
df['valid_time'] = df.reference_time + df.step

df = df.reindex(
    columns = [
        'start_byte',
        'end_byte',
        'range',
        'reference_time',
        'valid_time',
        'step',
        # --- Used for searchString ------------------------------------
        'param', # parameter field (variable)
        'levelist', # level
        'levtype', # sfc=surface, pl=pressure level, pt=potential vorticity
        # --------------------------------------------------------------
        'domain', # g=global
        'expver', # experiment version
        'class',  # classification (od=routing operations, rd=research, )
        'type',   # fc=forecast, an=analysis, 
        'stream', # oper=operationa, wave=wave, ef/enfo=ensemble, 
    ]
)

df['search_this'] = df.loc[:,'param':].astype(str).apply(
        lambda x: ":" + ":".join(x).rstrip(":").replace(':nan:', ":"), axis=1
    )

# df.attrs = dict(
#     url=self.idx,
#     source=self.idx_source,
#     description="Inventory index for the GRIB2 file.",
#     model=self.model,
#     product=self.product,
#     lead_time=self.fxx,
#     datetime=self.date,
# )

df
df.param.unique()
#searchString=':10(u|v):'
searchString=":(t|u|v|r):"

# Filter DataFrame by searchString
logic = df.search_this.str.contains(searchString)
if logic.sum() == 0:
    print(
        f"No GRIB messages found. There might be something wrong with {searchString=}"
    )
    #print(_searchString_help(searchString))

df.loc[logic]

"""
searchString examples for grib_ls style index files

Look at the ECMWF GRIB Parameter Database
https://apps.ecmwf.int/codes/grib/param-db

======================== ==============================================
searchString (oper/enso) Messages that will be downloaded 
======================== ==============================================
":2t:"                   2-m temperature
":10u:"                  10-m u wind vector      
":10v:"                  10-m v wind vector
":10(u|v):               **10m u and 10m v wind**
":d:"                    Divergence         
":gh:"                   geopotential height (all levels)        
":st:"                   soil temperature
":tp:"                   total precipitation
":msl:"                  mean sea level pressure
":q:"                    Specific Humidity         
":r:"                    relative humidity
":ro:"                   Runn-off          
":skt:"                  skin temperature
":sp:"                   surface pressure
":t:"                    temperature
":tcwv:"                 Total column vertically integrated water vapor
":vo:"                   Relative vorticity
":v:"                    v wind vector   
":u:"                    u wind vector
":(t|u|v|r):"            Temp, u/v wind, RH (all levels)

======================== ==============================================
searchString (wave/waef) Messages that will be downloaded 
======================== ==============================================
":swh:"                  Significant height of wind waves + swell
":mwp:"                  Mean wave period
":mwd:"                  Mean wave direction
":pp1d:"                 Peak wave period
":mp2:"                  Mean zero-crossing wave period

"""  

df.param.unique()
H2.read_idx()
H = Herbie('2022-01-26', model='ecmwf', product='oper')
r = requests.get(H.idx)
idxs = [
    json.loads(x) for x in r.text.split("\n") if x
]
df = pd.DataFrame(idxs)
df
df.param.unique(), len(df.param.unique())
msg = """
searchString examples for grib_ls style index files

Look at the ECMWF GRIB Parameter Database
https://apps.ecmwf.int/codes/grib/param-db

======================== ==============================================
searchString (oper/enso) Messages that will be downloaded
======================== ==============================================
":2t:"                   2-m temperature
":10u:"                  10-m u wind vector
":10v:"                  10-m v wind vector
":10(u|v):               **10m u and 10m v wind**
":d:"                    Divergence
":gh:"                   geopotential height (all levels)
":st:"                   soil temperature
":tp:"                   total precipitation
":msl:"                  mean sea level pressure
":q:"                    Specific Humidity
":r:"                    relative humidity
":ro:"                   Runn-off
":skt:"                  skin temperature
":sp:"                   surface pressure
":t:"                    temperature
":tcwv:"                 Total column vertically integrated water vapor
":vo:"                   Relative vorticity
":v:"                    v wind vector
":u:"                    u wind vector
":(t|u|v|r):"            Temp, u/v wind, RH (all levels)

======================== ==============================================
searchString (wave/waef) Messages that will be downloaded
======================== ==============================================
":swh:"                  Significant height of wind waves + swell
":mwp:"                  Mean wave period
":mwd:"                  Mean wave direction
":pp1d:"                 Peak wave period
":mp2:"                  Mean zero-crossing wave period
"""
print(msg)
df = H.read_idx()
df
df.reindex(columns=['start_byte', 'hi'])<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.2233794710154858, 'fraction_numerical': 0.016308071810333013, 'mean_word_length': 2.491866028708134, 'pattern_counts': {'":': 58, '<': 10, '<?xml version=': 0, '>': 11, 'https://': 4, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 5, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '242011', 'license': 'MIT', 'n_tokens_mistral': 2357, 'n_tokens_neox': 2192, 'n_words': 516, 'path': 'notebooks/issues/36_ecmwf_access.ipynb', 'repo_name': 'WToma/Herbie', 'size': '7297'}