<jupyter_start><jupyter_text>---<jupyter_code>import xarray as xr
#import rioxarray
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import sys
import cartopy.crs as ccrs
from scipy.interpolate import griddata
import geopandas as gpd
#from geopandas import GeoDataFrame
#from shapely.geometry import Point
from sklearn.neighbors import NearestNeighbors
from datetime import timedelta, date


#These dictionaries describe the local hour of the satellite
local_times = {"aquaDay":"13:30",
               "terraDay":"10:30",
               "terraNight":"22:30",
               "aquaNight":"01:30"
              }
# and are used to load the correct file for dealing with the date-line.
min_hours = {"aquaDay":2,
            "terraDay":-1,
            "aquaNight":-1,
            "terraNight":11}
max_hours = {"aquaDay":24,
            "terraDay":22,
            "aquaNight":13,
            "terraNight":24}


#Path to MODIS data
satellite_folder = '/network/group/aopp/predict/TIP016_PAXTON_RPSPEEDY/ML4L/ECMWF_files/raw/MODIS/'

#Path to ERA data
era_folder = '/network/group/aopp/predict/TIP016_PAXTON_RPSPEEDY/ML4L/ECMWF_files/raw'


pyproj.__version__
#Path for where to ouput saved files
IO_path = '/network/group/aopp/predict/TIP016_PAXTON_RPSPEEDY/ML4L/ECMWF_files/raw/outputs_multi/'

for dt in dates:
    d = dt.strftime("%Y-%m-%d")
    for h in hours:
        fname = satellite + '_'+str(d)+'_'+str(h)+'H_'+str(latitude_bound)+'L_'+str(tolerance)+'T.pkl'
        print(fname)
        df = pipeline(d,h,satellite,latitude_bound,ERA_fields,tolerance)
        df.to_pickle(IO_path+fname)



!python download_CDS.py
! conda activate ML2
import xarray as xr



root = '/network/group/aopp/predict/TIP016_PAXTON_RPSPEEDY/ML4L/ECMWF_files/'
f = 'ml_skin_unstructured_2010_01.grib'
f2 = 'sfc_skin2_unstructured_2010_01.grib'
f3 = 'TEMP.grib'


ds_grib = xr.open_dataset(root+f3, engine="cfgrib")
print (ds_grib)<jupyter_output><xarray.Dataset>
Dimensions:     (time: 744, values: 542080)
Coordinates:
  * time        (time) datetime64[ns] 2010-01-01 ... 2010-01-31T23:00:00
    step        timedelta64[ns] ...
    hybrid      float64 ...
    latitude    (values) float64 ...
    longitude   (values) float64 ...
    valid_time  (time) datetime64[ns] ...
Dimensions without coordinates: values
Data variables:
    q           (time, values) float32 ...
Attributes:
    GRIB_edition:            2
    GRIB_centre:             ecmf
    GRIB_centreDescription:  European Centre for Medium-Range Weather Forecasts
    GRIB_subCentre:          0
    Conventions:             CF-1.7
    institution:             European Centre for Medium-Range Weather Forecasts
    history:                 2021-12-01T14:09 GRIB to CDM+CF via cfgrib-0.9.9...
{'fraction_non_alphanumeric': 0.1106676342525399, 'fraction_numerical': 0.045718432510885344, 'mean_word_length': 3.013100436681223, 'pattern_counts': {'":': 12, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '356811', 'license': 'MIT', 'n_tokens_mistral': 1075, 'n_tokens_neox': 985, 'n_words': 211, 'path': 'legacy/legacy_scripts/legacy/Untitled.ipynb', 'repo_name': 'tomkimpson/ML4L', 'size': '2756'}