<jupyter_start><jupyter_text>Dependencies<jupyter_code>import numpy as np
import pandas as pd
import xarray as xr

# geo libs are optional
from shapely.geometry import Point
import geopandas as gpd
from geopandas import GeoDataFrame
# cd ../..
pwd   # should be top-level, i.e. ~/*/generative-downscaling/
temp_1 = xr.open_dataset("./data/raw/temp/1406/2m_temperature_1991_1.40625deg.nc")
temp_5 = xr.open_dataset("./data/raw/temp/5625/2m_temperature_1991_5.625deg.nc")<jupyter_output><empty_output><jupyter_text>Basic XArray<jupyter_code>temp_1
temp_1.dims  # 8760/365 = 24!
temp_5.dims
temp_1.coords
type(temp_1.t2m.data)<jupyter_output><empty_output><jupyter_text>Basic Filtering<jupyter_code>daily = temp_1.isel(time=(temp_1.time.dt.hour == 0))
daily.dims
daily_5 = temp_5.isel(time=(temp_5.time.dt.hour == 0))
daily.dims<jupyter_output><empty_output><jupyter_text>Visual Exploration  Location & Consistency  Where is the Data Located?<jupyter_code>def obs_by_geo(weatherbench: pd.DataFrame) -> GeoDataFrame:
    """ Return total observations by coordinate """
    obs_by_geo = weatherbench.groupby(["lat","lon"]).size()
    obs_by_geo.name = "count"
    obs_by_geo = obs_by_geo.reset_index()

    gdf = GeoDataFrame(
        obs_by_geo,
        geometry=gpd.points_from_xy(obs_by_geo.lon - 180, obs_by_geo.lat)
    )

    return gdf
fine_cts = obs_by_geo(daily.to_dataframe())
coarse_cts = obs_by_geo(daily_5.to_dataframe())
world = gpd.read_file(gpd.datasets.get_path('naturalearth_lowres'))

ax = fine_cts.plot(
    ax=world.plot(figsize=(15, 8), alpha=.33),
    marker='o', color='red', markersize=.05,
    label = "Fine Resolution"
);

coarse_cts.plot(
    ax=ax,
    marker='o', color='green', markersize=1,
    label = "Coarse Resolution"
);

ax.set_title("WeatherBench Dataset Geographic Coverage (Full)", fontsize=20);
ax.set_xlabel("Longitude", fontsize=14);
ax.set_ylabel("Latitude", fontsize=14);
ax.legend(fontsize=12);<jupyter_output><empty_output><jupyter_text>Americas Only<jupyter_code>EPSILON = .0001
NA_LAT_FINE = (-5, 85+EPSILON)  # epsilon to avoid non-inclusive right
NA_LON_FINE = (-137.5+180, -47.5+180+EPSILON)

NA_LAT_COARSE = (-10, 88+EPSILON)  # Coarse should be wider for full coverage
NA_LON_COARSE = (-145+180, -45+180+EPSILON)

daily_na = daily.sel(lat=slice(*NA_LAT_FINE), lon=slice(*NA_LON_FINE))
daily_na_5 = daily_5.sel(lat=slice(*NA_LAT_COARSE), lon=slice(*NA_LON_COARSE))
fine_cts_americas = fine_cts[fine_cts['lat'].between(*NA_LAT_FINE) & fine_cts['lon'].between(*NA_LON_FINE)]
coarse_cts_americas = coarse_cts[coarse_cts['lat'].between(*NA_LAT_COARSE) & coarse_cts['lon'].between(*NA_LON_COARSE)]
americas = world[world['continent'].isin(["North America", "South America"])]

ax = fine_cts_americas.plot(
    ax=americas.plot(figsize=(8,8), alpha=.33),
    marker='o', color='red', markersize=.05,
    label = "Fine Resolution"
);

coarse_cts_americas.plot(
    ax=ax,
    marker='o', color='green', markersize=1,
    label = "Coarse Resolution"
);

ax.set_xlim(-160,-20);
ax.set_ylim(-30,90);

ax.set_title("WeatherBench Dataset Coverage (Americas Only)", fontsize=20);
ax.set_xlabel("Longitude", fontsize=14);
ax.set_ylabel("Latitude", fontsize=14);
ax.legend(fontsize=12);<jupyter_output><empty_output><jupyter_text>Temperature<jupyter_code>t5_pdf = temp_5.to_dataframe()
# t5_pdf = t5_pdf[:100]
# coords = t5_pdf.reset_index()[['lat','lon']].drop_duplicates()
# coords[coords['lat'].between(40,50) & coords['lon'].between(70+180,80+180)]

coords_mtl_approx = (47.8125, 253.125)  # lat, lon
mtl_1991 = t5_pdf.loc[coords_mtl_approx[0]].loc[coords_mtl_approx[1]]
mtl_1991_hi_lo = mtl_1991\
    .groupby([mtl_1991.index.date])\
    .agg([np.min, np.max])\
    .rename(columns={'amin': 'low', 'amax': 'high'})

mtl_1991_hi_lo_c = mtl_1991_hi_lo - 273.15  # kelvin to celsius
mtl_1991_hi_lo_c.plot()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.15754132231404958, 'fraction_numerical': 0.05242768595041322, 'mean_word_length': 6.198884758364312, 'pattern_counts': {'":': 0, '<': 25, '<?xml version=': 0, '>': 26, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '6', 'dir': 'jupyter-structured-clean-dedup', 'id': '42513', 'license': 'MIT', 'n_tokens_mistral': 1756, 'n_tokens_neox': 1601, 'n_words': 224, 'path': 'src/data/explore.ipynb', 'repo_name': '6759-Project/generative-downscaling', 'size': '3872'}