<jupyter_start><jupyter_text>Spatial-Temporal Experiment<jupyter_code>import sys, os
from pyprojroot import here
# sys.path.append(here)

# standard python packages
import xarray as xr
import pandas as pd
import numpy as np

# 
from src.models.spatemp.train_models import Metrics

# # esdc tools
# from src.esdc.subset import select_pixel
# from src.esdc.shape import ShapeFileExtract, rasterize
# from esdc.transform import DensityCubes

from tqdm import tqdm

import matplotlib.pyplot as plt
import cartopy
import cartopy.crs as ccrs
plt.style.use(['fivethirtyeight', 'seaborn-poster'])
%matplotlib inline

%load_ext autoreload
%autoreload 2
!ls /media/disk/databases/ESDC/<jupyter_output>Cube_2019highColombiaCube_184x120x120.zarr
Cube_2019highColombiaCube_1x3360x2760.zarr
esdc-8d-0.083deg-184x270x270-2.0.0.zarr
esdc-8d-0.083deg-1x2160x4320-2.0.0.zarr
esdc-8d-0.25deg-184x90x90-2.0.0.zarr
esdc-8d-0.25deg-1x720x1440-2.0.0.zarr<jupyter_text>1. Get DataCubes<jupyter_code>filename = '/media/disk/databases/ESDC/esdc-8d-0.25deg-1x720x1440-2.0.0.zarr'

datacube = xr.open_zarr(filename)
lst_cube = datacube[['soil_moisture', 'land_surface_temperature']]
lst_cube<jupyter_output><empty_output><jupyter_text>2. Select Region<jupyter_code>europe = lst_cube.sel(lat=slice(71.5, 35.5), lon=slice(-18.0, 60.0))<jupyter_output><empty_output><jupyter_text>3. Get Density Cubes<jupyter_code>spatial = 7
temporal = 1

# initialize minicuber
minicuber = DensityCubes(
    spatial_window=spatial, 
    time_window=temporal, 
)

europe_df = minicuber.get_minicubes(europe.land_surface_temperature)
europe_df.shape
print(7 * 7 * 1 - 1)
print(5 * 5 * 2 - 1)
print(4 * 4 * 3 - 1)
print(3 * 3 * 5 - 1)
print(2 * 2 * 11 - 1)
print(1 * 1 * 46 - 1)
europe_df.shape<jupyter_output><empty_output><jupyter_text>4. ML Model Framework  4.1 Preprocessing  4.1.1 - Training and testing<jupyter_code>europe_df.head()

y = europe_df.iloc[:, 0][:, np.newaxis]
X = europe_df.iloc[:, 1:]

d_dimensions = X.shape[1]<jupyter_output><empty_output><jupyter_text>4.1.2 - Train-Test Split<jupyter_code>from sklearn.model_selection import train_test_split


train_size = 1_000
random_state = 123

xtrain, xtest, ytrain, ytest = train_test_split(
    X, y, train_size=train_size, random_state=random_state)

test_size = xtest.shape[0]<jupyter_output><empty_output><jupyter_text>4.1.1 - Normalize<jupyter_code>from sklearn.preprocessing import StandardScaler

# normalize inputs
x_normalizer = StandardScaler(with_mean=True, with_std=False)

xtrain_norm = x_normalizer.fit_transform(xtrain)
xtest_norm = x_normalizer.transform(xtest)

# remove mean outputs
y_normalizer = StandardScaler(with_std=False)

ytrain_norm = y_normalizer.fit_transform(ytrain)
ytest_norm = y_normalizer.transform(ytest)<jupyter_output><empty_output><jupyter_text>4.2 - Training<jupyter_code>from gpy.sparse import SparseGPR
import GPy
# gp params
n_dims = xtrain_norm.shape[1]
kernel = GPy.kern.RBF(input_dim=n_dims, ARD=False)
inference = 'vfe'
n_inducing = 300
verbose = 1
max_iters = 5_000
n_restarts = 0

# initialize GP Model
sgp_model = SparseGPR(
    kernel=kernel, 
    inference=inference, 
    n_inducing=n_inducing, 
    verbose=verbose,
    max_iters=max_iters,
    n_restarts=n_restarts
)

# train GP model
sgp_model.fit(xtrain_norm, ytrain_norm)
sgp_model.display_model()<jupyter_output><empty_output><jupyter_text>4.3 - Testing<jupyter_code>ypred = sgp_model.predict(xtest_norm, return_std=False)
ypred.shape, ytest_norm.shape
stats = Metrics().get_all(ypred.squeeze(), ytest_norm.squeeze())
stats
stats['r2'].values
def _predict(model, Xs, batch_size):
    ms = []
    n = max(len(Xs) / batch_size, 1)  # predict in small batches
    with tqdm(np.array_split(Xs, n)) as bar:
        for xs in bar:
            m = model.predict(xs,)
            ms.append(m)

    return np.vstack(ms)
batch_size = 5_000
ms = []
n = max(len(xtest_norm) / batch_size, 1)  # predict in small batches
with tqdm(np.array_split(xtest_norm, n)) as bar:
    for xs in bar:
        m = sgp_model.predict(xs,)
        ms.append(m)


np.vstack(ms).shape
ypred = _predict(sgp_model, xtest_norm, 5_000)
ypred.shape<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1156985117618819, 'fraction_numerical': 0.04704752760441671, 'mean_word_length': 5.352134146341464, 'pattern_counts': {'":': 0, '<': 36, '<?xml version=': 0, '>': 36, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '9', 'dir': 'jupyter-structured-clean-dedup', 'id': '244041', 'license': 'MIT', 'n_tokens_mistral': 1845, 'n_tokens_neox': 1707, 'n_words': 293, 'path': 'docs/notebooks/spatial_temporal/1_experiment_walkthrough.ipynb', 'repo_name': 'jejjohnson/2019_rbig_rs', 'size': '4166'}