<jupyter_start><jupyter_text>Evaluation  Get predictions<jupyter_code>val_crps, val_rmse = gen_eval(gen, dl_test, 30, ds_test.mins.tp.values, ds_test.maxs.tp.values, ds_test.tp_log, device)
print(val_crps)
print(val_rmse)<jupyter_output>0.9353182<jupyter_text>Simple reconstruction generator baseline:<jupyter_code>DATADRIVE = '/datadrive_ssd/'
dl_train = pickle.load(open(DATADRIVE+"saved_datasets/trainloader_single_forecast_only_log_trans_full.pkl", "rb"))
gen = StephanGenerator(nres=1, nf_in=1, nf=16, use_noise=True, activation_out='relu'
).to(device)
criterion = nn.MSELoss()
optimizer = torch.optim.Adam(gen.parameters(), lr=1e-4)
trainer = Trainer(gen, optimizer, criterion, dl_train)
trainer.fit(50)
torch.save(gen.state_dict(), f'gen_pretrained.pt')
gen.load_state_dict(torch.load('gen_pretrained.pt'))
X, y = next(iter(dl_train))
plot_sample_old(X, y, gen, 0)
def gen_eval(gen, dl_test, nens, ds_min, ds_max, tp_log, device):
    """
    gen: generator, which takes (forecast, noise) as arguments
    dl_test: dataloader
    ds_min and ds_max: the min and max values for unscaling
    tp_log: for undoing the log scaling
    """
    crps = []
    rmse = []
    for batch_idx, (x,y) in enumerate(dl_test):
        x = x.to(device)
        preds = []
        for i in range(nens):
            pred = gen(x).detach().to('cpu').numpy().squeeze()
            preds.append(pred)
        preds = np.array(preds).squeeze()
        truth = y.numpy().squeeze()
        truth = xr.DataArray(
                truth,
                dims=['sample','lat', 'lon'],
                name='tp'
            )
        preds = xr.DataArray(
                preds,
                dims=['member', 'sample', 'lat', 'lon'],
                name='tp'
            )

        truth = truth * (ds_max - ds_min) + ds_min

        preds = preds * (ds_max - ds_min) + ds_min
    
        if tp_log:
            truth = log_retrans(truth, tp_log)
            preds = log_retrans(preds, tp_log)
        for sample in range(x.shape[0]):

            sample_crps = xs.crps_ensemble(truth.sel(sample=sample), preds.sel(sample=sample)).values
            sample_rmse = xs.rmse(preds.sel(sample=sample).mean('member'), truth.sel(sample=sample), dim=['lat', 'lon']).values
            crps.append(sample_crps)
            rmse.append(sample_rmse)
    
    
    return np.mean(crps), np.mean(rmse)
val_crps, val_rmse = gen_eval(gen, dl_test, 30, ds_test.mins.tp.values, ds_test.maxs.tp.values, ds_test.tp_log, device)
print(val_crps)
print(val_rmse)<jupyter_output>0.90153366
{'fraction_non_alphanumeric': 0.13898704358068315, 'fraction_numerical': 0.012171181782489203, 'mean_word_length': 3.0768, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '327541', 'license': 'MIT', 'n_tokens_mistral': 982, 'n_tokens_neox': 947, 'n_words': 163, 'path': 'notebooks/stephan_notebooks/17-Patch-evaluation.ipynb', 'repo_name': 'raspstephan/nwp-downscale', 'size': '2547'}