<jupyter_start><jupyter_text>***Predictive performance***  Simulated data<jupyter_code>rmse_per_hour = pd.read_csv(osp.join(result_dir, 'abm', 
                'performance_evaluation', 'ablations', 'rmse_per_hour.csv'))
rmse_per_hour['rmse'] = rmse_per_hour['rmse'] / bscale['abm']
grouped = rmse_per_hour.query('horizon == 24').groupby(['model', 'experiment'])
means = grouped.rmse.aggregate(np.nanmean)
stds = grouped.rmse.aggregate(np.nanstd)
print('######## RMSE at H=24h #######')
print(means, stds, '\n')

pcc_per_hour = pd.read_csv(osp.join(result_dir, 'abm', 
                'performance_evaluation', 'ablations', 'pcc_per_hour.csv'))
grouped = pcc_per_hour.query('horizon == 24').groupby(['model', 'experiment'])
means = grouped.pcc.aggregate(np.nanmean)
stds = grouped.pcc.aggregate(np.nanstd)
print('######## PCC at H=24h #######')
print(means, stds)
fig, ax = plt.subplots(nrows=2, figsize=(6.5,7), sharex=True)

baseline_rmse = pd.read_csv(osp.join(result_dir, 'abm', 
                'performance_evaluation', 'final', 'rmse_per_hour.csv'))
baseline_pcc = pd.read_csv(osp.join(result_dir, 'abm', 
                'performance_evaluation', 'final', 'pcc_per_hour.csv'))

baseline_rmse['rmse'] /= bscale['abm']

rmse_per_hour = pd.read_csv(osp.join(result_dir, 'abm', 
                'performance_evaluation', 'ablations', 'rmse_per_hour.csv'))
pcc_per_hour = pd.read_csv(osp.join(result_dir, 'abm', 
                'performance_evaluation', 'ablations', 'pcc_per_hour.csv'))
rmse_per_hour['rmse'] /= bscale['abm']

df = rmse_per_hour.query('model == "FluxRGNN" & experiment == "final"')
best_t = df.groupby('trial')['rmse'].aggregate(np.nanmean).idxmin()
print(f'best trial (rmse): {best_t}')

df = pcc_per_hour.query('model == "FluxRGNN" & experiment == "final"')
best_t = df.groupby('trial')['pcc'].aggregate(np.nanmean).idxmax()
print(f'best trial (pcc) : {best_t}')

for i, ((m, e), label) in enumerate(baseline_models.items()):
    plot_performance(ax[0], baseline_rmse.query(f'model == "{m}" & experiment == "{e}"'), 
                     label, 'rmse', static=(m in static_models), H=H, 
                     color=baseline_colors[i], ls=baseline_ls[i])
    plot_performance(ax[1], baseline_pcc.query(f'model == "{m}" & experiment == "{e}"'), 
                     label, 'pcc', static=(m in static_models), H=H, 
                     color=baseline_colors[i], ls=baseline_ls[i])

for i, ((m, e), label) in enumerate(flux_models.items()):
    if m == 'FluxRGNN' and e == 'final':
        plot_performance(ax[0], rmse_per_hour.query(f'model == "{m}" & experiment == "{e}"'), 
                         label, 'rmse', static=(m in static_models), H=H, color=flux_colors[i])
        plot_performance(ax[1], pcc_per_hour.query(f'model == "{m}" & experiment == "{e}"'), 
                         label, 'pcc', static=(m in static_models), H=H, color=flux_colors[i])


    
ax[0].set_ylabel('RMSE', fontsize=14)
ax[1].set_ylabel('Pearson r', fontsize=14)
ax[1].set_xlabel('forcasting horizon [h]', fontsize=14)
ax[1].legend(loc='upper right', fontsize=12, 
            bbox_to_anchor=(0.97,1.47), framealpha=1)
ax[0].set_title('Simulated data', fontsize=14)

for axis in ax:
    axis.tick_params(axis='both', which='major', labelsize=12)
    axis.grid(color = 'gray', linestyle = '-', alpha=0.2)

fig.subplots_adjust(wspace=0.25, hspace=0)
fig.align_ylabels(ax)
fig.savefig(osp.join(output_dir, 'abm_rmse_pcc_per_hour.png'), bbox_inches='tight', dpi=200)<jupyter_output>best trial (rmse): 2
best trial (pcc) : 2<jupyter_text>Radar data<jupyter_code>rmse_per_hour = pd.read_csv(osp.join(result_dir, 'radar', 
                'performance_evaluation', 'ablations', 'rmse_per_hour.csv'))
rmse_per_hour['rmse'] = rmse_per_hour['rmse'] / bscale['radar']
grouped = rmse_per_hour.query('horizon == 24').groupby(['model', 'experiment'])
means = grouped.rmse.aggregate(np.nanmean)
stds = grouped.rmse.aggregate(np.nanstd)
print('######## RMSE at H=24h #######')
print(means, stds, '\n')

pcc_per_hour = pd.read_csv(osp.join(result_dir, 'radar', 
                'performance_evaluation', 'ablations', 'pcc_per_hour.csv'))
grouped = pcc_per_hour.query('horizon == 24').groupby(['model', 'experiment'])
means = grouped.pcc.aggregate(np.nanmean)
stds = grouped.pcc.aggregate(np.nanstd)
print('######## PCC at H=24h #######')
print(means, stds)
fig, ax = plt.subplots(nrows=2, figsize=(6.5,7), sharex=True)

baseline_rmse = pd.read_csv(osp.join(result_dir, 'radar', 
                'performance_evaluation', 'final', 'rmse_per_hour.csv'))
baseline_pcc = pd.read_csv(osp.join(result_dir, 'radar', 
                'performance_evaluation', 'final', 'pcc_per_hour.csv'))
baseline_rmse['rmse'] /= bscale['radar']

pcc_per_hour = pd.read_csv(osp.join(result_dir, 'radar', 
                'performance_evaluation', 'ablations', 'pcc_per_hour.csv'))
rmse_per_hour = pd.read_csv(osp.join(result_dir, 'radar', 
                'performance_evaluation', 'ablations', 'rmse_per_hour.csv'))
rmse_per_hour['rmse'] = rmse_per_hour['rmse'] / bscale['radar']

df = rmse_per_hour.query('model == "FluxRGNN" & experiment == "final"')
best_t = df.groupby('trial')['rmse'].aggregate(np.nanmean).idxmin()
print(f'best trial (rmse): {best_t}')

df = pcc_per_hour.query('model == "FluxRGNN" & experiment == "final"')
best_t = df.groupby('trial')['pcc'].aggregate(np.nanmean).idxmax()
print(f'best trial (pcc) : {best_t}')

for i, ((m, e), label) in enumerate(baseline_models.items()):
    plot_performance(ax[0], baseline_rmse.query(f'model == "{m}" & experiment == "{e}"'), 
                     label, 'rmse', static=(m in static_models), H=H, 
                     color=baseline_colors[i], ls=baseline_ls[i])
    plot_performance(ax[1], baseline_pcc.query(f'model == "{m}" & experiment == "{e}"'), 
                     label, 'pcc', static=(m in static_models), H=H, 
                     color=baseline_colors[i], ls=baseline_ls[i])

for i, ((m, e), label) in enumerate(flux_models.items()):
    if m == 'FluxRGNN' and e == 'final':
        plot_performance(ax[0], rmse_per_hour.query(f'model == "{m}" & experiment == "{e}"'), 
                         label, 'rmse', static=(m in static_models), H=H, color=flux_colors[i])
        plot_performance(ax[1], pcc_per_hour.query(f'model == "{m}" & experiment == "{e}"'), 
                         label, 'pcc', static=(m in static_models), H=H, color=flux_colors[i])


ax[0].set_ylabel('RMSE', fontsize=14)
ax[1].set_ylabel('PCC', fontsize=14)
ax[1].set_xlabel('forcasting horizon [h]', fontsize=14)
ax[1].legend(loc='upper right', fontsize=12, 
            bbox_to_anchor=(0.97,1.47), framealpha=1)
ax[0].set_title('Radar data', fontsize=14)

for axis in ax:
    axis.tick_params(axis='both', which='major', labelsize=12)
    axis.grid(color = 'gray', linestyle = '-', alpha=0.2)

fig.subplots_adjust(wspace=0.25, hspace=0)
fig.align_ylabels(ax)
fig.savefig(osp.join(output_dir, 'radar_rmse_pcc_per_hour.png'), bbox_inches='tight', dpi=200)<jupyter_output>best trial (rmse): 1
best trial (pcc) : 1<jupyter_text>***Ablation study***<jupyter_code>fig, ax = plt.subplots(nrows=2, ncols=2, figsize=(14,7), sharex=True)

for i, datasource in enumerate(['abm', 'radar']):
    
    # load baseline performance
    baseline_rmse = pd.read_csv(osp.join(result_dir, datasource, 
                    'performance_evaluation', 'final', 'rmse_per_hour.csv'))
    baseline_pcc = pd.read_csv(osp.join(result_dir, datasource, 
                    'performance_evaluation', 'final', 'pcc_per_hour.csv'))
    baseline_rmse['rmse'] /= bscale[datasource]
    
    # load FluxRGNN performance
    rmse_per_hour = pd.read_csv(osp.join(result_dir, datasource, 
                    'performance_evaluation', 'ablations', 'rmse_per_hour.csv'))
    pcc_per_hour = pd.read_csv(osp.join(result_dir, datasource, 
                    'performance_evaluation', 'ablations', 'pcc_per_hour.csv'))
    rmse_per_hour['rmse'] /= bscale[datasource]


    for j, ((m, e), label) in enumerate(baseline_models.items()):
        plot_performance(ax[0, i], baseline_rmse.query(f'model == "{m}" & experiment == "{e}"'), 
                         label, 'rmse', static=(m in static_models), H=H, 
                         color=baseline_colors[j], ls=baseline_ls[j])
        plot_performance(ax[1, i], baseline_pcc.query(f'model == "{m}" & experiment == "{e}"'), 
                         label, 'pcc', static=(m in static_models), H=H, 
                         color=baseline_colors[j], ls=baseline_ls[j])

    for j, ((m, e), label) in enumerate(flux_models.items()):
        if m == 'FluxRGNN' and e == 'final':
            z = 2
        else:
            z = 1
        plot_performance(ax[0, i], rmse_per_hour.query(f'model == "{m}" & experiment == "{e}"'), 
                         label, 'rmse', static=(m in static_models), H=H, color=flux_colors[j], z=z)
        plot_performance(ax[1, i], pcc_per_hour.query(f'model == "{m}" & experiment == "{e}"'), 
                         label, 'pcc', static=(m in static_models), H=H, color=flux_colors[j], z=z)

    ax[1,i].set_xlabel('forcasting horizon [h]', fontsize=14)
    
ax[0,0].set_ylabel('RMSE', fontsize=14)
ax[1,0].set_ylabel('Pearson r', fontsize=14)
ax[1,0].legend(loc='upper right', fontsize=12, 
            bbox_to_anchor=(0.97,1.37), framealpha=1)
ax[0,0].set_title('Simulated data', fontsize=14)
ax[0,1].set_title('Radar data', fontsize=14)

for axis in ax.flatten():
    axis.tick_params(axis='both', which='major', labelsize=12)
    axis.grid(color = 'gray', linestyle = '-', alpha=0.2)

fig.subplots_adjust(wspace=0.25, hspace=0)
fig.align_ylabels(ax)
fig.savefig(osp.join(output_dir, 'rmse_pcc_per_hour_combined.png'), bbox_inches='tight', dpi=200)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.18655720882503848, 'fraction_numerical': 0.015802975885069265, 'mean_word_length': 4.089295039164491, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '648021', 'license': 'MIT', 'n_tokens_mistral': 3829, 'n_tokens_neox': 3575, 'n_words': 590, 'path': 'notebooks/performance_evaluation.ipynb', 'repo_name': 'FionaLippert/FluxRGNN', 'size': '9745'}