<jupyter_start><jupyter_text>Save the necessary files<jupyter_code>%%time

# s3 = S3FileSystem()
validation_path = "/glade/scratch/zhonghua/CESM_validation/"
save_dir = "/glade/scratch/zhonghua/model-validation-diff/"
diff_by_lat_lon_ls = []

for i in tqdm(range(2,34)):
        member = (str(i).zfill(3))
        #print("member:",member)

        df_2006=pd.read_csv(validation_path+"2006/"+member+"_ens.csv")
        diff_by_lat_lon_2006 = df_2006.groupby(["lat","lon"])\
                                   .apply(lambda df_2006: 
                                          np.mean(df_2006[member]-df_2006["TREFMXAV_U"]))

        df_2061=pd.read_csv(validation_path+"2061/"+member+"_ens.csv")
        diff_by_lat_lon_2061 = df_2061.groupby(["lat","lon"])\
                                   .apply(lambda df_2061: 
                                          np.mean(df_2061[member]-df_2061["TREFMXAV_U"]))
        
        diff_by_lat_lon_ls.append(diff_by_lat_lon_2061 - diff_by_lat_lon_2006)
        
        del df_2006, df_2061, diff_by_lat_lon_2006, diff_by_lat_lon_2061
        gc.collect()
        
diff_by_lat_lon=pd.concat(diff_by_lat_lon_ls,axis=1)
diff_by_lat_lon.to_csv(save_dir+"diff_by_lat_lon.csv")<jupyter_output>100%|██████████| 32/32 [12:56<00:00, 24.26s/it]<jupyter_text>visualization<jupyter_code># s3 = S3FileSystem()
validation_path = "/glade/scratch/zhonghua/CESM_validation/"
save_dir = "/glade/scratch/zhonghua/model-validation-diff/"
df = pd.read_csv(save_dir+"diff_by_lat_lon.csv")
df_mean = df.set_index(["lat","lon"]).mean(axis=1)
display(df.describe())
display(df_mean.describe())
global_map_plot(df_mean.reset_index(), 0, -0.2, 0.2, "bwr","")<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.15199530516431925, 'fraction_numerical': 0.06690140845070422, 'mean_word_length': 3.3274111675126905, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 8, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 6, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '544877', 'license': 'MIT', 'n_tokens_mistral': 742, 'n_tokens_neox': 653, 'n_words': 60, 'path': '3_model_valid/eval/model_diff_evaluation.ipynb', 'repo_name': 'zzheng93/code_uhws', 'size': '1704'}