<jupyter_start><jupyter_text>Table of Contents Notebook to see how percentage improvement varies with reconstruction error for AEsHypothesis: they will be inversely correlated.     This experiement is more important if I do not find evidence for this hypothesis. I am just confirming that optimizing AEs to reduce reconstruction error will actually make more useful AEs.<jupyter_code>import os
import sys
nb_dir = os.path.split(os.getcwd())[0]
if nb_dir not in sys.path:
    sys.path.append(nb_dir)
from pipeline import config
from pipeline import TrainAE, DAPipeline
import pickle
import pandas as pd
test_size = 327
train_size = 659

modes = [2, 4, 8, 16, 32, 64]
hidden = [2, 4, 8, 16, 32, 64]
epochs = 100
experiment_name = "{}_epoch_summary.pkl".format(epochs)
settings = config.ToyAEConfig()
settings.DEBUG = False

mode_vals = []
hid_vals = []
recon_errs_valid = []
recon_errs_train = []
refMA_means = []
daMA_means = []
percent_improve_vals = []

for mode in modes:
    settings.NUMBER_MODES = mode
    for hid in hidden:
        settings.HIDDEN = hid
        settings.kwargs = {"inn":mode, "hid":hid, "out": settings.n}

        trainer = TrainAE(settings)
        model = trainer.train(epochs)
        DA = DAPipeline()
        _, stats = DA.Var_DA_routine(settings, return_stats = True)
        
        
        results_fp_train = settings.RESULTS_FP + "toy_train_mode{}_hid{}.txt".format(settings.NUMBER_MODES, settings.HIDDEN)
        results_fp_test = settings.RESULTS_FP + "toy_test_mode{}_hid{}.txt".format(settings.NUMBER_MODES, settings.HIDDEN)
        with open(results_fp_train, 'rb') as fp:
            train_data = pickle.load(fp)
        with open(results_fp_test, 'rb') as fp:
            test_data = pickle.load(fp)
        
        mode_vals.append(mode)
        hid_vals.append(hid)
        recon_errs_valid.append(test_data[-1][1])
        recon_errs_train.append(train_data[-1][1])
        percent_improve_vals.append(stats["Percent_improvement"])
        refMA_means.append(stats["ref_MAE_mean"])
        daMA_means.append(stats["da_MAE_mean"])
        
data = {"mode_vals": mode_vals,
    "hid_vals": hid_vals,
    "recon_errs_valid": recon_errs_valid,
    "recon_errs_train": recon_errs_train,
    "refMA_means": refMA_means,
    "daMA_means": daMA_means,
    "percent_improve_vals": percent_improve_vals}
df = pd.DataFrame(data)
fp = settings.RESULTS_FP +  experiment_name
df.to_pickle(fp) 

df2 = pd.read_pickle(fp)
df2
import matplotlib.pyplot as plt

plt.scatter(df2["percent_improve_vals"], df2["recon_errs_valid"] )
plt.xlabel("percent improvement")
plt.ylabel("Reconstruction Error")
plt.show()
test_data<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.10845175766641735, 'fraction_numerical': 0.013462976813762155, 'mean_word_length': 3.962894248608534, 'pattern_counts': {'":': 10, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '466626', 'license': 'MIT', 'n_tokens_mistral': 965, 'n_tokens_neox': 918, 'n_words': 202, 'path': 'notebooks/improvement_vs_reconstruction_err.ipynb', 'repo_name': 'scheng1992/Data_Assimilation', 'size': '2674'}