<jupyter_start><jupyter_text>Table of Contents1&nbsp;&nbsp;Comparing AE results with variable latent dimension  Comparing AE results with variable latent dimension<jupyter_code>import pickle
import os
import matplotlib.pyplot as plt
import numpy as np
import seaborn as sns
import pandas as pd
import matplotlib.pyplot as plt

results_dir = "/home/jfm1118/results/"
test_size = 327
train_size = 659
results_fps = os.listdir(results_dir)
sorted(results_fps)
def extract_test_train(ls, dim):
    """Helper function to extract the test and train fp of a given 
    dimension dim, from list ls"""
    test, train = None, None
    for fp in ls:
        text = fp.replace("epoch120.txt", "")
        split = text.split("_dim" + str(dim) + "_")
        if len(split) == 2:
            first, _ = split
            if first == "test":
                test = fp
            elif first == "train":
                train = fp
            else:
                raise ValueError("Should have a filename starting with \'train\' or \'test\'")
    if test and train:
        return train, test
    else:
        raise ValueError("No test or train fp present in ls")
def plot_dim(dim, ax):
    train, test = extract_test_train(results_fps, dim)
    
    with open(results_dir + train, 'rb') as fp:
        train = pickle.load(fp)
    with open(results_dir + test, 'rb') as fp:
        test = pickle.load(fp)
    epoch, data = list(zip(*test))
    data = np.array(data) / test_size
    axes = ax.plot(epoch, data, label = "test")
    
    epoch, data = list(zip(*train))
    data =  np.array(data) / train_size
    
    ax.plot(epoch, data, label = "train")
    ax.legend()
    ax.set_xlim([1, 50])
    ax.set_ylim([0, 800])
    return axes
extract_test_train(results_fps, 1)
latent_dims = [1, 2, 4, 10, 20, 40, 75, 100, 150, 200]
rows = 5
cols = 2
fig, ax =plt.subplots(rows, cols, sharex = True,sharey = True, figsize = (10, 20))

for idx, dim in enumerate(latent_dims):
    pos_x = idx % cols
    pos_y = idx // cols
    axes = ax[pos_y, pos_x]
    plot_dim(dim, axes)
    axes.set_title("Latent dimensions = {}".format(dim))
fig.show()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.11901441190144119, 'fraction_numerical': 0.02278010227801023, 'mean_word_length': 2.808849557522124, 'pattern_counts': {'":': 2, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '304766', 'license': 'MIT', 'n_tokens_mistral': 780, 'n_tokens_neox': 726, 'n_words': 222, 'path': 'notebooks/AE_results.ipynb', 'repo_name': 'scheng1992/Data_Assimilation', 'size': '2151'}