<jupyter_start><jupyter_text>August 29 - Training on 9M dataset metrics 2  Final qualitative metric comparison - Delta difference and its uncertainity<jupyter_code># Default imports
import math
import os
import sys
import pandas as pd
import numpy as np

import matplotlib as mpl
import matplotlib.pyplot as plt
from matplotlib.colors import LogNorm
import matplotlib.ticker as mtick

# Add the path to the parent directory to augment search for module
par_dir = os.path.abspath(os.path.join(os.getcwd(), os.pardir))
if par_dir not in sys.path:
    sys.path.append(par_dir)
    
# Import the custom plotting module
from plot_utils import plot_utils
import random
import torch
from plot_utils import notebook_utils_2<jupyter_output><empty_output><jupyter_text>Method : Construct 5 dataframes ( one of each number of latent dimensions ), use the dataset index as the index in the dataframe, sort the dataframes, and then compare the values.<jupyter_code>latent_dims = [16, 32, 64, 128, 256]
dumps = ["20190829_010238", "20190829_010252", "20190829_010339", "20190829_010405", "20190829_010431"]

dump_dfs = []

# First check that all the indices from the test validation set exist in all the dumps

ldump_idx_arr = None

# Iterate over the dumps and check the indices
for latent_dim, dump in zip(latent_dims, dumps):
    print("----------------------------------------------------")
    print("Reading metrics from VAE with {0} latent dimensions :".format(latent_dim))
    print("----------------------------------------------------")
    
    dump_npz_path = "/home/akajal/WatChMaL/VAE/dumps/{0}/test_valid_iteration_metrics.npz".format(dump)
    dump_npz_arr = np.load(dump_npz_path)
    
    dump_dfs.append(pd.DataFrame(data={"index":dump_npz_arr["indices"], "recon_loss":dump_npz_arr["recon_loss"],
                                       "kl_loss":dump_npz_arr["kl_loss"]}))
print(dump_dfs[0])<jupyter_output><empty_output><jupyter_text>Sort the dataframes using the dataset index as the index<jupyter_code>for df in dump_dfs:
    df.sort_values(by="index", inplace=True)
print(dump_dfs[0])
print(dump_dfs[1])<jupyter_output><empty_output><jupyter_text>Now that we have sorted dfs, find the delta difference for each of the values w.r.t. the reference df<jupyter_code>for df in dump_dfs:
    df["delta recon_loss"] = df["recon_loss"].values - dump_dfs[3]["recon_loss"].values
    df["delta kl_loss"] = df["kl_loss"].values - dump_dfs[3]["kl_loss"].values
print(dump_dfs[0])
print(dump_dfs[3])
print(dump_dfs[0]["delta recon_loss"].values.shape[0])<jupyter_output><empty_output><jupyter_text>For each of the latent dimensions and their corresponding df's, find the mean and stderr of the delta values and plot<jupyter_code>delta_recon_mean, delta_kl_mean, delta_recon_stderr, delta_kl_stderr = [], [], [], []

for df in dump_dfs:
    delta_recon_loss, delta_kl_loss = df["delta recon_loss"], df["delta kl_loss"]
    
    delta_recon_mean.append(np.mean(delta_recon_loss.values))
    delta_kl_mean.append(np.mean(delta_kl_loss.values))
    
    delta_recon_stderr.append(np.std(delta_recon_loss.values)/math.sqrt(df["delta recon_loss"].values.shape[0]))
    delta_kl_stderr.append(np.std(delta_kl_loss.values)/math.sqrt(df["delta kl_loss"].values.shape[0]))
print("delta recon mean :", delta_recon_mean)
print("delta recon stderr :", delta_recon_stderr)
print("delta kl mean :", delta_kl_mean)
print("delta kl stderr :", delta_kl_stderr)<jupyter_output><empty_output><jupyter_text>Plot the metrics collected above<jupyter_code># Plot the metrics for the training subset

# Initialize the plot
fig, ax1 = plt.subplots(figsize=(16,9))

# Set the x-axes ticks for the plot
ax1.set_xticks(latent_dims)

# Use the same x-axis to plot the KL loss
ax2 = ax1.twinx()

# Plot the MSE values collected above
ax1.errorbar(latent_dims, delta_recon_mean, yerr=delta_recon_stderr, linestyle='dashed', marker='o',
             markersize=10, color="blue", label=r"Mean $\Delta$ MSE loss")

# Plot the KL values collected above
ax2.errorbar(latent_dims, delta_kl_mean, yerr=delta_kl_stderr, linestyle='dashed', marker='o',
             markersize=10, color="red", label=r"Mean $\Delta$ KL loss")

# Setup plot characteristics
ax1.tick_params(axis="x", labelsize=30)
ax1.set_xlabel("Number of latent dimensions", fontsize=30)
ax1.set_xscale("log", basex=2)

ax1.set_ylabel(r"$\Delta$ MSE loss", fontsize=30, color="blue")
ax1.tick_params(axis="y", labelsize=30, colors="blue")

ax2.set_ylabel(r"$\Delta$ KL loss", fontsize=30, color="red")
ax2.tick_params(axis="y", labelsize=30, colors="red")

plt.margins(0.2)
ax1.grid(True)
ax2.grid(True)
plt.title(r"$\Delta$ Per-sample Loss vs Latent dimensions", fontsize=30)

lgd = fig.legend(prop={"size":25}, loc='center', bbox_to_anchor=(0.5, 0.8))
plt.savefig("vae_latent_dimension_comparison.svg", format="svg", dpi=300)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1410125025620004, 'fraction_numerical': 0.03053904488624718, 'mean_word_length': 5.1, 'pattern_counts': {'":': 4, '<': 25, '<?xml version=': 0, '>': 25, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '6', 'dir': 'jupyter-structured-clean-dedup', 'id': '372439', 'license': 'CNRI-Python', 'n_tokens_mistral': 1786, 'n_tokens_neox': 1685, 'n_words': 398, 'path': 'notebooks/notebooks_archive/August 29 - Training on 9M dataset metrics 2.ipynb', 'repo_name': 'WatChMaL/CNN', 'size': '4879'}