<jupyter_start><jupyter_text>Analysis<jupyter_code>from glob import glob
import seaborn as sns
sns.set(style="darkgrid")
result_files = sorted(glob("logs/0602/0602-q1_adam*.train"))
result_files = sorted(glob("/home/jok120/Desktop/safekeeping/0602-q3_adam-0457c.train"))
result_files[0]
dfs = [pd.read_csv(f) for f in result_files]
titles = [f[10:-6] for f in result_files]
dfes = []
for df in dfs:
    dfes.append(df[df["is_end_of_epoch"]].reset_index())
!pwd
def do_plot(df, title):
    dfval = df[df["is_val"]]
    dftrain = df[df["is_val"] != True]
    sns.lineplot(x=dftrain.index, y="drmsd", data=dftrain, label="train-drmsd")
    sns.lineplot(x=dfval.index, y="drmsd", data=dfval, label="val-drmsd",color="lightblue")
    sns.lineplot(x=dftrain.index, y="rmsd", data=dftrain, label="rmsd")
    sns.lineplot(x=dftrain.index, y="rmse", data=dftrain, label="rmse")
    sns.lineplot(x=dftrain.index, y="combined", data=dftrain, label="drmsd+mse")
    plt.ylabel("Loss Value")
    plt.xlabel("Epoch")
    plt.legend(loc=(1.04,.7))
    plt.title("Transformer Training Loss".format(title))
    plt.savefig("figs/transtrain.pdf", pad_inches=1, bbox_inches="tight")
do_plot(dfes[0], titles[0])
dfes[0][dfes[0]["is_val"]]
mins = []
for df, title in zip(dfes, titles):
    try:
        row = df[df["combined"] == df["combined"].min()]
    except KeyError:
        print(title)
        continue
    row["title"] = title[:]
    mins.append(row)
mins_df = pd.concat(mins)
mins_df.sort_values("combined", inplace=True)
mins_df
names =  [int(t[-3:]) for t in mins_df["title"][:10]]
print(names)
for n in names:
    do_plot(dfes[n], titles[n])
    plt.show()
import re
def get_arg(namespace, arg):
    pattern = f"{arg}=(.+?),"
    return re.search(pattern, namespace).group(1)
for row in mins:
    t = row["title"].item()
    with open(f"logs/0602/{t}.log", "r") as f:
        args = f.readline()
    for a in ["d_model","d_k","d_v","n_head","n_layers"]:
        row[a] = get_arg(args, a)
mins = pd.concat(mins)
mins.sort_values("combined", inplace=True)
mins
sorted(list(map(int,set(mins["d_model"]))))
def do_loss_plots_on_var(d, var, swarm=True):
    sns.boxplot(x=var, y="combined", data=d)
    if swarm: sns.swarmplot(x=var, y="combined", data=d, color="black")
    plt.figure()
    sns.boxplot(x=var, y="rmsd", data=d)
    if swarm: sns.swarmplot(x=var, y="rmsd", data=d, color="black")
    plt.figure()
    sns.boxplot(x=var, y="rmse", data=d)
    if swarm: sns.swarmplot(x=var, y="rmse", data=d, color="black")

do_loss_plots_on_var(mins, "d_model", swarm=False)
do_loss_plots_on_var(mins, "d_k", swarm=False)
do_loss_plots_on_var(mins, "d_v", swarm=False)
do_loss_plots_on_var(mins, "n_head", swarm=False)
do_loss_plots_on_var(mins, "n_layers", swarm=False)
summary = mins[["title", "combined", "drmsd", "rmsd", "rmse", "d_model","d_k","d_v","n_head","n_layers"]]
" ".join(summary["title"][-1:].to_list())
summary<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.19741760108732587, 'fraction_numerical': 0.014610941216445803, 'mean_word_length': 5.3585313174946005, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '251939', 'license': 'BSD-3-Clause', 'n_tokens_mistral': 1279, 'n_tokens_neox': 1250, 'n_words': 204, 'path': 'research/notebooks/pre190620/19-06-02 Q3 Hyperparameter Search, vanilla adam.ipynb', 'repo_name': 'hengwei-chan/protein_transformer', 'size': '2943'}