<jupyter_start><jupyter_text>Imports<jupyter_code>!pip install biotorch --upgrade
!git clone https://github.com/jsalbert/biotorch.git
import matplotlib.pyplot as plt
%matplotlib inline

import glob
import os
import pprint
import traceback
import numpy as np
import torch

import pandas as pd
from functools import cmp_to_key
from tensorboard.backend.event_processing.event_accumulator import EventAccumulator<jupyter_output><empty_output><jupyter_text>Extraction and plot functions<jupyter_code># Extraction function
def tflog2pandas(path: str) -> pd.DataFrame:
    """convert single tensorflow log file to pandas DataFrame
    Parameters
    ----------
    path : str
        path to tensorflow log file
    Returns
    -------
    pd.DataFrame
        converted dataframe
    """
    DEFAULT_SIZE_GUIDANCE = {
        "compressedHistograms": 1,
        "images": 1,
        "scalars": 0,  # 0 means load all
        "histograms": 1,
    }
    runlog_data = pd.DataFrame({"metric": [], "value": [], "step": []})
    try:
        event_acc = EventAccumulator(path, DEFAULT_SIZE_GUIDANCE)
        event_acc.Reload()
        tags = event_acc.Tags()["scalars"]
        for tag in tags:
            event_list = event_acc.Scalars(tag)
            values = list(map(lambda x: x.value, event_list))
            step = list(map(lambda x: x.step, event_list))
            r = {"metric": [tag] * len(step), "value": values, "step": step}
            r = pd.DataFrame(r)
            runlog_data = pd.concat([runlog_data, r])
    # Dirty catch of DataLossError
    except Exception:
        print("Event file possibly corrupt: {}".format(path))
        traceback.print_exc()
    return runlog_data
# Custom sorting and metrics selection 
# (This functions will need to be changed if using custom architectures)

def sorting_function(x1, x2):
    x1_s = x1.split('_')
    x2_s = x2.split('_')
    if int(x1_s[1]) < int(x2_s[1]):
        return -1
    elif int(x1_s[1]) > int(x2_s[1]):
        return 1
    else:
        if x1_s[0] <= x2_s[0]:
            return -1
        else:
            return 1

def get_layer_alignment(dir_logs, net='resnet'):
    layers_paths = [folder for folder in os.listdir(dir_logs)]
    event_paths = []
    layers_alignment = {}
    for l_p in layers_paths:
        if 'layer_alignment' in l_p:
            log_path = glob.glob(os.path.join(dir_logs, l_p, "event*"))
            if len(log_path) > 0: 
                df = tflog2pandas(log_path[0])
                layers_alignment[l_p.replace('layer_alignment_train_', '')] = df['value'].tolist()
    
    keys = list(layers_alignment.keys())
    keys = sorted(keys, key=cmp_to_key(sorting_function))
    if net == 'resnet':
        keys.remove('fc_0')
        keys.append('fc_0')
    elif net == 'lenet':
        keys.remove('fc1_0')
        keys.remove('fc2_0')
        keys.append('fc1_0')
        keys.append('fc2_0')
    return {key: layers_alignment[key] for key in keys}


def get_layer_weights(dir_logs, net='resnet', normalization=None):
    layers_paths = [folder for folder in os.listdir(dir_logs)]
    event_paths = []
    layers_alignment = {}
    for l_p in layers_paths:
        if 'weight_difference' in l_p:
            log_path = glob.glob(os.path.join(dir_logs, l_p, "event*"))
            if len(log_path) > 0: 
                df = tflog2pandas(log_path[0])
                layers_alignment[l_p.replace('weight_difference_train_', '')] = df['value'].tolist()
    
    keys = list(layers_alignment.keys())
    keys = sorted(keys, key=cmp_to_key(sorting_function))
    if net == 'resnet':
        keys.remove('fc_0')
        keys.append('fc_0')
    elif net == 'lenet':
        keys.remove('fc1_0')
        keys.remove('fc2_0')
        keys.append('fc1_0')
        keys.append('fc2_0')
    
    layer_weights = {key: layers_alignment[key] for key in keys}
    if normalization:
        count = 0
        for key, value in layer_weights.items():
            layer_weights[key] = np.array(value) * normalization[count]
            count += 1
                
    return layer_weights

def mkdir(path):
    if not os.path.exists(path):
        return os.makedirs(path)
    

def plot_multiple_lists(ydata, xdata, x_axis_name, y_axis_name, title, save_dir, figname, cmap='winter'):
    n = len(ydata)
    cmap_ = plt.cm.get_cmap(cmap)
    colors = iter(cmap_(np.linspace(0, 1, n)))
    colors_cmap = cmap_(np.arange(cmap_.N))
    
    # Using contourf to provide the colorbar info, then clearing the figure
    Z = [[0,0],[0,0]]
    levels = range(0, n, 1)
    CS3 = plt.contourf(Z, levels, cmap=cmap_)
    plt.clf()
    
    mkdir(save_dir)
    
    with plt.style.context('ggplot'):
        for name, data in ydata.items():
            plt.plot(xdata, data, color=next(colors))
        # plt.legend(tuple(list(ydata.keys())))
        
        plt.title(title, pad=10)
        plt.xlabel(x_axis_name)
        plt.ylabel(y_axis_name)
        cbar = plt.colorbar(CS3)
        cbar.set_label('Layer Depth', labelpad=10)
        
        # plt.xticks(xdata)
        plt.tight_layout() 
        plt.savefig('{}/{}.pdf'.format(save_dir, figname), dpi=200)
        plt.show()
        # plt.clf()<jupyter_output><empty_output><jupyter_text>Train a model for a few epochs<jupyter_code>import yaml
from biotorch.benchmark.run import Benchmark

# Open an example configuration file
config_path = "biotorch/benchmark_configs/cifar10/resnet_20/adam/fa_adam.yaml"
with open(config_path, 'r') as stream:
    try:
        file = yaml.safe_load(stream)
    except yaml.YAMLError as exc:
        print(exc)

# Deterministic won't work from Colab because we have to set CUDA environments
file['experiment']['deterministic'] = False
# Run only 15 epochs
file['training']['hyperparameters']['epochs'] = 15

with open(config_path, 'w') as outfile:
    yaml.dump(file, outfile) 
benchmark = Benchmark(config_path)
benchmark.run()<jupyter_output>Preparing CIFAR-10 Benchmark Dataset and storing data in ./datasets/cifar10
Downloading https://www.cs.toronto.edu/~kriz/cifar-10-python.tar.gz to ./datasets/cifar10/cifar-10-python.tar.gz<jupyter_text>Plot metrics<jupyter_code>logs_dir = 'trained_models/cifar10/resnet20/fa_adam/logs'
xdata = list(range(0, 16))
layers_alignment = get_layer_alignment(logs_dir)
plot_multiple_lists(layers_alignment, 
                    xdata, 
                    'Epoch', 
                    'Matrix Alignment (°)', 
                    'Weights Alignment', 
                    'plots/resnet20/fa', 
                    'fa_angles_resnet_20_sgd', 
                    'winter')

layers_alignment = get_layer_weights(logs_dir)
plot_multiple_lists(layers_alignment, 
                    xdata, 
                    'Epoch', 
                    'Weight Ratio  ' + r'($||B||_2 ~/~ ||W||_2$)', 
                    'Weight Norm Ratio', 
                    'plots/resnet20/fa', 
                    'fa_weights_resnet_20_sgd',  
                    'winter')<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.12335243553008596, 'fraction_numerical': 0.015472779369627506, 'mean_word_length': 2.589203084832905, 'pattern_counts': {'":': 10, '<': 18, '<?xml version=': 0, '>': 20, 'https://': 2, 'lorem ipsum': 0, 'www.': 1, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '191727', 'license': 'Apache-2.0', 'n_tokens_mistral': 2466, 'n_tokens_neox': 2329, 'n_words': 484, 'path': 'notebooks/accuracy_benchmark/metrics_visualizations.ipynb', 'repo_name': 'jsalbert/biotorch', 'size': '6980'}