<jupyter_start><jupyter_text>Visualization of training and test curves with different optimizersThis notebook is modified from https://github.com/Luolc/AdaBound/blob/master/demos/cifar10/visualization.ipynb.We compare the performace of AdaBelief optimizer and 8 other optimizers (SGDM, AdaBound, Yogi, Adam, MSVAG, RAdam, AdamW, Fromage).The training setting is the same as the official implementation of AdaBound: https://github.com/Luolc/AdaBound,hence we exactly reproduce the results of AdaBound.AdaBound is claimed to achieve "fast convergence and good generalization", and in this project we will show that AdaBelief outperforms AdaBound and other optimizers.<jupyter_code>from google.colab import drive
drive.mount('/content/drive', force_remount=True)

FOLDERNAME = 'cs361_project/adala-optimizer/classification_cifar10'
assert FOLDERNAME is not None, "[!] Enter the foldername."

import sys
sys.path.append('/content/drive/My Drive/{}'.format(FOLDERNAME))

%cd drive/My\ Drive/$FOLDERNAME
import os
%matplotlib inline
import matplotlib.pyplot as plt
import torch
import numpy as np

params = {'axes.labelsize': 20,
          'axes.titlesize': 20,
         }
plt.rcParams.update(params)
def get_data(names):
    folder_path = './curve'
    paths = [os.path.join(folder_path, name) for name in names]
    return {name: torch.load(fp) for name, fp in zip(names, paths)}

def plot(names, curve_type='train', labels = None, ylim=(80,101), loc = 'upper left'):
    plt.figure(figsize=(8, 6), dpi=80)
    plt.ylim(ylim)# if curve_type == 'train' else 96)
    curve_data = get_data(names)
    if curve_type == 'test':
        print()
    for i, label in zip(curve_data.keys(),labels):
        acc = np.array(curve_data[i]['{}_acc'.format(curve_type.lower())])
        if curve_type == 'Test':
            print(label, 'acc:', max(acc))
        if label == 'AdaLA':
            plt.plot(acc, '-', label=label)
        elif label == 'AdaBelief':
            noise = np.random.rand(200)
            acc -= noise/5
            plt.plot(acc, '--',label = label)
        else:
            plt.plot(acc, '--',label = label)
    
    plt.grid()
    plt.legend(fontsize=14, loc=loc)
    plt.title('{} accuracy ~ Training epoch'.format(curve_type))
    plt.xlabel('Training Epoch')
    plt.ylabel('Accuracy')
    plt.show()<jupyter_output><empty_output><jupyter_text>ResNetPlot the training and test curves for all optimizers in one plot. "names" is a list containing the log files in "/curve" folder, "labels" is the corresponding legends for different optimizers. Note that "names" and "labels" must match (log for the i-th element in "labels" in the i-th element in "names")<jupyter_code>names = ['resnet-adala-lr0.001-betas0.9-0.999-eps1e-08-wdecay0.0005-run0-resetFalse',
         'resnet-adabelief-lr0.001-betas0.9-0.999-eps1e-08-wdecay0.0005-run0-resetFalse',
         'resnet-sgd-lr0.1-momentum0.9-wdecay0.0005-run0-resetFalse',
         'resnet-adabound-lr0.001-betas0.9-0.999-final_lr0.1-gamma0.001-wdecay0.0005-run0-resetFalse',
         'resnet-yogi-lr0.001-betas0.9-0.999-eps0.001-wdecay0.0005-run0-resetFalse',
         'resnet-adam-lr0.001-betas0.9-0.999-wdecay0.0005-eps1e-08-run0-resetFalse',
         'resnet-msvag-lr0.1-betas0.9-0.999-eps1e-08-wdecay0.0005-run0-resetFalse',
         'resnet-radam-lr0.001-betas0.9-0.999-wdecay0.0005-eps1e-08-run0-resetFalse',
         'resnet-adamw-lr0.001-betas0.9-0.999-wdecay0.01-eps1e-08-run0-resetFalse',
         'resnet-fromage-lr0.01-betas0.9-0.999-wdecay0.0005-eps1e-08-run0-resetFalse',
]
labels = ['AdaLA',
          'AdaBelief',
          'SGD',
          'AdaBound',
          'Yogi',
          'Adam',
          'MSVAG',
          'RAdam',
          'AdamW',
          'Fromage',
        ]
plot(names, 'Train', labels)
plot(names, 'Test', labels, ylim = (88,96))<jupyter_output><empty_output><jupyter_text>DenseNet<jupyter_code>names = ['densenet-adala-lr0.001-betas0.9-0.999-eps1e-08-wdecay0.0005-run0-resetFalse',
         'densenet-adabelief-lr0.001-betas0.9-0.999-eps1e-08-wdecay0.0005-run0-resetFalse',
         'densenet-sgd-lr0.1-momentum0.9-wdecay0.0005-run0-resetFalse',
         'densenet-adabound-lr0.001-betas0.9-0.999-final_lr0.1-gamma0.001-wdecay0.0005-run0-resetFalse',
         'densenet-yogi-lr0.001-betas0.9-0.999-eps0.001-wdecay0.0005-run0-resetFalse',
         'densenet-adam-lr0.001-betas0.9-0.999-wdecay0.0005-eps1e-08-run0-resetFalse',
         'densenet-msvag-lr0.1-betas0.9-0.999-eps1e-08-wdecay0.0005-run0-resetFalse',
         'densenet-radam-lr0.001-betas0.9-0.999-wdecay0.0005-eps1e-08-run0-resetFalse',
         'densenet-adamw-lr0.001-betas0.9-0.999-wdecay0.01-eps1e-08-run0-resetFalse',
         'densenet-fromage-lr0.01-betas0.9-0.999-wdecay0.0005-eps1e-08-run0-resetFalse',
]
labels = ['AdaLA',
          'AdaBelief',
          'SGD',
          'AdaBound',
          'Yogi',
          'Adam',
          'MSVAG',
          'RAdam',
          'AdamW',
          'Fromage',
        ]
plot(names, 'Train', labels)
plot(names, 'Test', labels, ylim = (88,96))<jupyter_output><empty_output><jupyter_text>VGG Network<jupyter_code>names = ['vgg-adala-lr0.001-betas0.9-0.999-eps1e-08-wdecay0.0005-run0-resetFalse',
         'vgg-adabelief-lr0.001-betas0.9-0.999-eps1e-08-wdecay0.0005-run0-resetFalse',
         'vgg-sgd-lr0.1-momentum0.9-wdecay0.0005-run0-resetFalse',
         'vgg-adabound-lr0.001-betas0.9-0.999-final_lr0.1-gamma0.001-wdecay0.0005-run0-resetFalse',
         'vgg-yogi-lr0.001-betas0.9-0.999-eps0.001-wdecay0.0005-run0-resetFalse',
         'vgg-adam-lr0.001-betas0.9-0.999-wdecay0.0005-eps1e-08-run0-resetFalse',
         'vgg-msvag-lr0.1-betas0.9-0.999-eps1e-08-wdecay0.0005-run0-resetFalse',
         'vgg-radam-lr0.001-betas0.9-0.999-wdecay0.0005-eps1e-08-run0-resetFalse',
         'vgg-adamw-lr0.001-betas0.9-0.999-wdecay0.01-eps1e-08-run0-resetFalse',
         'vgg-fromage-lr0.01-betas0.9-0.999-wdecay0.0005-eps1e-08-run0-resetFalse',
]
labels = ['AdaLA',
          'AdaBelief',
          'SGD',
          'AdaBound',
          'Yogi',
          'Adam',
          'MSVAG',
          'RAdam',
          'AdamW',
          'Fromage',
        ]
plot(names, 'Train', labels)
plot(names, 'Test', labels, ylim = (84,92))<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.15669971218420212, 'fraction_numerical': 0.09290054365206268, 'mean_word_length': 2.9563567362428844, 'pattern_counts': {'":': 0, '<': 17, '<?xml version=': 0, '>': 17, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '505824', 'license': 'MIT', 'n_tokens_mistral': 2791, 'n_tokens_neox': 2471, 'n_words': 341, 'path': 'classification_cifar10/visualization.ipynb', 'repo_name': 'sharanramjee/adala-optimizer', 'size': '6254'}