<jupyter_start><jupyter_text>[01/03/22] Compare Gradient Norms Over First 200 Steps: Random vs Diet Data<jupyter_code>import sys
sys.path = ['/home/mansheej/open_lth'] + sys.path
import datasets.registry
from foundations import hparams, step
import matplotlib.pyplot as plt
import models.registry
import numpy as np
import pandas as pd
from pathlib import Path
import seaborn as sns
import torch
from torch import optim
import torch.nn.functional as F
from tqdm import tqdm
plt.style.use('default')
sns.set_theme(
    style='ticks',
    font_scale=1.2,
    rc={
        'axes.linewidth': '0.8',
        'axes.grid': True,
        'figure.constrained_layout.use': True,
        'grid.linewidth': '0.8',
        'legend.edgecolor': '1.0',
        'legend.fontsize': 'small',
        'legend.title_fontsize': 'small',
        'xtick.major.width': '0.8',
        'ytick.major.width': '0.8'
    },
)
grad_norms_alls = []
for _ in range(4):
    dataset_hparams = hparams.DatasetHparams(dataset_name='cifar10', batch_size=128)
    data = datasets.registry.get(dataset_hparams=dataset_hparams, train=True)
    model_hparams = hparams.ModelHparams(
        model_name='cifar_resnet_20', model_init='kaiming_normal', batchnorm_init='uniform'
    )
    model = models.registry.get(model_hparams=model_hparams, outputs=10)
    model = model.cuda()
    model = model.train()
    optimizer = optim.SGD(model.parameters(), lr=0.4, momentum=0.9, weight_decay=0.0001)
    grad_norms = []
    data_iter = iter(data)
    for step in tqdm(range(200)):
        inputs, targets = next(data_iter)
        inputs, targets = inputs.cuda(), targets.cuda()
        optimizer.zero_grad()
        logits = model(inputs)
        loss = model.loss_criterion(logits, targets)
        loss.backward()
        with torch.no_grad():
            grad = torch.cat([param.grad.view(-1) for param in model.parameters()])
            grad_norms.append(torch.linalg.norm(grad).item())
        optimizer.step()
    grad_norms_alls.append(np.array(grad_norms))
grad_norms_diets = []
for _ in range(4):
    dataset_hparams = hparams.DatasetHparams(
        dataset_name='cifar10', 
        batch_size=128, 
        score_path='error_norm/cifar10_resnet20_3ep_10runs.npy', 
        subset_size=25600, 
        offset=0,
    )
    data = datasets.registry.get(dataset_hparams=dataset_hparams, train=True)
    model_hparams = hparams.ModelHparams(
        model_name='cifar_resnet_20', model_init='kaiming_normal', batchnorm_init='uniform'
    )
    model = models.registry.get(model_hparams=model_hparams, outputs=10)
    model = model.cuda()
    model = model.train()
    optimizer = optim.SGD(model.parameters(), lr=0.4, momentum=0.9, weight_decay=0.0001)
    grad_norms = []
    data_iter = iter(data)
    for step in tqdm(range(200)):
        inputs, targets = next(data_iter)
        inputs, targets = inputs.cuda(), targets.cuda()
        optimizer.zero_grad()
        logits = model(inputs)
        loss = model.loss_criterion(logits, targets)
        loss.backward()
        with torch.no_grad():
            grad = torch.cat([param.grad.view(-1) for param in model.parameters()])
            grad_norms.append(torch.linalg.norm(grad).item())
        optimizer.step()
    grad_norms_diets.append(np.array(grad_norms))<jupyter_output><empty_output><jupyter_text>Figure 0017<jupyter_code>plt.figure()
plt.hist(
    np.concatenate(grad_norms_alls), 
    bins=np.linspace(0, 3, 31), 
    density=True, 
    histtype='stepfilled', 
    alpha=0.75,
    label='25600 Random Examples',
    ec='C0',
    lw=2,
)
plt.hist(
    np.concatenate(grad_norms_diets), 
    bins=np.linspace(0, 3, 31), 
    density=True, 
    histtype='stepfilled', 
    alpha=0.75,
    label='25600 Smallest Scores at Epoch 3',
    ec='C1',
    lw=2,
)
plt.legend()
plt.grid(axis='x')
plt.xlim(0, 3)
sns.despine()
plt.xlabel('Gradient Norm')
plt.ylabel('Density')
plt.title('CIFAR10 ResNet20: First 200 Steps (Aggregate 4 Runs)')
plt.savefig('/home/mansheej/open_lth/figs/0017.svg')
plt.show()<jupyter_output><empty_output><jupyter_text>Figure 0018<jupyter_code>plt.figure()
plt.plot(np.stack(grad_norms_alls).mean(0), label='25600 Random Examples')
plt.plot(np.stack(grad_norms_diets).mean(0), label='25600 Smallest Scores at Epoch 3')
plt.legend()
sns.despine()
plt.xlim(0, 200)
plt.xlabel('Step')
plt.ylabel('Mean Gradient Norm')
plt.title('CIFAR10 ResNet20: Gradient Norm (Mean of 4 Runs)')
plt.savefig('/home/mansheej/open_lth/figs/0018.svg')
plt.show()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1379539415411869, 'fraction_numerical': 0.034765279007971654, 'mean_word_length': 3.6808290155440413, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '612774', 'license': 'MIT', 'n_tokens_mistral': 1759, 'n_tokens_neox': 1621, 'n_words': 282, 'path': 'nbs/22_01_03__Gradient_Norm_200_Steps_Random_v_Diet.ipynb', 'repo_name': 'mansheej/open_lth', 'size': '4516'}