<jupyter_start><jupyter_text>Experiment Tutorial<jupyter_code>import sys
sys.path.insert(0, '/home/hongky/project/model_compress/shrinkbench/')
from experiment import PruningExperiment
import pruning
# import importlib
# importlib.reload(pruning)<jupyter_output><empty_output><jupyter_text>The `DATAPATH` and `WEIGHTSPATH` environment variables are used to tell the framework where to look for datasets and pretrained weights respectively.<jupyter_code>import os
os.environ['DATAPATH'] = '/home/hongky/project/model_compress/shrinkbench/data/'
#os.environ['WEIGHTSPATH'] = '/home/kylee/work/projects/7_model_compress/2_shrinkbench/pretrained/mnist/mnistnet.pt'
import torchvision
root = '/home/hongky/project/model_compress/shrinkbench/data'
torchvision.datasets.MNIST(os.environ['DATAPATH'], train=True, transform=None, target_transform=None, download=True)
from IPython.display import clear_output
clear_output()
import pathlib
import pydantic.json

pydantic.json.ENCODERS_BY_TYPE[pathlib.PosixPath] = str
pydantic.json.ENCODERS_BY_TYPE[pathlib.WindowsPath] = str


from experiment import TrainingExperiment
run_path = '/home/hongky/project/model_compress/shrinkbench/pretrained/mnist'
p = pathlib.Path(run_path)
train_exp = TrainingExperiment(
                        dataset='MNIST', 
                        model='MnistNet', 
                        path=run_path, 
                        train_kwargs={'epochs':11}
                        )
train_exp.run()


import torch 
save_model_path = '/home/hongky/project/model_compress/shrinkbench/pretrained/mnist/mnistnet.pt'
torch.save(train_exp.model.state_dict(), save_model_path)<jupyter_output><empty_output><jupyter_text>We run experiments for our MNIST network for logarithmically spaced compression ratios<jupyter_code>model_path = '/home/hongky/project/model_compress/shrinkbench/jupyter/results/'
#os.environ['WEIGHTSPATH'] = model_path
for strategy in ['RandomPruning', 'GlobalMagWeight', 'LayerMagWeight']:
    for  c in [1,2,4,8,16,32,64]:
        exp = PruningExperiment(dataset='MNIST', 
                                model='MnistNet',
                                strategy=strategy,
                                compression=c,
                                path=model_path + '_' + strategy + '_' + str(c),
                                train_kwargs={'epochs':11})
        clear_output()
        exp.run()<jupyter_output>[33mRunning {
    "save_freq": 10,
    "resume_optim": false,
    "resume": null,
    "pretrained": true,
    "debug": false,
    "train_kwargs": {
        "optim": "SGD",
        "epochs": 11,
        "lr": 0.001
    },
    "dl_kwargs": {
        "batch_size": 128,
        "pin_memory": false,
        "num_workers": 8
    },
    "path": "/home/hongky/project/model_compress/shrinkbench/jupyter/results/_LayerMagWeight_64",
    "seed": 42,
    "model": "MnistNet",
    "dataset": "MNIST",
    "strategy": "LayerMagWeight",
    "compression": 64
}
[0m[95mLogging results to /home/hongky/project/model_compress/shrinkbench/jupyter/results/_LayerMagWeight_64
[0mType-path:: <class 'str'><jupyter_text>We then collect output from experiment folders and plot the diferent metrics easily<jupyter_code>from plot import df_from_results, plot_df
result_path = '/home/hongky/project/model_compress/shrinkbench/jupyter/results'
df = df_from_results(result_path)<jupyter_output><empty_output><jupyter_text>With the provided functions, it is easy to generate plots<jupyter_code>plot_df(df, 'compression', 'pre_acc5', markers='strategy', line='--', colors='strategy', suffix=' - pre')
plot_df(df, 'compression', 'post_acc5', markers='strategy', fig=False, colors='strategy')<jupyter_output><empty_output><jupyter_text>We can also check the theoretical speedup and see that layerwise provides larger FLOPS speedups because of the even pruning of the conv layers<jupyter_code>from matplotlib import pyplot as plt
import numpy as np
plot_df(df, 'speedup', 'post_acc5', colors='strategy', markers='strategy')
# plt.yscale('log')
# plt.ylim(0.996,0.9995)
# plt.xticks(2**np.arange(7))
# plt.gca().set_xticklabels(map(str, 2**np.arange(7)))
# None<jupyter_output><empty_output><jupyter_text>We can easily check if the compression is matching our expectation by looking at the relative error. As expected, random pruning does worst.<jupyter_code>df['compression_err'] = (df['real_compression'] - df['compression'])/df['compression']
plot_df(df, 'compression', 'compression_err', colors='strategy', markers='strategy')<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.12400706090026478, 'fraction_numerical': 0.013018534863195057, 'mean_word_length': 4.295560747663552, 'pattern_counts': {'":': 19, '<': 29, '<?xml version=': 0, '>': 29, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '7', 'dir': 'jupyter-structured-clean-dedup', 'id': '453509', 'license': 'MIT', 'n_tokens_mistral': 1544, 'n_tokens_neox': 1479, 'n_words': 274, 'path': 'jupyter/experiment_tutorial.ipynb', 'repo_name': 'kyhoolee/shrinkbench', 'size': '4532'}