<jupyter_start><jupyter_text>MNIST arch exploration<jupyter_code>%reload_ext autoreload
%autoreload 2
%matplotlib inline
from fastai.vision import *
from fastai.utils.mem import *
gpu_mem_get_free_no_cache()
path = untar_data(URLs.MNIST)
path.ls()
(path/'training').ls()
(path/'testing').ls()
data = ImageDataBunch.from_folder(path, train='training', valid='testing')
data.show_batch(3, figsize=(5,5))
# data.normalize()
data.one_batch()[0].shape
type(data)
1 + 1
def conv(i, o):
    return nn.Conv2d(i, o, kernel_size=3, stride=2, padding=1)
arch = nn.Sequential(
    conv(3, 6), # 14
#     nn.BatchNorm2d(6),
    nn.ReLU(),
    conv(6, 12), # 7
#     nn.BatchNorm2d(12),
    nn.ReLU(),
    conv(12, 24), # 4
#     nn.BatchNorm2d(24),
    nn.ReLU(),
    conv(24, 10), # 2
#     nn.BatchNorm2d(10),
    Flatten()
)
arch
learn = Learner(data, arch, loss_func=nn.CrossEntropyLoss(), metrics=accuracy)
learn.lr_find()
learn.recorder.plot()
learn.fit_one_cycle(1, 1e-1)
print(learn.summary())
arch = nn.Sequential(
    conv(3, 6), # 14
    nn.BatchNorm2d(6),
    nn.ReLU(),
    conv(6, 12), # 7
    nn.BatchNorm2d(12),
    nn.ReLU(),
    conv(12, 24), # 4
    nn.BatchNorm2d(24),
    nn.ReLU(),
    conv(24, 24), # 2
    nn.BatchNorm2d(24),
    nn.ReLU(),
    conv(24, 10), # 1
    nn.BatchNorm2d(10),
    Flatten()
)
learn = Learner(data, arch, loss_func=nn.CrossEntropyLoss(), metrics=accuracy)
learn.lr_find()
learn.recorder.plot()
learn.fit_one_cycle(1, 3e-1)
print(learn.summary())
arch = nn.Sequential(
    conv_layer(3, 6, stride=2), # 14
    conv_layer(6, 12, stride=2), # 7
    conv_layer(12, 24, stride=2), # 4
    conv_layer(24, 24, stride=2), # 2
    conv_layer(24, 10, stride=2), # 1
    Flatten()
)
learn = Learner(data, arch, loss_func=nn.CrossEntropyLoss(), metrics=accuracy)
learn.lr_find()
learn.recorder.plot()
learn.fit_one_cycle(1, 3e-1)
learn.fit_one_cycle(5, 3e-1)
learn.recorder.plot_losses()
learn.fit_one_cycle(5, 3e-2)
learn.recorder.plot_losses()
class ResBlock(nn.Module):
    def __init__(self, nf):
        super().__init__()
        self.conv1 = conv_layer(nf, nf)
        self.conv2 = conv_layer(nf, nf)
        
    def forward(self, x):
        return x + self.conv1(self.conv2(x))
arch = nn.Sequential(
    conv_layer(3, 6, stride=2), # 14
    ResBlock(6),
    conv_layer(6, 12, stride=2), # 7
    ResBlock(12),
    conv_layer(12, 24, stride=2), # 4
    ResBlock(24),
    conv_layer(24, 24, stride=2), # 2
    ResBlock(24),
    conv_layer(24, 10, stride=2), # 1
    ResBlock(10),
    Flatten()
)
learn = Learner(data, arch, loss_func=nn.CrossEntropyLoss(), metrics=accuracy)
learn.lr_find()
learn.recorder.plot()
learn.fit_one_cycle(1, 3e-1)
print(learn.summary())
learn.fit_one_cycle(5, 3e-2)
learn.fit_one_cycle(5, 3e-3)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.17277298850574713, 'fraction_numerical': 0.0610632183908046, 'mean_word_length': 3.964349376114082, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '637474', 'license': 'MIT', 'n_tokens_mistral': 1247, 'n_tokens_neox': 1143, 'n_words': 208, 'path': 'personal-learning/simple-practice/mnist-arch.ipynb', 'repo_name': 'wdoug/misc-ml', 'size': '2784'}