<jupyter_start><jupyter_text>import modules<jupyter_code>import torch
import torch.nn as nn
import torch.nn.functional as F
import torch.optim as optim
from torch.utils.data import DataLoader
from torchvision import datasets, transforms

import matplotlib.pyplot as plt
import numpy as np
np.random.seed(123)
torch.manual_seed(123)<jupyter_output><empty_output><jupyter_text>define model architecture<jupyter_code>class ConvNet(nn.Module):
    def __init__(self):
        super(ConvNet, self).__init__()
        self.cn1 = nn.Conv2d(1, 16, 3, 1)
        self.cn2 = nn.Conv2d(16, 32, 3, 1)
        self.dp1 = nn.Dropout2d(0.10)
        self.dp2 = nn.Dropout2d(0.25)
        self.fc1 = nn.Linear(4608, 64) # 4608 is basically 12 X 12 X 32
        self.fc2 = nn.Linear(64, 10)
 
    def forward(self, x):
        x = self.cn1(x)
        x = F.relu(x)
        x = self.cn2(x)
        x = F.relu(x)
        x = F.max_pool2d(x, 2)
        x = self.dp1(x)
        x = torch.flatten(x, 1)
        x = self.fc1(x)
        x = F.relu(x)
        x = self.dp2(x)
        x = self.fc2(x)
        op = F.log_softmax(x, dim=1)
        return op<jupyter_output><empty_output><jupyter_text>define training and inference routines<jupyter_code>def train(model, device, train_dataloader, optim, epoch):
    model.train()
    for b_i, (X, y) in enumerate(train_dataloader):
        X, y = X.to(device), y.to(device)
        optim.zero_grad()
        pred_prob = model(X)
        loss = F.nll_loss(pred_prob, y) # nll is the negative likelihood loss
        loss.backward()
        optim.step()
        if b_i % 10 == 0:
            print('epoch: {} [{}/{} ({:.0f}%)]\t training loss: {:.6f}'.format(
                epoch, b_i * len(X), len(train_dataloader.dataset),
                100. * b_i / len(train_dataloader), loss.item()))
def test(model, device, test_dataloader):
    model.eval()
    loss = 0
    success = 0
    with torch.no_grad():
        for X, y in test_dataloader:
            X, y = X.to(device), y.to(device)
            pred_prob = model(X)
            loss += F.nll_loss(pred_prob, y, reduction='sum').item()  # loss summed across the batch
            pred = pred_prob.argmax(dim=1, keepdim=True)  # us argmax to get the most likely prediction
            success += pred.eq(y.view_as(pred)).sum().item()

    loss /= len(test_dataloader.dataset)

    print('\nTest dataset: Overall Loss: {:.4f}, Overall Accuracy: {}/{} ({:.0f}%)\n'.format(
        loss, success, len(test_dataloader.dataset),
        100. * success / len(test_dataloader.dataset)))<jupyter_output><empty_output><jupyter_text>create data loaders<jupyter_code># The mean and standard deviation values are calculated as the mean of all pixel values of all images in the training dataset
train_dataloader = torch.utils.data.DataLoader(
    datasets.MNIST('../data', train=True, download=True,
                   transform=transforms.Compose([
                       transforms.ToTensor(),
                       transforms.Normalize((0.1302,), (0.3069,))])), # train_X.mean()/256. and train_X.std()/256.
    batch_size=32, shuffle=True)

test_dataloader = torch.utils.data.DataLoader(
    datasets.MNIST('../data', train=False, 
                   transform=transforms.Compose([
                       transforms.ToTensor(),
                       transforms.Normalize((0.1302,), (0.3069,)) 
                   ])),
    batch_size=500, shuffle=True)<jupyter_output><empty_output><jupyter_text>define optimizer and run training epochs<jupyter_code>device = torch.device("cpu")

model = ConvNet()
optimizer = optim.Adadelta(model.parameters(), lr=0.5)<jupyter_output><empty_output><jupyter_text>model training<jupyter_code>for epoch in range(1, 20):
    train(model, device, train_dataloader, optimizer, epoch)
    test(model, device, test_dataloader)<jupyter_output>epoch: 1 [0/60000 (0%)]	 training loss: 2.324445
epoch: 1 [320/60000 (1%)]	 training loss: 1.727462
epoch: 1 [640/60000 (1%)]	 training loss: 1.428922
epoch: 1 [960/60000 (2%)]	 training loss: 0.717944
epoch: 1 [1280/60000 (2%)]	 training loss: 0.572199
epoch: 1 [1600/60000 (3%)]	 training loss: 0.847963
epoch: 1 [1920/60000 (3%)]	 training loss: 0.542137
epoch: 1 [2240/60000 (4%)]	 training loss: 0.384015
epoch: 1 [2560/60000 (4%)]	 training loss: 0.228308
epoch: 1 [2880/60000 (5%)]	 training loss: 0.645798
epoch: 1 [3200/60000 (5%)]	 training loss: 0.228496
epoch: 1 [3520/60000 (6%)]	 training loss: 0.163134
epoch: 1 [3840/60000 (6%)]	 training loss: 0.362703
epoch: 1 [4160/60000 (7%)]	 training loss: 0.564533
epoch: 1 [4480/60000 (7%)]	 training loss: 0.438789
epoch: 1 [4800/60000 (8%)]	 training loss: 0.337578
epoch: 1 [5120/60000 (9%)]	 training loss: 0.147629
epoch: 1 [5440/60000 (9%)]	 training loss: 0.138111
epoch: 1 [5760/60000 (10%)]	 training loss: 0.365426
epoch: 1 [6080/60[...]<jupyter_text>run inference on trained model<jupyter_code>test_samples = enumerate(test_dataloader)
b_i, (sample_data, sample_targets) = next(test_samples)

plt.imshow(sample_data[0][0], cmap='gray', interpolation='none')
plt.show()
print(f"Model prediction is : {model(sample_data).data.max(1)[1][0]}")
print(f"Ground truth is : {sample_targets[0]}")<jupyter_output>Model prediction is : 9
Ground truth is : 9<jupyter_text>visualize filters<jupyter_code>model_children_list = list(model.children())
convolutional_layers = []
model_parameters = []
model_children_list
for i in range(len(model_children_list)):
    if type(model_children_list[i]) == nn.Conv2d:
        model_parameters.append(model_children_list[i].weight)
        convolutional_layers.append(model_children_list[i])
plt.figure(figsize=(5, 4))
for i, flt in enumerate(model_parameters[0]):
    plt.subplot(4, 4, i+1)
    plt.imshow(flt[0, :, :].detach(), cmap='gray')
    plt.axis('off')
plt.show()
plt.figure(figsize=(5, 8))
for i, flt in enumerate(model_parameters[1]):
    plt.subplot(8, 4, i+1)
    plt.imshow(flt[0, :, :].detach(), cmap='gray')
    plt.axis('off')
plt.show()<jupyter_output><empty_output><jupyter_text>visualize feature maps<jupyter_code>per_layer_results = [convolutional_layers[0](sample_data)]
for i in range(1, len(convolutional_layers)):
    per_layer_results.append(convolutional_layers[i](per_layer_results[-1]))
plt.figure(figsize=(5, 4))
layer_visualisation = per_layer_results[0][0, :, :, :]
layer_visualisation = layer_visualisation.data
print(layer_visualisation.size())
for i, flt in enumerate(layer_visualisation):
    plt.subplot(4, 4, i + 1)
    plt.imshow(flt, cmap='gray')
    plt.axis("off")
plt.show()
plt.figure(figsize=(5, 8))
layer_visualisation = per_layer_results[1][0, :, :, :]
layer_visualisation = layer_visualisation.data
print(layer_visualisation.size())
for i, flt in enumerate(layer_visualisation):
    plt.subplot(8, 4, i + 1)
    plt.imshow(flt, cmap='gray')
    plt.axis("off")
plt.show()<jupyter_output>torch.Size([32, 24, 24])
{'fraction_non_alphanumeric': 0.1478955007256894, 'fraction_numerical': 0.07329462989840348, 'mean_word_length': 3.6877551020408164, 'pattern_counts': {'":': 0, '<': 34, '<?xml version=': 0, '>': 34, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '9', 'dir': 'jupyter-structured-clean-dedup', 'id': '356749', 'license': 'MIT', 'n_tokens_mistral': 2868, 'n_tokens_neox': 2542, 'n_words': 542, 'path': 'Chapter13/pytorch_interpretability.ipynb', 'repo_name': 'guyk1971/Mastering-PyTorch', 'size': '6890'}