<jupyter_start><jupyter_text>Importing Libraries<jupyter_code># ! pip install cython
# import torch
# import torch
import torch
from torchvision import datasets,transforms
%matplotlib inline
import matplotlib.pyplot as plt # for visualizing images
import random # for random image index
import torch.nn as nn # for network
import torch.nn.functional as F # for forward method
import torch.optim as optim # for optimizer
!pip install torchsummary
from torchsummary import summary # for model summary and params
from tqdm import tqdm # for beautiful model training updates
from model import Network<jupyter_output>Requirement already satisfied: torchsummary in /usr/local/lib/python3.7/dist-packages (1.5.1)<jupyter_text>Seed and Cuda<jupyter_code># check for cuda
cuda = torch.cuda.is_available()
print (f' Cuda Status : {cuda}')

# setting seed
SEED = 42 # arbit seed, why 42 - because in hitch hikers guide to galaxy it is answer to everything
# torch.cuda.seed(SEED) 
torch.cuda.manual_seed_all(SEED) if cuda else torch.manual_seed(SEED)<jupyter_output>Cuda Status : True<jupyter_text>Downloading dataset, splitting datasetsloading dataset<jupyter_code>train = datasets.MNIST(
    root = './',# directory where data needs to be stored
    train = True, # get the training portion of the dataset
    download = True, # downloads
    transform = transforms.Compose([
                  transforms.RandomRotation((-7.0, 7.0), fill=(1,)), #random rotation
                  transforms.ToTensor(),# converts to tesnor
                  transforms.Normalize((0.1307,), (0.3081,))# Normalize
                ])
    )

test = datasets.MNIST(
    root = './',# directory where data needs to be stored
    train = False, # get the test portion of the dataset
    download = True, # downloads
    transform = transforms.Compose([
                  transforms.ToTensor(),# converts to tesnor
                  transforms.Normalize((0.1307,), (0.3081,))# Normalize
                ])
    )<jupyter_output>Downloading http://yann.lecun.com/exdb/mnist/train-images-idx3-ubyte.gz
Downloading http://yann.lecun.com/exdb/mnist/train-images-idx3-ubyte.gz to ./MNIST/raw/train-images-idx3-ubyte.gz<jupyter_text>Train and Test Dataloader<jupyter_code>dataloader_args = dict(shuffle=True, batch_size=128, num_workers=4, pin_memory = True) if cuda else dict(shuffle=True, batch_size=64)

train_loader = torch.utils.data.DataLoader(
    dataset=train,# train dataset
    **dataloader_args # the dataloader arguments change dependent on cuda is available or not
    )

test_loader = torch.utils.data.DataLoader(
    dataset = test,# test dataset 
    **dataloader_args # the dataloader arguments change dependent on cuda is available or not
)<jupyter_output>/usr/local/lib/python3.7/dist-packages/torch/utils/data/dataloader.py:481: UserWarning: This DataLoader will create 4 worker processes in total. Our suggested max number of worker in current system is 2, which is smaller than what this DataLoader is going to create. Please be aware that excessive worker creation might get DataLoader running slow or even freeze, lower the worker number to avoid potential slowness/freeze if necessary.
  cpuset_checked))<jupyter_text>Checking Dataloaders- sample data<jupyter_code>images, labels  = next(iter(train_loader))
print(images.shape)
print(labels.shape)
# printing random image and seeing
plt.imshow(images[random.randint(0,len(images))].numpy().squeeze(), cmap='gray_r')
# Looking at more images
figure = plt.figure()
for index in range(1, len(images) + 1): # assumption: batch size would be atleast 8
    plt.subplot(8, int(len(images)/8), index)
    plt.axis('off')
    plt.imshow(images[index-1].numpy().squeeze(), cmap='gray_r')<jupyter_output><empty_output><jupyter_text>Network<jupyter_code>import torch.nn as nn # for network

# drop_out_value = 0.1

# class Network(nn.Module):
#   def __init__(self, mode = None):
#     super(Network,self).__init__() # extending super class method
#     self.mode = mode
#     print ("mode :", mode)

#     # Input Block
#     self.convblock1 = nn.Sequential(
#         nn.Conv2d(1,12,3), # In- 1x28x28, Out- 12x26x26, RF- 3x3, Jump_in -1, Jump_out -1
#         nn.ReLU()
#         # ,
#         # nn.BatchNorm2d(12),
#         # nn.Dropout(drop_out_value)
#     ) 

#     if mode =="BN":
#       self.norm1 = nn.BatchNorm2d(12) # BatchNorm
#     if mode == "LN":
#       self.norm1 = nn.LayerNorm([12,26,26]) # Layer Norm
#     if mode == "GN":
#       self.norm1 = nn.GroupNorm(2,12) # Group Norm

#     self.dropout1 = nn.Dropout(drop_out_value) # drop out
#     ###---------------------------------------------------

#     # Conv Block 2
#     self.convblock2 = nn.Sequential(
#         nn.Conv2d(12,12,3), # In- 12x26x26, Out- 12x24x24, RF- 5x5, Jump_in -1, Jump_out -1
#         nn.ReLU()
#         # ,
#         # nn.BatchNorm2d(12),
#         # nn.Dropout(drop_out_value)
#     ) 

#     if mode =="BN":
#       self.norm2 = nn.BatchNorm2d(12) # BatchNorm
#     if mode == "LN":
#       self.norm2 = nn.LayerNorm([12,24,24]) # Layer Norm
#     if mode == "GN":
#       self.norm2 = nn.GroupNorm(2,12) # Group Norm

#     self.dropout2 = nn.Dropout(drop_out_value) # drop out
#     ###---------------------------------------------------


#     # Conv Block 3
#     self.convblock3 = nn.Sequential(
#         nn.Conv2d(12,12,3), # In- 12x24x24, Out- 12x22x22, RF- 7x7, Jump_in -1, Jump_out -1
#         nn.ReLU()
#         # ,
#         # nn.BatchNorm2d(12),
#         # nn.Dropout(drop_out_value)
#     ) 

#     if mode =="BN":
#       self.norm3 = nn.BatchNorm2d(12) # BatchNorm
#     if mode == "LN":
#       self.norm3 = nn.LayerNorm([12,22,22]) # Layer Norm
#     if mode == "GN":
#       self.norm3 = nn.GroupNorm(2,12) # Group Norm

#     self.dropout3 = nn.Dropout(drop_out_value) # drop out
#     ###---------------------------------------------------

#     self.convblock3_1 = nn.Sequential(
#         nn.Conv2d(12,12,3), # In- 12x22x22, Out- 12x20x20, RF- 7x7, Jump_in -1, Jump_out -1
#         nn.ReLU()
#         # ,
#         # nn.BatchNorm2d(12),
#         # nn.Dropout(drop_out_value)
#     ) 

#     if mode =="BN":
#       self.norm3_1 = nn.BatchNorm2d(12) # BatchNorm
#     if mode == "LN":
#       self.norm3_1 = nn.LayerNorm([12,20,20]) # Layer Norm
#     if mode == "GN":
#       self.norm3_1 = nn.GroupNorm(2,12) # Group Norm

#     self.dropout3_1 = nn.Dropout(drop_out_value) # drop out
#     ###---------------------------------------------------

#     self.convblock3_2 = nn.Sequential(
#         nn.Conv2d(12,12,3), # In- 12x20x20, Out- 12x18x18, RF- 7x7, Jump_in -1, Jump_out -1
#         nn.ReLU()
#         # ,
#         # nn.BatchNorm2d(12),
#         # nn.Dropout(drop_out_value)
#     )

#     if mode =="BN":
#       self.norm3_2 = nn.BatchNorm2d(12) # BatchNorm
#     if mode == "LN":
#       self.norm3_2 = nn.LayerNorm([12,18,18]) # Layer Norm
#     if mode == "GN":
#       self.norm3_2 = nn.GroupNorm(2,12) # Group Norm

#     self.dropout3_2 = nn.Dropout(drop_out_value) # drop out
#     ###---------------------------------------------------
    

#     #### Transition Block 1 
#     self.pool1 = nn.MaxPool2d(kernel_size=2, stride=2) # In- 12x18x18 Out- 12x9x9 RF- 8x8, Jump_in -1, Jump_out -2


#     # Conv Block 5
#     self.convblock5 = nn.Sequential(
#         nn.Conv2d(12,16,3), # In- 12x9x9 Out- 16x7x7 RF- 12x12, Jump_in -2, Jump_out -2
#         nn.ReLU()
#         # ,
#         # nn.BatchNorm2d(16),
#         # nn.Dropout(drop_out_value)
#     ) 

#     if mode =="BN":
#       self.norm5 = nn.BatchNorm2d(16) # BatchNorm
#     if mode == "LN":
#       self.norm5 = nn.LayerNorm([16,7,7]) # Layer Norm
#     if mode == "GN":
#       self.norm5 = nn.GroupNorm(2,16) # Group Norm

#     self.dropout5 = nn.Dropout(drop_out_value) # drop out
#     ###---------------------------------------------------

#     # Conv Block 6
#     self.convblock6 = nn.Sequential(
#         nn.Conv2d(16,16,3), # In- 16x7x7 Out- 16x5x5 RF- 16x16, Jump_in -2, Jump_out -2
#         nn.ReLU()
#         # ,
#         # nn.BatchNorm2d(16),
#         # nn.Dropout(drop_out_value)
#     ) 

#     if mode =="BN":
#       self.norm6 = nn.BatchNorm2d(16) # BatchNorm
#     if mode == "LN":
#       self.norm6 = nn.LayerNorm([16,5,5]) # Layer Norm
#     if mode == "GN":
#       self.norm6 = nn.GroupNorm(2,16) # Group Norm

#     self.dropout6 = nn.Dropout(drop_out_value) # drop out
#     ###---------------------------------------------------

#     # Output Block
#     self.convblock7 = nn.Sequential(
#         nn.Conv2d(16,10,1), # In- 16x5x5 Out- 10x5x5, RF- 16x16, Jump_in -2, Jump_out -2
#         nn.ReLU()
#         # ,
#         # nn.BatchNorm2d(10),
#         # nn.Dropout(drop_out_value)
#     ) 

#     if mode =="BN":
#       self.norm7 = nn.BatchNorm2d(10) # BatchNorm
#     if mode == "LN":
#       self.norm7 = nn.LayerNorm([10,5,5]) # Layer Norm
#     if mode == "GN":
#       self.norm7 = nn.GroupNorm(2,10) # Group Norm

#     self.dropout7 = nn.Dropout(drop_out_value) # drop out
#     ###---------------------------------------------------

#     self.gap = nn.AvgPool2d(5) # In- 10x5x5, Out- 10x1x1, RF- 16x16, Jump_in -2, Jump_out -2

  
#   def forward(self,x):

#     x = self.convblock1(x)
#     if mode: 
#       x = self.norm1(x)
#     x = self.dropout1(x)


#     x = self.convblock2(x)
#     if mode: 
#       x = self.norm2(x)
#     x = self.dropout2(x)
  

#     x = self.convblock3(x)
#     if mode: 
#       x = self.norm3(x)
#     x = self.dropout3(x)

#     x = self.convblock3_1(x)
#     if mode: 
#       x = self.norm3_1(x)
#     x = self.dropout3_1(x)

#     x = self.convblock3_2(x)
#     if mode: 
#       x = self.norm3_2(x)
#     x = self.dropout3_2(x)

#     x = self.pool1(x)

#     x = self.convblock5(x)
#     if mode: 
#       x = self.norm5(x)
#     x = self.dropout5(x)


#     x = self.convblock6(x)
#     if mode: 
#       x = self.norm6(x)
#     x = self.dropout6(x)

#     x = self.convblock7(x)
#     if mode: 
#       x = self.norm7(x)
#     x = self.dropout7(x)


#     x = self.gap(x)

#     # Flattening
#     x = x.view(-1,10)
#     return F.log_softmax(x,dim=-1)

# # model = Network(mode="GN")
# # print(model)<jupyter_output><empty_output><jupyter_text>Model Params  - Checking the model summary and number of parameters<jupyter_code>device = torch.device("cuda" if cuda else "cpu")
print(device)
# mode = 'BN'
# mode = 'LN'
# mode ='GN'
# lambda_l1 = 0.01 
lambda_l1 = 0.001 

list_mode = ['BN','LN','GN']
for mode in list_mode:
  # mode = None
  print (f'------------Normalization : {mode}-------------')
  model = Network(mode=mode).to(device)
  # print(model)
  summary(model, input_size=(1, 28, 28))<jupyter_output>cuda
------------Normalization : BN-------------
mode : BN<jupyter_text>Training and Testing- includes test and train functions- includes loop function, where test can happen after each epoch is trained<jupyter_code># Training Function
train_losses = [] # to capture train losses over training epochs
train_accuracy = [] # to capture train accuracy over training epochs

def train(model,device, train_loader,optimizer,epoch,regularization=None):
  model.train() # setting the model in training 
  pbar = tqdm(train_loader) # putting the iterator in pbar
  correct = 0 # for accuracy numerator
  processed =0 # for accuracy denominator

  for batch_idx, (images,labels) in enumerate(pbar):
    images, labels = images.to(device),labels.to(device)#sending data to CPU or GPU as per device
    optimizer.zero_grad() # setting gradients to zero to avoid accumulation

    y_preds = model(images) # forward pass, result captured in y_preds (plural as there are many images in a batch)
    # the predictions are in one hot vector

    loss = F.nll_loss(y_preds,labels) # capturing loss
    # l1 = 0
    ########-------L1-Block- starts ----------
    if (regularization == 'L1'):
      l1=0
      for p in model.parameters():
        l1 = l1 + p.abs().sum()
      loss = loss + lambda_l1 * l1
    ########-------L1-Block- ends ----------

    train_losses.append(loss) # to capture loss over many epochs

    loss.backward() # backpropagation
    optimizer.step() # updating the params

    preds = y_preds.argmax(dim=1, keepdim=True)  # get the index of the max log-probability
    correct += preds.eq(labels.view_as(preds)).sum().item()
    processed += len(images)

    if (regularization == 'L1'):
        pbar.set_description(desc= f'L1 Regularization : Loss={loss.item()} Batch_id={batch_idx} Accuracy={100*correct/processed:0.2f}')
    else:
        pbar.set_description(desc= f'Loss={loss.item()} Batch_id={batch_idx} Accuracy={100*correct/processed:0.2f}')


    train_accuracy.append(100*correct/processed)

# Test Function
test_losses = [] # to capture test losses 
test_accuracy = [] # to capture test accuracy 

def test(model,device, test_loader):
  model.eval() # setting the model in evaluation mode
  test_loss = 0
  correct = 0 # for accuracy numerator

  with torch.no_grad():
    for (images,labels) in test_loader:
      images, labels = images.to(device),labels.to(device)#sending data to CPU or GPU as per device
      outputs = model(images) # forward pass, result captured in outputs (plural as there are many images in a batch)
      # the outputs are in batch size x one hot vector 

      test_loss = F.nll_loss(outputs,labels, reduction='sum').item()  # sum up batch loss
      preds = outputs.argmax(dim=1, keepdim=True)  # get the index of the max log-probability
      correct += preds.eq(labels.view_as(preds)).sum().item()

    test_loss /= len(test_loader.dataset) # average test loss
    test_losses.append(test_loss) # to capture loss over many batches

    print('\nTest set: Average loss: {:.4f}, Accuracy: {}/{} ({:.2f}%)\n'.format(
    test_loss, correct, len(test_loader.dataset),
    100. * correct / len(test_loader.dataset)))

    test_accuracy.append(100*correct/len(test_loader.dataset))

from torch.optim.lr_scheduler import StepLR
# list_mode =['BN']
# list_mode = ['BN','LN']
list_mode = ['LN','GN','BN']
models=[None]*len(list_mode)
# EPOCHS = 2
EPOCHS = 20

fig, axs = plt.subplots(1,2,figsize=(15,10))

for idx,mode in enumerate(list_mode):
  models[idx]=Network(mode=mode).to(device)
  optimizer = optim.SGD(model.parameters(), lr=0.01, momentum=0.9)
  # scheduler = StepLR(optimizer, step_size=7, gamma=0.1)
  
  if mode == 'BN':
    regularization='L1'
  else:
    regularization=None
  
  train_losses = [] # to capture train losses over training epochs
  train_accuracy = [] # to capture train accuracy over training epochs
  test_losses = [] # to capture test losses 
  test_accuracy = [] # to capture test accuracy 
  # summary(models[idx], input_size=(1, 28, 28))

  print(f'------Normalization = {mode} , Regularization = {regularization}------')
  for epoch in range(EPOCHS):
    print("EPOCH:", epoch)
    train(models[idx], device, train_loader, optimizer, epoch, regularization=regularization)
    test(models[idx], device, test_loader)
  # for idx,mode in enumerate(list_mode):
  # axs[0].plot(train_losses,label = str(mode)+ " Train Loss")
  # axs[0].plot(test_losses,label = str(mode)+' Test loss')
  # axs[0].set_title("Loss")
  # axs[0].legend()
  # axs[1].plot(train_accuracy,label = str(mode)+ " Train Accuracy")
  # axs[1].plot(test_accuracy,label = str(mode)+ ' Test Accuracy')
  # axs[1].set_title("Accuracy")
  # axs[1].legend()
  ######################
  # axs[0].plot(train_losses,label = str(mode)+ " Train Loss")
  axs[0].plot(test_losses,label = str(mode)+ " Test Loss")
  axs[0].set_title("Loss")
  axs[0].legend()
  # axs[1].plot(train_accuracy,label = str(mode)+ " Train Accuracy")
  axs[1].plot(test_accuracy, label = str(mode)+ ' Test Accuracy')
  axs[1].set_title("Accuracy")
  axs[1].legend()
  ######################<jupyter_output>mode : LN
------Normalization = LN , Regularization = None------
EPOCH: 0<jupyter_text>Plotting Misclassified images<jupyter_code># Plotting misclassified Images

for idx,mode in enumerate(list_mode):
  # print(models[idx])
  incorrect_image_list =[]
  predicted_label_list =[]
  correct_label_list = []
  # incorrect = 0
  for i,(images,labels) in enumerate(test_loader):
    images, labels = images.to(device),labels.to(device)#sending data to CPU or GPU as per device
    outputs = models[idx](images) # forward pass, result captured in outputs (plural as there are many images in a batch)
    # preds = outputs.argmax(dim=1, keepdim=True)  # get the index of the max log-probability
    # incorrect += preds.neq(labels.view_as(preds))
    # print(incorrect)
    # break
    preds = outputs.argmax(dim=1, keepdim=True).squeeze(1)         
    idxs_mask = (preds !=  labels).view(-1)
    img_nm = images[idxs_mask].cpu().numpy()
    img_nm = img_nm.reshape(img_nm.shape[0], 28, 28)
    if img_nm.shape[0] > 0:
        img_list = [img_nm[i] for i in range(img_nm.shape[0])]
        incorrect_image_list.extend(img_list)
        predicted_label_list.extend(preds[idxs_mask].cpu().numpy())
        correct_label_list.extend(labels[idxs_mask].cpu().numpy())
  plt.figure(figsize=(15,5))
  i= 0
  # Display the list of 10 misclassified images
  for index, image in enumerate(incorrect_image_list) :
      ax = plt.subplot(2, 5, i+1)
      ax.set_title(str(mode) + ", Actual: " + str(correct_label_list[index]) + ", Predicted: " + str(predicted_label_list[index]))
      ax.axis('off')
      plt.imshow(image)
      i +=1
      if i==10:
          break<jupyter_output>/usr/local/lib/python3.7/dist-packages/torch/utils/data/dataloader.py:481: UserWarning: This DataLoader will create 4 worker processes in total. Our suggested max number of worker in current system is 2, which is smaller than what this DataLoader is going to create. Please be aware that excessive worker creation might get DataLoader running slow or even freeze, lower the worker number to avoid potential slowness/freeze if necessary.
  cpuset_checked))
{'fraction_non_alphanumeric': 0.16446124763705103, 'fraction_numerical': 0.032858890247970646, 'mean_word_length': 2.841734301580521, 'pattern_counts': {'":': 24, '<': 30, '<?xml version=': 0, '>': 31, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 7, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '9', 'dir': 'jupyter-structured-clean-dedup', 'id': '280061', 'license': 'MIT', 'n_tokens_mistral': 6653, 'n_tokens_neox': 6275, 'n_words': 1463, 'path': 'S6-AssignmentSolution/EVA_S6.ipynb', 'repo_name': 'peeyushsinghal/EVA', 'size': '17986'}