<jupyter_start><jupyter_text>**These code cells were run on google colab with GPU support.**<jupyter_code>import torch
if torch.cuda.is_available():
  device = torch.device("cuda:0")
  print("GPU")
else:
  device = torch.device("cpu")
  print("CPU")<jupyter_output>GPU<jupyter_text>**Fetching the data**<jupyter_code>from sklearn.datasets import fetch_openml
import matplotlib as mpl
import matplotlib.pyplot as plt
import numpy as np

mnist = fetch_openml("mnist_784")
print(mnist.keys())<jupyter_output>dict_keys(['data', 'target', 'frame', 'feature_names', 'target_names', 'DESCR', 'details', 'categories', 'url'])<jupyter_text>**Plotting one example of each class**<jupyter_code>X = mnist["data"]
Y = mnist["target"]
Y = Y.astype(int)
X=(X/255 - 0.5)*2

fig, ax = plt.subplots(nrows=2, ncols=5, sharex=True, sharey=True)
ax = ax.flatten()

for i in range(10):
    for x, y in zip(X, Y):
        if y==i:
            img=np.array(x).reshape((28,28))
            ax[i].imshow(img, cmap="Greys")
            break
ax[0].set_yticks([])
ax[0].set_xticks([])
plt.tight_layout()
plt.show()<jupyter_output><empty_output><jupyter_text>**Out of 70k examples, 10k will be used for test set, and remaining will be used for training and validation.**<jupyter_code>X_train, X_test, Y_train, Y_test = X[:60000], X[60000:], Y[:60000], Y[60000:]
from sklearn.model_selection import StratifiedShuffleSplit
split = StratifiedShuffleSplit(n_splits=1, test_size=0.1, random_state=42)
for train_index, val_index in split.split(X_train,Y_train):
    X_train_strat = X_train[train_index, :]
    Y_train_strat = Y_train[train_index]
    X_val_strat = X_train[val_index, :]
    Y_val_strat = Y_train[val_index]
import torch.nn as nn
import torch.nn.functional as Func
from torch.autograd import Variable
import torch.optim as optim
import torch.utils.data as data
import random
from scipy.io import savemat
import os
from os import path
from sklearn.preprocessing import normalize
from torch.nn.utils import clip_grad_norm_
import torch.nn.parallel.data_parallel as data_parallel
from sklearn.metrics import confusion_matrix<jupyter_output><empty_output><jupyter_text>**Extending and overriding methods for our own dataset**<jupyter_code>class mnist_dataset(data.Dataset):
    def label_transformer(self, labels):
        return labels

    def __init__(self, input_data, labels):
        input_data = input_data.reshape((len(input_data),1,28,28))
        self.feats = input_data
        self.labels = self.label_transformer(labels)
    def __len__(self):
        return len(self.labels)
    def __getitem__(self, index):
        x = self.feats[index]
        y = self.labels[index]
        return x,y<jupyter_output><empty_output><jupyter_text>**Creating dataloader for each of the train, validation, test dataset.**<jupyter_code>class hyperparam:
    bs = 100
    lr = 0.05
    num_epochs = 50

params = {
    "batch_size": hyperparam.bs,
    "shuffle": True,
    "num_workers": 2,
    "drop_last": False,
    "pin_memory": True
}

train_set = mnist_dataset(X_train_strat, Y_train_strat)
val_set = mnist_dataset(X_val_strat, Y_val_strat)
test_set = mnist_dataset(X_test, Y_test)

training_gen = data.DataLoader(train_set, **params)
val_gen = data.DataLoader(val_set, **params)
test_gen = data.DataLoader(test_set, **params)<jupyter_output><empty_output><jupyter_text>**Created a DNN with 2 layers of CNN with 12 filters each and adding two fully connected layers of 100 and 10 neurons respectively.Used Relu activation function, with initial learning rate = 0.05 with glorot initialization. We have added Batch normalization2s layer before each CNN layer as well.**<jupyter_code>from torch.nn import Conv2d, Linear, BatchNorm2d
from torch import flatten

class batch_cnn(nn.Module):
    
    def glorot_initialize(self, layers):
        
        for layer in layers:
            torch.nn.init.xavier_normal_(layer.weight)
            torch.nn.init.zeros_(layer.bias)
    
    def __init__(self):
        super(batch_cnn, self).__init__()
        self.batchn1 = BatchNorm2d(1)
        self.conv1 = Conv2d(1,12,kernel_size=(3,3), padding = 1)

        self.batchn2 = BatchNorm2d(12)
        self.conv2 = Conv2d(12,12,kernel_size=(3,3), padding = 1)
        self.fc1 = Linear(588, 100)
        self.fc2 = Linear(100, 10)
        
        self.glorot_initialize([self.conv1, self.conv2, self.fc1, self.fc2])
        
        
    def forward(self, sig):
        sig = self.batchn1(sig)
        sig = Func.max_pool2d(Func.relu(self.conv1(sig)), (2, 2))
        sig = self.batchn2(sig)
        sig = Func.max_pool2d(Func.relu(self.conv2(sig)), (2, 2))
        sig = sig.view(-1, 12*7*7)
        sig = Func.relu(self.fc1(sig))
        sig = self.fc2(sig)
        return sig
        # return Func.softmax(sig, dim = 1)

cnn_model = batch_cnn().to(device)
optimizer = torch.optim.SGD(cnn_model.parameters(), lr = hyperparam.lr, momentum=0.9) # TODO ADAM optimizer
loss = nn.CrossEntropyLoss()<jupyter_output><empty_output><jupyter_text>**Model is trained for 50 epochs, after each epochs, printing the validation accuracy. and resulting learning rate after adjusting learning rate by 10% each 10 epochsAlso using early stopping mechanism, which stops the learning if the validation accuracy starts dropping for a consecutive 5 cycles. This is done to prevent overfitting.**<jupyter_code>from tqdm import tqdm
from datetime import datetime
from torch.optim.lr_scheduler import StepLR

scheduler = StepLR(optimizer, step_size=10, gamma=0.9)

tr_avg_loss_list = []
tr_accuracy_list = []
val_avg_loss_list = []
val_accuracy_list = []

print(datetime.now())

for epoch in tqdm(range(hyperparam.num_epochs)):
    tr_num_correct = 0
    tr_num_samples = 0
    tr_total_loss = 0
    val_num_correct = 0
    val_num_samples = 0
    val_total_loss = 0
    
    print("Learning rate: " + str(optimizer.param_groups[0]['lr']))

    cnn_model.train(True)
    with torch.set_grad_enabled(True):
        for ind, (local_batch, local_labels) in enumerate(training_gen):
            
            optimizer.zero_grad()
            local_batch = local_batch
            local_labels = local_labels
            local_batch, local_labels = Variable(local_batch).float(), Variable(local_labels)
            local_batch = local_batch.to(device)
            local_labels = local_labels.to(device)
            out1 = cnn_model(local_batch)
            
            ploss = loss(out1, local_labels.long())
            tr_total_loss += ploss * hyperparam.bs
            ploss.backward()
            optimizer.step()
            sel_class = torch.argmax(out1, dim=1)
            tr_num_correct += sel_class.eq(local_labels).sum().item()
            tr_num_samples += hyperparam.bs
                
                                  
        tr_avg_loss = tr_total_loss / len(training_gen.dataset)
        tr_avg_loss_list.append(tr_avg_loss)
        tr_accuracy = tr_num_correct / len(training_gen.dataset)
        tr_accuracy_list.append(tr_accuracy)
    
    with torch.set_grad_enabled(False):
        cnn_model.eval()
        for local_batch, local_labels in val_gen:
            local_batch = local_batch.float()
            local_labels = local_labels.float()
            local_batch, local_labels = Variable(local_batch), Variable(local_labels)
            local_batch = local_batch.to(device)
            local_labels = local_labels.to(device)
            out1 = cnn_model(local_batch)
            ploss = loss(out1, local_labels.long())
            val_total_loss += ploss * hyperparam.bs
            sel_class = torch.argmax(out1, dim=1)
            val_num_correct += sel_class.eq(local_labels).sum().item()
            val_num_samples += local_labels.size(0)

        val_avg_loss = val_total_loss / len(val_gen.dataset)
        val_avg_loss_list.append(val_avg_loss)
        val_accuracy = val_num_correct / len(val_gen.dataset)
        val_accuracy_list.append(val_accuracy)
        print("Validation accuracy: " + str(val_accuracy))

        scheduler.step()

        # Early stop
        if epoch > 10:
            if sum([val_accuracy_list[i] < val_accuracy_list[i-1] for i in range(epoch-5, epoch)]) == 5:
                break<jupyter_output>0%|          | 0/50 [00:00<?, ?it/s]<jupyter_text>**Plotting learning curves for validation and train dataset**<jupyter_code>def plot_x_y_vals(x_vals, y_vals, x_label, y_label, label, line_titles):
    for i in range(len(x_vals)):
        plt.plot(x_vals[i], y_vals[i], label=line_titles[i])
    plt.title(label)
    plt.legend()
    plt.xlabel(x_label)
    plt.ylabel(y_label)
    plt.show()

epocs = [i+1 for i in range(len(tr_accuracy_list))]
plot_x_y_vals([epocs, epocs], [tr_accuracy_list, val_accuracy_list], "Epochs", "Accuracy", "Train & Validation Accuracy", ["train", "validation"])
plot_x_y_vals([epocs, epocs], [tr_avg_loss_list, val_avg_loss_list], "Epochs", "Loss", "Train & Validation Loss", ["train", "validation"])<jupyter_output><empty_output><jupyter_text>**Overfit or Underfit?**The plot is very similar to baseline model. Part of the reason for this could be that the data we fed to our baseline model was already normalized, as we had changed the range of data to -1 to 1. Similar to baseline model,In the above learning curves we see that, a very high accuracy on both training and validation set is observed. While the training accuracy is almost 1, the validation accuracy is very close to one, and is around ~0.99.From the loss plot, we see that train loss is almost zero while validation loss, even though very less, has been increasing by very small amount each epoch in the end.In the plot, even though plot looks to be diverging but since the scale is very small, the plots arent actually that far apart. However seeing almost 100% accuracy and slight increase in the validation loss, towards the end of the epochs, we can say that the model ever so slightly overfits, but still yields good results.<jupyter_code>total_accurate = 0
total_values = 0
errors={i:{j:0 for j in range(10)} for i in range(10)}
incorrect_samples = []
correct_samples = []

def calculate_class_wise_errors(local_labels, sel_class, local_batch):
    true_labels = local_labels[sel_class.not_equal(local_labels)]
    predicted = sel_class[sel_class.not_equal(local_labels)]
    for (i, t), (i,p) in zip(enumerate(true_labels), enumerate(predicted)):
        errors[t.item()][p.item()] += 1
    
    true_labels = local_labels[sel_class.eq(local_labels)]
    predicted = sel_class[sel_class.eq(local_labels)]
    for (i, t), (i,p) in zip(enumerate(true_labels), enumerate(predicted)):
        errors[t.item()][p.item()] += 1

    if len(incorrect_samples) < 10:
        samples = local_batch[sel_class.not_equal(local_labels)]
        predicted = sel_class[sel_class.not_equal(local_labels)]
        true_labels = local_labels[sel_class.not_equal(local_labels)]
        for (i,s), (i,p), (i, t) in zip(enumerate(samples), enumerate(predicted), enumerate(true_labels)):
            incorrect_samples.append((s.cpu().numpy(), p.cpu().numpy(), t.cpu().numpy()))

    if len(correct_samples) < 10:
        samples = local_batch[sel_class.eq(local_labels)]
        predicted = sel_class[sel_class.eq(local_labels)]
        true_labels = local_labels[sel_class.eq(local_labels)]
        for (i,s), (i,p), (i, t) in zip(enumerate(samples), enumerate(predicted), enumerate(true_labels)):
            correct_samples.append((s.cpu().numpy(), p.cpu().numpy(), t.cpu().numpy()))

with torch.set_grad_enabled(False):
    cnn_model.eval()
    for local_batch, local_labels in test_gen:
        local_batch = local_batch.float()
        local_labels = local_labels.float()
        local_batch, local_labels = Variable(local_batch), Variable(local_labels)
        local_batch = local_batch.to(device)
        local_labels = local_labels.to(device)
        out1 = cnn_model(local_batch)
        ploss = loss(out1, local_labels.long())
        sel_class = torch.argmax(out1, dim=1)
        calculate_class_wise_errors(local_labels, sel_class, local_batch)
        total_accurate += sel_class.eq(local_labels).sum().item()
        total_values += local_labels.size(0)

    
print("Predicted " + str(total_accurate) +" correctly out of " + str(total_values))<jupyter_output>Predicted 9897 correctly out of 10000<jupyter_text>**We see that on test set as well we have the accuracy almost equal to 99%, which is good. This also indicates that our model is good fit for data. Below we are plotting the heatmap, where y-axis represents the actual label and x-axis represents the predicted labels. Pleaase mind that all the diagonal elements have been set to zero. So the heatmap only represents the incorrectly classified label counts.****For example row = 4, col = 3 represents the count of images, which were 4, but were actually classified as 3. And the cell (4,4) is left empty, although it should ideally contain count of all the correctly classified images of 4. As this heatmap is generated only to see if there is any pair that is mistaken a lot in the classification OR if there is any bias in our model for any label.**<jupyter_code>import seaborn as sns
class_acc = np.zeros((10,10))
for i in range(10):
    for j in range(10):
        if i!=j:
            class_acc[i,j] = errors[i][j]
        else:
            class_acc[i,j] = 0

sns.heatmap(class_acc)
plt.show()<jupyter_output><empty_output><jupyter_text>**Plotting few incorrectly classified images by our model. We see that these images are ambiguous and little bit hard to interpret or blurry. Each image contains the true value - predicted value pair on top of it.**<jupyter_code>fig, ax = plt.subplots(nrows=2, ncols=5, sharex=True, sharey=True)
ax = ax.flatten()

for i in range(10):
    img=np.array(incorrect_samples[i][0]).reshape(28,28)
    ax[i].imshow(img, cmap="Greys")
    ax[i].title.set_text(str(int(incorrect_samples[i][2])) + "-" + str(incorrect_samples[i][1]))
ax[0].set_yticks([])
ax[0].set_xticks([])
plt.tight_layout()
plt.show()

print("Incorrectly classified samples. (True and predicted values)")<jupyter_output><empty_output><jupyter_text>**Plotting few correctly classified images by our model.**<jupyter_code>fig, ax = plt.subplots(nrows=2, ncols=5, sharex=True, sharey=True)
ax = ax.flatten()

for i in range(10):
    img=np.array(correct_samples[i][0]).reshape(28,28)
    ax[i].imshow(img, cmap="Greys")
    ax[i].title.set_text(str(int(correct_samples[i][2])) + "-" + str(correct_samples[i][1]))
ax[0].set_yticks([])
ax[0].set_xticks([])
plt.tight_layout()
plt.show()

print("Correctly classified samples. (true and predicted values)")<jupyter_output><empty_output><jupyter_text>**Plotting below confusion matrix for each class. By each class, we mean all the correct prediction fot that class = true positive. all the images of a class, that were incorrectly classfied as false negative. All the images, not of that class, but classified as of that class as false positive. And all the images that were not of a class and were also classified as not belonging to that class as true negative.**<jupyter_code>import pandas as pd
# Confusion matrix
confusion_arr = np.zeros((10, 4))
for i in range(10):
    confusion_arr[i][0] = errors[i][i] # tp
    for j in range(10):
        if i!=j:
            confusion_arr[i][1]+=errors[j][i] # fp
    for j in range(10):
        if i!=j:
            confusion_arr[i][2]+= errors[i][j] # fn
    confusion_arr[i][3] = total_values - sum(confusion_arr[i][:3]) # tn

confusion_df = pd.DataFrame(confusion_arr, columns=["tp", "fp", "fn", "tn"])

confusion_df["precision"] = confusion_df["tp"] / (confusion_df["tp"] + confusion_df["fp"])
confusion_df["recall"] = confusion_df["tp"] / (confusion_df["tp"] + confusion_df["fn"])
confusion_df["accuracy"] = (confusion_df["tp"] + confusion_df["tn"]) / 10000

print("Overall Accuracy:" + str(total_accurate/total_values))

confusion_df<jupyter_output>Overall Accuracy:0.9897
{'fraction_non_alphanumeric': 0.11486316846598042, 'fraction_numerical': 0.017762992719055987, 'mean_word_length': 3.941997518610422, 'pattern_counts': {'":': 5, '<': 56, '<?xml version=': 0, '>': 53, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '14', 'dir': 'jupyter-structured-clean-dedup', 'id': '353088', 'license': 'Apache-2.0', 'n_tokens_mistral': 5457, 'n_tokens_neox': 5240, 'n_words': 1343, 'path': 'hw3/baseline-batch.ipynb', 'repo_name': 'radhe2205/applied-ml-probs', 'size': '15932'}