<jupyter_start><jupyter_text>Lecture 20: Denoising Autoencoders for MNIST classification==  Load Packages<jupyter_code>%matplotlib inline
import torch
from PIL import Image
import matplotlib.pyplot as plt
import numpy as np
import torchvision
import torchvision.transforms as transforms
from torch.autograd import Variable
from torch.autograd import Function
import torch.nn as nn
import torch.nn.functional as F
import torch.optim as optim
import copy<jupyter_output><empty_output><jupyter_text>Load Data:<jupyter_code>transform = transforms.Compose([transforms.ToTensor()])
BatchSize = 1000

trainset = torchvision.datasets.MNIST(root='./MNIST', train=True,
                                        download=True, transform=transform)
trainloader = torch.utils.data.DataLoader(trainset, batch_size=BatchSize,
                                          shuffle=True, num_workers=4) # Creating dataloader

testset = torchvision.datasets.MNIST(root='./MNIST', train=False,
                                       download=True, transform=transform)
testloader = torch.utils.data.DataLoader(testset, batch_size=BatchSize,
                                         shuffle=False, num_workers=4) # Creating dataloader

classes = ('zero', 'one', 'two', 'three',
           'four', 'five', 'six', 'seven', 'eight', 'nine')
# Check availability of GPU
use_gpu = torch.cuda.is_available()
if use_gpu:
    print('GPU is available!')<jupyter_output><empty_output><jupyter_text>Define the Autoencoder:<jupyter_code>class L1Penalty(Function):

    @staticmethod
    def forward(ctx, input, l1weight):
        ctx.save_for_backward(input)
        ctx.l1weight = l1weight
        return input

    @staticmethod
    def backward(ctx, grad_output):
        input, = ctx.saved_variables
        grad_input = input.clone().sign().mul(self.l1weight)
        grad_input += grad_output
        return grad_input
class autoencoder(nn.Module):
    def __init__(self):
        super(autoencoder, self).__init__()
        self.encoder = nn.Sequential(
            nn.Linear(28*28, 400),
            nn.Tanh())
        self.decoder = nn.Sequential(
            nn.Linear(400, 28*28),
            nn.Sigmoid())

    def forward(self, x):
        x = self.encoder(x)
        x = L1Penalty.apply(x, 0.1)
        x = self.decoder(x)
        return x


net = autoencoder()
print(net)

if use_gpu:
    net = net.double().cuda()
else:
    net = net.double()
init_weightsE = copy.deepcopy(net.encoder[0].weight.data)
init_weightsD = copy.deepcopy(net.decoder[0].weight.data)<jupyter_output><empty_output><jupyter_text>Train Autoencoder:<jupyter_code>iterations = 20
learning_rate = 1e-3
noise_mean = 0.1
noise_std = 0.2
criterion = nn.MSELoss()
optimizer = optim.Adam(net.parameters(), lr = learning_rate) # Adam optimizer for optimization
for epoch in range(iterations):  # loop over the dataset multiple times

    runningLoss = 0.0
    for i, data in enumerate(trainloader, 0):
        # get the inputs
        inputs, labels = data
        
        # wrap them in Variable
        if use_gpu:
            ideal_outputs = Variable(inputs.view(-1, 28*28).double()).cuda()
            # Noise
            noise = Variable(ideal_outputs.data.new(ideal_outputs.size()).normal_(noise_mean, noise_std).double()).cuda()
            # Adding Noise (Noisy Input)
            inputs = Variable(torch.clamp((ideal_outputs + noise).data,0,1).double()).cuda()
        else:
            ideal_outputs = Variable(inputs.view(-1, 28*28).double())
            noise = Variable(ideal_outputs.data.new(ideal_outputs.size()).normal_(noise_mean, noise_std).double())
            inputs = Variable(torch.clamp((ideal_outputs + noise).data,0,1).double())

        optimizer.zero_grad()  # zeroes the gradient buffers of all parameters
        outputs = net(inputs) # forward 
        loss = criterion(outputs, ideal_outputs) # calculate loss
        loss.backward() #  backpropagate the loss
        optimizer.step()
        runningLoss += loss.data[0]
    print('At Iteration : %d / %d  ;  Mean-Squared Error : %f'%(epoch + 1,iterations,
                                                                        runningLoss/(60000/BatchSize)))
print('Finished Training')<jupyter_output><empty_output><jupyter_text>Autoencoder Performance:<jupyter_code># functions to show an image
def imshow(img, strlabel):
    npimg = img.numpy()
    npimg = np.abs(npimg)
    fig_size = plt.rcParams["figure.figsize"]
    fig_size[0] = 10
    fig_size[1] = 10
    plt.rcParams["figure.figsize"] = fig_size
    plt.figure()
    plt.title(strlabel)
    plt.imshow(np.transpose(npimg, (1, 2, 0)))

# get some random training images
dataiter = iter(testloader)
images, labels = dataiter.next()

if use_gpu:
    ideal_outputs = Variable(images[0].view(-1,28*28).double()).cuda()
    noise = Variable(ideal_outputs.data.new(ideal_outputs.size()).normal_(noise_mean, noise_std).double()).cuda()
    inputs = ideal_outputs + noise
    outImg = net(inputs).data
    outImg = outImg.view(-1,28,28).cpu()
else:
    ideal_outputs = Variable(images[0].view(-1,28*28).double())
    noise = Variable(ideal_outputs.data.new(ideal_outputs.size()).normal_(noise_mean, noise_std).double())
    inputs = ideal_outputs + noise
    outImg = net(inputs).data
    outImg = outImg.view(-1,28,28)

dispImg = torch.Tensor(2,1,28,28)
dispImg[0] = torch.clamp(inputs.data.view(-1,28,28).cpu(),0,1)
dispImg[1] = outImg

# show images
imshow(torchvision.utils.make_grid(dispImg), 'Noisy Input                                              Denoised Output')<jupyter_output><empty_output><jupyter_text>Encoder Weight Visualisation:<jupyter_code>trained_weightsE = copy.deepcopy(net.encoder[0].weight.data)
d_weightsE = init_weightsE - trained_weightsE 

if use_gpu:
    init_weightsE = init_weightsE.view(400,1,28,28).cpu()
    trained_weightsE = trained_weightsE.view(400,1,28,28).cpu()
    d_weightsE = d_weightsE.view(400,1,28,28).cpu()
else:
    init_weightsE = init_weightsE.view(400,1,28,28)
    trained_weightsE = trained_weightsE.view(400,1,28,28)
    d_weightsE = d_weightsE.view(400,1,28,28)

imshow(torchvision.utils.make_grid(init_weightsE,nrow=20,normalize=True),'Initial Weights')
imshow(torchvision.utils.make_grid(trained_weightsE,nrow=20,normalize=True),'Trained Weights')
imshow(torchvision.utils.make_grid(d_weightsE,nrow=20,normalize=True), 'Weight update')<jupyter_output><empty_output><jupyter_text>Decoder Weight Visualisation:<jupyter_code>trained_weightsD = copy.deepcopy(net.decoder[0].weight.data)
d_weightsD = init_weightsD - trained_weightsD 

if use_gpu:
    init_weightsD = init_weightsD.view(784,1,20,20).cpu()
    trained_weightsD = trained_weightsD.view(784,1,20,20).cpu()
    d_weightsD = d_weightsD.view(784,1,20,20).cpu()
else:
    init_weightsD = init_weightsD.view(784,1,20,20)
    trained_weightsD = trained_weightsD.view(784,1,20,20)
    d_weightsD = d_weightsD.view(784,1,20,20)

imshow(torchvision.utils.make_grid(init_weightsD,nrow=28,normalize=True),'Initial Weights')
imshow(torchvision.utils.make_grid(trained_weightsD,nrow=28,normalize=True),'Trained Weights')
imshow(torchvision.utils.make_grid(d_weightsD,nrow=28,normalize=True), 'Weight update')<jupyter_output><empty_output><jupyter_text>Modifying the autoencoder for classification:<jupyter_code># Removing the decoder module from the autoencoder
new_classifier = nn.Sequential(*list(net.children())[:-1])
net = new_classifier
# Adding linear layer for 10-class classification problem
net.add_module('classifier', nn.Sequential(nn.Linear(400, 10),nn.LogSoftmax()))
print(net)
if use_gpu:
    net = net.double().cuda()
else:
    net = net.double()<jupyter_output><empty_output><jupyter_text>Train Classifier:<jupyter_code>iterations = 10
learning_rate = 0.1
criterion = nn.NLLLoss()

for epoch in range(iterations):  # loop over the dataset multiple times

    runningLoss = 0.0
    for i, data in enumerate(trainloader, 0):
        # get the inputs
        inputs, labels = data
        
        # wrap them in Variable
        if use_gpu:
            inputs, labels = Variable(inputs.view(-1, 28*28).double()).cuda(), Variable(labels).cuda()
        else:
            inputs, labels = Variable(inputs.view(-1, 28*28).double()), Variable(labels)

        net.zero_grad()  # zeroes the gradient buffers of all parameters
        outputs = net(inputs) # forward 
        loss = criterion(outputs, labels) # calculate loss
        loss.backward() #  backpropagate the loss
        for f in net.parameters():
            f.data.sub_(f.grad.data * learning_rate) # weight = weight - learning_rate * gradient (Update Weights)
        runningLoss += loss.data[0]
        correct = 0
        total = 0
    for data in testloader:
        inputs, labels = data
        if use_gpu:
            inputs, labels = Variable(inputs.view(-1, 28*28).double()).cuda(), labels.cuda()
        else:
            inputs, labels = Variable(inputs.view(-1, 28*28).double()), labels
        outputs = net(inputs)
        _, predicted = torch.max(outputs.data, 1)
        total += labels.size(0)
        correct += (predicted == labels).sum()
    print('At Iteration : %d / %d  ;  Train Error : %f ;Test Accuracy : %f'%(epoch + 1,iterations,
                                                                        runningLoss/(60000/BatchSize),100 * correct /float(total)))
print('Finished Training')<jupyter_output><empty_output><jupyter_text>Performance of different Classes:<jupyter_code>class_correct = list(0. for i in range(10))
class_total = list(0. for i in range(10))
for data in testloader:
    images, labels = data
    if use_gpu:
        outputs = net(Variable(images.view(-1, 28*28).double().cuda()))
        _, predicted = torch.max(outputs.data.cpu(), 1)
    else:
        outputs = net(Variable(images.view(-1, 28*28).double()))
        _, predicted = torch.max(outputs.data, 1)
    c = (predicted == labels).squeeze()
    for i in range(BatchSize):
        label = labels[i]
        class_correct[label] += c[i]
        class_total[label] += 1

for i in range(10):
    print('Accuracy of %5s : %f %%' % (
        classes[i], 100 * class_correct[i] / float(class_total[i])))<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.12764913657770802, 'fraction_numerical': 0.028846153846153848, 'mean_word_length': 3.408737024221453, 'pattern_counts': {'":': 0, '<': 41, '<?xml version=': 0, '>': 41, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 16, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '10', 'dir': 'jupyter-structured-clean-dedup', 'id': '432465', 'license': 'MIT', 'n_tokens_mistral': 3495, 'n_tokens_neox': 3262, 'n_words': 633, 'path': 'lecture20.ipynb', 'repo_name': 'Shahzadnit/Deep-Learning-For-Visual-Computing', 'size': '10192'}