<jupyter_start><jupyter_text>DLVC WS 2017Tutorial 6: CIFAR Classification (Autoencoders)== Load Packages==<jupyter_code>%matplotlib inline
import torch
from PIL import Image
import matplotlib.pyplot as plt
import numpy as np
import torchvision
import torchvision.transforms as transforms
from torch.autograd import Variable
import torch.nn as nn
import torch.nn.functional as F
import copy<jupyter_output><empty_output><jupyter_text>Load Data:===============<jupyter_code>transform = transforms.Compose([transforms.ToTensor()])
BatchSize = 100

trainset = torchvision.datasets.CIFAR10(root='./CIFAR10', train=True,
                                        download=True, transform=transform)
trainloader = torch.utils.data.DataLoader(trainset, batch_size=BatchSize,
                                          shuffle=True, num_workers=4) # Creating dataloader

testset = torchvision.datasets.CIFAR10(root='./CIFAR10', train=False,
                                       download=True, transform=transform)
testloader = torch.utils.data.DataLoader(testset, batch_size=BatchSize,
                                         shuffle=False, num_workers=4) # Creating dataloader

classes = ('plane', 'car', 'bird', 'cat',
           'deer', 'dog', 'frog', 'horse', 'ship', 'truck')
# Check availability of GPU
use_gpu = torch.cuda.is_available()
if use_gpu:
    print('GPU is available!')<jupyter_output><empty_output><jupyter_text>Define the Autoencoder:===============<jupyter_code>class autoencoder(nn.Module):
    def __init__(self):
        super(autoencoder, self).__init__()
        self.encoder = nn.Sequential(
            nn.Linear(3*32*32, 900),
            nn.ReLU(),
            nn.Linear(900, 256),
            nn.ReLU())
        self.decoder = nn.Sequential(
            nn.Linear(256, 900),
            nn.ReLU(),
            nn.Linear(900, 3*32*32),
            nn.ReLU())

    def forward(self, x):
        x = self.encoder(x)
        x = self.decoder(x)
        return x


net = autoencoder()
print(net)

if use_gpu:
    net = net.double().cuda()
else:
    net = net.double()
        
init_weights = copy.deepcopy(net.encoder[0].weight.data)<jupyter_output><empty_output><jupyter_text>Train Autoencoder:===========<jupyter_code># functions to show an image
def imshow(img, strlabel):
    npimg = img.numpy()
    npimg = np.abs(npimg)
    fig_size = plt.rcParams["figure.figsize"]
    fig_size[0] = 10
    fig_size[1] = 10
    plt.rcParams["figure.figsize"] = fig_size
    plt.figure()
    plt.title(strlabel)
    plt.imshow(np.transpose(npimg, (1, 2, 0)))
iterations = 100
learning_rate = 0.9
criterion = nn.MSELoss()

dataiter = iter(testloader)
images, labels = dataiter.next()
printImg = torch.Tensor(iterations/10,2,3,32,32)

for epoch in range(iterations):  # loop over the dataset multiple times
    runningLoss = 0.0
    for i, data in enumerate(trainloader, 0):
        # get the inputs
        inputs, labels = data
        # wrap them in Variable
        if use_gpu:
            inputs = Variable(inputs.view(-1, 3*32*32).double()).cuda()
        else:
            inputs = Variable(inputs.view(-1, 3*32*32).double())

        net.zero_grad()  # zeroes the gradient buffers of all parameters
        outputs = net(inputs) # forward 
        loss = criterion(outputs, inputs) # calculate loss
        loss.backward() #  backpropagate the loss
        for f in net.parameters():
            f.data.sub_(f.grad.data * learning_rate) # weight = weight - learning_rate * gradient (Update Weights)
        runningLoss += loss.data[0]
        
    print('At Iteration : %d / %d  ;  Mean-Squared Error : %f'%(epoch + 1,iterations,
                                                                        runningLoss/(60000/BatchSize)))
    if epoch%10 == 0:
        if use_gpu:
            outImg = net(Variable((images[0].view(-1,3*32*32).double().cuda()))).data
            outImg = outImg.view(3,32,32).cpu()
        else:
            outImg = net(Variable((images[0].view(-1,3*32*32).double()))).data
            outImg = outImg.view(3,32,32).cpu()

        dispImg = torch.Tensor(2,3,32,32)
        dispImg[0] = images[0]
        dispImg[1] = outImg
        printImg[epoch/10] = dispImg
print('Finished Training')
for i in range(iterations/10):
    imshow(torchvision.utils.make_grid(printImg[i]), 'Autoencoder Performance')<jupyter_output><empty_output><jupyter_text>Encoder Weights Visualization:=======================<jupyter_code>trained_weights = copy.deepcopy(net.encoder[0].weight.data)
d_weights = init_weights - trained_weights 

if use_gpu:
    init_weights = init_weights.view(900,3,32,32).cpu()
    trained_weights = trained_weights.view(900,3,32,32).cpu()
    d_weights = d_weights.view(900,3,32,32).cpu()
else:
    init_weights = init_weights.view(900,3,32,32)
    trained_weights = trained_weights.view(900,3,32,32)
    d_weights = d_weights.view(900,3,32,32)

imshow(torchvision.utils.make_grid(init_weights,nrow=30,normalize=True),'Initial Weights')
imshow(torchvision.utils.make_grid(trained_weights,nrow=30,normalize=True),'Trained Weights')
imshow(torchvision.utils.make_grid(d_weights,nrow=30,normalize=True), 'Weight update')<jupyter_output><empty_output><jupyter_text>Modifying the autoencoder for classification: ================================<jupyter_code># Removing the decoder module from the autoencoder
new_classifier = nn.Sequential(*list(net.children())[:-1])
net = new_classifier
# Adding linear layer for 10-class classification problem
net.add_module('classifier', nn.Sequential(nn.Linear(256, 10),nn.LogSoftmax()))
print(net)
if use_gpu:
    net = net.double().cuda()
else:
    net = net.double()
# Copying initial weights  for visualization
cll_weights = copy.deepcopy(net[0][0].weight.data)
init_classifier_weights = copy.deepcopy(net.classifier[0].weight.data)<jupyter_output><empty_output><jupyter_text>Train Classifier:===========<jupyter_code>iterations = 10
learning_rate = 0.1
criterion = nn.CrossEntropyLoss()

for epoch in range(iterations):  # loop over the dataset multiple times

    runningLoss = 0.0
    for i, data in enumerate(trainloader, 0):
        # get the inputs
        inputs, labels = data
        
        # wrap them in Variable
        if use_gpu:
            inputs, labels = Variable(inputs.view(-1, 3*32*32).double()).cuda(), Variable(labels).cuda()
        else:
            inputs, labels = Variable(inputs.view(-1, 3*32*32).double()), Variable(labels)

        net.zero_grad()  # zeroes the gradient buffers of all parameters
        outputs = net(inputs) # forward 
        loss = criterion(outputs, labels) # calculate loss
        loss.backward() #  backpropagate the loss
        for f in net.parameters():
            f.data.sub_(f.grad.data * learning_rate) # weight = weight - learning_rate * gradient (Update Weights)
        runningLoss += loss.data[0]
        correct = 0
        total = 0
    for data in testloader:
        inputs, labels = data
        if use_gpu:
            inputs, labels = Variable(inputs.view(-1, 3*32*32).double()).cuda(), labels.cuda()
        else:
            inputs, labels = Variable(inputs.view(-1, 3*32*32).double()), labels
        outputs = net(inputs)
        _, predicted = torch.max(outputs.data, 1)
        total += labels.size(0)
        correct += (predicted == labels).sum()
    print('At Iteration : %d / %d  ;  Train Error : %f ;Test Accuracy : %f'%(epoch + 1,iterations,
                                                                        runningLoss/(60000/BatchSize),100 * correct /float(total)))
print('Finished Training')<jupyter_output><empty_output><jupyter_text>Encoder Weights Visualization:=======================<jupyter_code>cll_weights_ft = copy.deepcopy(net[0][0].weight.data)
d_weights = cll_weights-cll_weights_ft 

if use_gpu:
    cll_weights = cll_weights.view(900,3,32,32).cpu()
    cll_weights_ft = cll_weights_ft.view(900,3,32,32).cpu()
    d_weights = d_weights.view(900,3,32,32).cpu()
else:
    cll_weights = cll_weights.view(900,3,32,32)
    cll_weights_ft = cll_weights_ft.view(900,3,32,32)
    d_weights = d_weights.view(900,3,32,32)

imshow(torchvision.utils.make_grid(cll_weights,nrow=30,normalize=True),'Trained Weights')
imshow(torchvision.utils.make_grid(cll_weights_ft,nrow=30,normalize=True),'Finetuned Weights')
imshow(torchvision.utils.make_grid(d_weights,nrow=30,normalize=True), 'Weight update')<jupyter_output><empty_output><jupyter_text>Performance of different Classes:==================<jupyter_code>class_correct = list(0. for i in range(10))
class_total = list(0. for i in range(10))
for data in testloader:
    images, labels = data
    if use_gpu:
        outputs = net(Variable(images.view(-1, 3*32*32).double().cuda()))
        _, predicted = torch.max(outputs.data.cpu(), 1)
    else:
        outputs = net(Variable(images.view(-1, 3*32*32).double()))
        _, predicted = torch.max(outputs.data, 1)
    c = (predicted == labels).squeeze()
    for i in range(BatchSize):
        label = labels[i]
        class_correct[label] += c[i]
        class_total[label] += 1

for i in range(10):
    print('Accuracy of %5s : %f %%' % (
        classes[i], 100 * class_correct[i] / float(class_total[i])))<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.14521201605031991, 'fraction_numerical': 0.035245634963669885, 'mean_word_length': 3.317415730337079, 'pattern_counts': {'":': 0, '<': 37, '<?xml version=': 0, '>': 37, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 14, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '9', 'dir': 'jupyter-structured-clean-dedup', 'id': '98971', 'license': 'MIT', 'n_tokens_mistral': 3202, 'n_tokens_neox': 2959, 'n_words': 579, 'path': 'Notebooks/6_CIFAR Classification (Autoencoders).ipynb', 'repo_name': 'Sonal10/Deep-Learning-Workbooks', 'size': '9221'}