<jupyter_start><jupyter_text>DLVC WS 2017Tutorial 7: Stacked Denoising and Sparse Autoencoders== Load Packages==<jupyter_code>%matplotlib inline
import torch
from PIL import Image
import matplotlib.pyplot as plt
import numpy as np
import torchvision
import torchvision.transforms as transforms
from torch.autograd import Variable
from torch.autograd import Function
import torch.nn as nn
import torch.nn.functional as F
import torch.optim as optim<jupyter_output><empty_output><jupyter_text>Load Data:===============<jupyter_code>transform = transforms.Compose([transforms.ToTensor()])
BatchSize = 1000

trainset = torchvision.datasets.MNIST(root='./MNIST', train=True,
                                        download=True, transform=transform)
trainloader = torch.utils.data.DataLoader(trainset, batch_size=BatchSize,
                                          shuffle=True, num_workers=4) # Creating dataloader

testset = torchvision.datasets.MNIST(root='./MNIST', train=False,
                                       download=True, transform=transform)
testloader = torch.utils.data.DataLoader(testset, batch_size=BatchSize,
                                         shuffle=False, num_workers=4) # Creating dataloader
# Check availability of GPU
use_gpu = torch.cuda.is_available()
if use_gpu:
    print('GPU is available!')<jupyter_output><empty_output><jupyter_text>Define the Sparse Autoencoder:===============<jupyter_code>class L1Penalty(Function):

    @staticmethod
    def forward(ctx, input, l1weight):
        ctx.save_for_backward(input)
        ctx.l1weight = l1weight
        return input

    @staticmethod
    def backward(ctx, grad_output):
        input, = ctx.saved_variables
        grad_input = input.clone().sign().mul(self.l1weight)
        grad_input += grad_output
        return grad_input
class autoencoder(nn.Module):
    def __init__(self):
        super(autoencoder, self).__init__()
        self.encoder = nn.Sequential(
            nn.Linear(28*28, 400),
            nn.Tanh())
        self.decoder = nn.Sequential(
            nn.Linear(400, 28*28),
            nn.Sigmoid())

    def forward(self, x):
        x = self.encoder(x)
        x = L1Penalty.apply(x, 0.1)
        x = self.decoder(x)
        return x


net = autoencoder()
print(net)

if use_gpu:
    net = net.double().cuda()
else:
    net = net.double()<jupyter_output><empty_output><jupyter_text>Train Autoencoder:===========<jupyter_code>iterations = 20
learning_rate = 1e-3
noise_mean = 0.1
noise_std = 0.2
criterion = nn.MSELoss()
optimizer = optim.Adam(net.parameters(), lr = learning_rate) # Adam optimizer for optimization
for epoch in range(iterations):  # loop over the dataset multiple times

    runningLoss = 0.0
    for i, data in enumerate(trainloader, 0):
        # get the inputs
        inputs, labels = data
        
        # wrap them in Variable
        if use_gpu:
            ideal_outputs = Variable(inputs.view(-1, 28*28).double()).cuda()
            noise = Variable(ideal_outputs.data.new(ideal_outputs.size()).normal_(noise_mean, noise_std).double()).cuda()
            inputs = Variable(torch.clamp((ideal_outputs + noise).data,0,1).double()).cuda()
        else:
            ideal_outputs = Variable(inputs.view(-1, 28*28).double())
            noise = Variable(ideal_outputs.data.new(ideal_outputs.size()).normal_(noise_mean, noise_std).double())
            inputs = Variable(torch.clamp((ideal_outputs + noise).data,0,1).double())

        optimizer.zero_grad()  # zeroes the gradient buffers of all parameters
        outputs = net(inputs) # forward 
        loss = criterion(outputs, ideal_outputs) # calculate loss
        loss.backward() #  backpropagate the loss
        optimizer.step()
        runningLoss += loss.data[0]
    print('At Iteration : %d / %d  ;  Mean-Squared Error : %f'%(epoch + 1,iterations,
                                                                        runningLoss/(60000/BatchSize)))
print('Finished Training')<jupyter_output><empty_output><jupyter_text>Stacked Denoising and Sparse Autoencoder:================================<jupyter_code># Adding New Layer (Stacking)
net.encoder.add_module('New_Encoder_Layer', nn.Sequential(nn.Linear(400, 256),nn.Tanh()))
net.encoder.add_module('New_Decoder_Layer', nn.Sequential(nn.Linear(256, 400),nn.Tanh()))
print(net)
if use_gpu:
    net = net.double().cuda()
else:
    net = net.double()<jupyter_output><empty_output><jupyter_text>Train Autoencoder:==========<jupyter_code>for epoch in range(iterations):  # loop over the dataset multiple times

    runningLoss = 0.0
    for i, data in enumerate(trainloader, 0):
        # get the inputs
        inputs, labels = data
        
        # wrap them in Variable
        if use_gpu:
            ideal_outputs = Variable(inputs.view(-1, 28*28).double()).cuda()
            noise = Variable(ideal_outputs.data.new(ideal_outputs.size()).normal_(noise_mean, noise_std).double()).cuda()
            inputs = Variable(torch.clamp((ideal_outputs + noise).data,0,1).double()).cuda()
        else:
            ideal_outputs = Variable(inputs.view(-1, 28*28).double())
            noise = Variable(ideal_outputs.data.new(ideal_outputs.size()).normal_(noise_mean, noise_std).double())
            inputs = Variable(torch.clamp((ideal_outputs + noise).data,0,1).double())

        optimizer.zero_grad()  # zeroes the gradient buffers of all parameters
        outputs = net(inputs) # forward 
        loss = criterion(outputs, ideal_outputs) # calculate loss
        loss.backward() #  backpropagate the loss
        optimizer.step()
        runningLoss += loss.data[0]
    print('At Iteration : %d / %d  ;  Mean-Squared Error : %f'%(epoch + 1,iterations,
                                                                        runningLoss/(60000/BatchSize)))
print('Finished Training')<jupyter_output><empty_output><jupyter_text>Testing Autoencoder Performance:=================<jupyter_code># functions to show an image
def imshow(img):
    npimg = img.numpy()
    plt.imshow(np.transpose(npimg, (1, 2, 0)))

# get some random training images
dataiter = iter(testloader)
images, labels = dataiter.next()

if use_gpu:
    ideal_outputs = Variable(images[0].view(-1,28*28).double()).cuda()
    noise = Variable(ideal_outputs.data.new(ideal_outputs.size()).normal_(noise_mean, noise_std).double()).cuda()
    inputs = ideal_outputs + noise
    outImg = net(inputs).data
    outImg = outImg.view(-1,28,28).cpu()
else:
    ideal_outputs = Variable(images[0].view(-1,28*28).double())
    noise = Variable(ideal_outputs.data.new(ideal_outputs.size()).normal_(noise_mean, noise_std).double())
    inputs = ideal_outputs + noise
    outImg = net(inputs).data
    outImg = outImg.view(-1,28,28)

dispImg = torch.Tensor(2,1,28,28)
dispImg[0] = torch.clamp(inputs.data.view(-1,28,28).cpu(),0,1)
dispImg[1] = outImg

# show images
imshow(torchvision.utils.make_grid(dispImg))<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.13503386655137628, 'fraction_numerical': 0.020319930825767402, 'mean_word_length': 3.216281895504253, 'pattern_counts': {'":': 0, '<': 29, '<?xml version=': 0, '>': 29, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 14, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '7', 'dir': 'jupyter-structured-clean-dedup', 'id': '468050', 'license': 'MIT', 'n_tokens_mistral': 2258, 'n_tokens_neox': 2089, 'n_words': 422, 'path': 'Notebooks/7_Stacked Denoising and Sparse Autoencoders.ipynb', 'repo_name': 'Sonal10/Deep-Learning-Workbooks', 'size': '6939'}