<jupyter_start><jupyter_text>Variational Auto-Encoder - Vanilla Version Importing the required packages<jupyter_code>import torch
from torch.autograd import Variable
import numpy as np
import torch.nn.functional as F
import torchvision
from torchvision import transforms
import torch.optim as optim
from torch import nn
import matplotlib.pyplot as plt<jupyter_output><empty_output><jupyter_text>Defining the modelThe model that I have used is a simple fully connected one with one hidden layer in each of the parts( i.e. Encoder and Decoder).<jupyter_code>class VAE(nn.Module):
  
    def __init__(self):
        super(VAE, self).__init__()
        
        self.fc1 = nn.Linear(28*28, 400)
        self.fc2_mu = nn.Linear(400, 20)
        self.fc2_sig = nn.Linear(400, 20)
        self.fc3 = nn.Linear(20, 400)
        self.fc4 = nn.Linear(400, 784)
  
    def encode(self,x):
        a1 = F.relu(self.fc1(x))
        a_mu = self.fc2_mu(a1)
        a_logvar = self.fc2_sig(a1)
        return a_mu, a_logvar
  
    def decode(self,z):
        a3 = F.relu(self.fc3(z))
        return torch.sigmoid(self.fc4(a3))
  
    def reparameterize(self, mu, logvar):
        std = torch.exp(0.5*logvar)
        eps = torch.randn_like(std)
        return eps.mul(std).add_(mu)
  
    def forward(self,x):
        mu, logvar = self.encode(x.view(-1, 784))
        z = self.reparameterize(mu, logvar)
        return self.decode(z), mu, logvar<jupyter_output><empty_output><jupyter_text>Getting the Dataset<jupyter_code>train_dataset = torchvision.datasets.MNIST(root='./data',train=True, transform=transforms.ToTensor(), download=True)
test_dataset = torchvision.datasets.MNIST(root='./data',train=False, transform=transforms.ToTensor(),download = True)

train_loader = torch.utils.data.DataLoader(dataset=train_dataset, batch_size=100, shuffle=True)
test_loader = torch.utils.data.DataLoader(dataset=test_dataset,  batch_size=100, shuffle=False)<jupyter_output><empty_output><jupyter_text>Instantiating the model and Optimizer<jupyter_code>model = VAE()
optimizer = optim.Adam(model.parameters(), lr=1e-3)<jupyter_output><empty_output><jupyter_text>Defining the loss functionThe loss consists of two parts :-1. The cross entropy loss between the original and reconstructed image2. The KL Divergence between Q(z|x) and P(z|X). This loss makes the learned distribution on z to be similar to some assumed distribution(Gaussian in this case). For more on this, read this [tutorial](https://arxiv.org/pdf/1606.05908). Formula is derived on Appendix B of the [paper](https://arxiv.org/pdf/1312.6114).<jupyter_code>def loss_function(recon_x, x, mu, logvar):
    BCE = F.binary_cross_entropy(recon_x, x.view(-1, 784), reduction='sum')

    KLD = -0.5 * torch.sum(1 + logvar - mu.pow(2) - logvar.exp())

    return BCE + KLD<jupyter_output><empty_output><jupyter_text>Training<jupyter_code>num_epochs = 5
print_per = 100
model.train()
loss_record = []
for epoch in range(num_epochs):
    train_loss = 0
    print_loss = 0
    for i, (images, _) in enumerate(train_loader):
        images = images.reshape(-1, 28*28)
        optimizer.zero_grad()
        recon_batch, mu, logvar = model(images)
        loss = loss_function(recon_batch, images, mu, logvar)
        loss.backward()
        if (epoch == 0):
            loss_record.append(loss.item())
        train_loss += loss.item()
        print_loss += loss.item()
        optimizer.step()
        
        if (i%print_per == 0):
            print("Epoch : {} , Minibatch : {} Loss = {:.4f}".format(epoch+1, i, print_loss))
#             loss_record.append(print_loss)
            print_loss = 0
    print("Epoch {} : Loss = ({:.4f}) ".format(epoch+1, train_loss))<jupyter_output>Epoch : 1 , Minibatch : 0 Loss = 54789.6172
Epoch : 1 , Minibatch : 100 Loss = 2296336.5566
Epoch : 1 , Minibatch : 200 Loss = 1641241.8613
Epoch : 1 , Minibatch : 300 Loss = 1460765.7021
Epoch : 1 , Minibatch : 400 Loss = 1366460.5518
Epoch : 1 , Minibatch : 500 Loss = 1306802.2734
Epoch 1 : Loss = (9380086.2510) 
Epoch : 2 , Minibatch : 0 Loss = 11909.7256
Epoch : 2 , Minibatch : 100 Loss = 1230171.1699
Epoch : 2 , Minibatch : 200 Loss = 1209696.3896
Epoch : 2 , Minibatch : 300 Loss = 1185796.5020
Epoch : 2 , Minibatch : 400 Loss = 1173013.1475
Epoch : 2 , Minibatch : 500 Loss = 1170144.8301
Epoch 2 : Loss = (7125041.2344) 
Epoch : 3 , Minibatch : 0 Loss = 10819.3555
Epoch : 3 , Minibatch : 100 Loss = 1149355.5850
Epoch : 3 , Minibatch : 200 Loss = 1136657.8994
Epoch : 3 , Minibatch : 300 Loss = 1129757.7314
Epoch : 3 , Minibatch : 400 Loss = 1127398.5312
Epoch : 3 , Minibatch : 500 Loss = 1122224.3330
Epoch 3 : Loss = (6785680.2803) 
Epoch : 4 , Minibatch : 0 Loss = 11511.7969
Epoch[...]<jupyter_text>Loss recordAs we can clearly see, the loss is decreasing as we pass more and more number of batches i.e. as we perform more number of iterations. The losses here are only for the first epoch. The fluctuations are due to the use of mini batch gradient descent.<jupyter_code>plt.plot(loss_record)<jupyter_output><empty_output><jupyter_text>Reconstructing a random image from the training set :-<jupyter_code>print(model(images)[0].data[0].numpy().shape)
image1 = images[8]
print(image1.shape)
plt.imshow(model(images)[0].data[8].numpy().reshape(28, 28), cmap='gray')
plt.show(block=True)<jupyter_output>(784,)
torch.Size([784])<jupyter_text>Reconstructing images from the test set :-<jupyter_code>test_loss = 0
print_per = 10
with torch.no_grad():
    for i, (images, _) in enumerate(test_loader):
        images = images.reshape(-1, 28*28)
        recon_batch, mu, logvar = model(images)
        test_loss += loss_function(recon_batch, images, mu, logvar).item()
        if (i%print_per == 0):
            plt.imshow(model(images)[0].data[0].numpy().reshape(28, 28), cmap='gray')
            plt.show(block=True)
image2 = images[1]
print(image2.shape)<jupyter_output>torch.Size([784])<jupyter_text>Original image of some random training example :-<jupyter_code>plt.imshow(images[1].numpy().reshape(28, 28), cmap='gray')
plt.show(block=True)<jupyter_output><empty_output><jupyter_text>Reconstructed image :-<jupyter_code>plt.imshow(model(images)[0].data[1].numpy().reshape(28, 28), cmap='gray')
plt.show(block=True)<jupyter_output><empty_output><jupyter_text>ExperimentsFirst we will calculate mu, logvar fomr two images by passing them through the encoder of our network. We will use them later to perform some experiments<jupyter_code>with torch.no_grad():
    mu1, logvar1 = model.encode(image1)
    std1 = torch.exp(0.5*logvar1)
    mu2, logvar2 = model.encode(image2)
    std2 = torch.exp(0.5*logvar2)<jupyter_output><empty_output><jupyter_text>1. Sampling epsilon values at regular intervals and reconstructing the images We can see that the reconstructed images are not actually same, i.e. their values differ but the look very similar. This means that our network tries to maps many z values (following a particular gaussian) to a singe image.<jupyter_code>with torch.no_grad():
    recon_images1 = []
    for ctr in range(0, 100, 5):
        eps_val = torch.full_like(mu1, fill_value = ctr * 0.01 )
        z_val1 = eps_val.mul(std1).add_(mu1)
        recon_image1 = model.decode(z_val1)
        recon_images1.append(recon_image1)
print(recon_images1[0] - recon_images1[1])
fig=plt.figure(figsize=(28, 28))
columns = 4
rows = 5
for i in range(1, columns*rows +1):
    img = recon_images1[i-1].detach().numpy().reshape(28, 28)
    fig.add_subplot(rows, columns, i)
    plt.imshow(img, cmap="gray")
plt.show()<jupyter_output><empty_output><jupyter_text>2. Changing just one dimension of the latent variable and reconstructing the imagesThis doesn't affect the constructed images a lot since we have 19 more dimensions which point to the same digit so changing one dimension doesn't create a visual effect.<jupyter_code>with torch.no_grad():
    recon_images1 = []
    eps_val = torch.randn_like(mu1)
    for ctr in range(0, 100, 5):
        eps_val[7] = ctr * 0.05 * std1[7] + mu1[7]
        z_val1 = eps_val.mul(std1).add_(mu1)
        recon_image1 = model.decode(z_val1)
        recon_images1.append(recon_image1)
fig=plt.figure(figsize=(28, 28))
columns = 4
rows = 5
for i in range(1, columns*rows +1):
    img = recon_images1[i-1].detach().numpy().reshape(28, 28)
    fig.add_subplot(rows, columns, i)
    plt.imshow(img, cmap="gray")
plt.show()<jupyter_output><empty_output><jupyter_text>3. Transitioning from one digit image to another digit image in the latent space and reconstructing images at regular intervalsIt is important to note that we have used the same value of epsilon to construct z from the images.<jupyter_code>eps_any = torch.randn_like(mu1)
z1 = eps_any.mul(std1).add_(mu1)
z2 = eps_any.mul(std2).add_(mu2)
all_recons = []
for i in range(20):
    z_bet = z1 + torch.full_like(mu1, fill_value = 0.05*i).mul(z2 - z1)
    recon_image = model.decode(z_bet)
    all_recons.append(recon_image)
fig=plt.figure(figsize=(28, 28))
columns = 4
rows = 5
for i in range(1, columns*rows +1):
    img = all_recons[i-1].detach().numpy().reshape(28, 28)
    fig.add_subplot(rows, columns, i)
    plt.imshow(img, cmap="gray")
plt.show()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.12296328481425157, 'fraction_numerical': 0.06245926569628503, 'mean_word_length': 4.109322974472808, 'pattern_counts': {'":': 0, '<': 58, '<?xml version=': 0, '>': 58, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '15', 'dir': 'jupyter-structured-clean-dedup', 'id': '351472', 'license': 'MIT', 'n_tokens_mistral': 3662, 'n_tokens_neox': 3273, 'n_words': 814, 'path': '.ipynb_checkpoints/VAE-checkpoint.ipynb', 'repo_name': 'ac-alpha/VAEs-using-Pytorch', 'size': '9206'}