<jupyter_start><jupyter_text>*Accompanying code examples of the book "Introduction to Artificial Neural Networks and Deep Learning: A Practical Guide with Applications in Python" by [<NAME>](https://sebastianraschka.com). All code examples are released under the [MIT license](https://github.com/rasbt/deep-learning-book/blob/master/LICENSE). If you find this content useful, please consider supporting the work by buying a [copy of the book](https://leanpub.com/ann-and-deeplearning).*  Other code examples and content are available on [GitHub](https://github.com/rasbt/deep-learning-book). The PDF and ebook versions of the book are available through [Leanpub](https://leanpub.com/ann-and-deeplearning).<jupyter_code>%load_ext watermark
%watermark -a '<NAME>' -v -p torch<jupyter_output><NAME> 

CPython 3.6.4
IPython 6.2.1

torch 0.4.0<jupyter_text>- Runs on CPU or GPU (if available)  Model Zoo -- Variational Autoencoder A simple variational autoencoder that compresses 768-pixel MNIST images down to a 15-pixel latent vector representation.  Imports<jupyter_code>import numpy as np
import torch
import torch.nn.functional as F
from torch.utils.data import DataLoader
from torchvision import datasets
from torchvision import transforms
##########################
### SETTINGS
##########################

# Device
device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
print('Device:', device)

# Hyperparameters
random_seed = 123
learning_rate = 0.001
num_epochs = 5
batch_size = 128

# Architecture
num_features = 784
num_hidden_1 = 500
num_latent = 15


##########################
### MNIST DATASET
##########################

# Note transforms.ToTensor() scales input images
# to 0-1 range
train_dataset = datasets.MNIST(root='data', 
                               train=True, 
                               transform=transforms.ToTensor(),
                               download=True)

test_dataset = datasets.MNIST(root='data', 
                              train=False, 
                              transform=transforms.ToTensor())


train_loader = DataLoader(dataset=train_dataset, 
                          batch_size=batch_size, 
                          shuffle=True)

test_loader = DataLoader(dataset=test_dataset, 
                         batch_size=batch_size, 
                         shuffle=False)

# Checking the dataset
for images, labels in train_loader:  
    print('Image batch dimensions:', images.shape)
    print('Image label dimensions:', labels.shape)
    break<jupyter_output>Device: cpu
Image batch dimensions: torch.Size([128, 1, 28, 28])
Image label dimensions: torch.Size([128])<jupyter_text>Model<jupyter_code>##########################
### MODEL
##########################

class VariationalAutoencoder(torch.nn.Module):

    def __init__(self, num_features, num_hidden_1, num_latent):
        super(VariationalAutoencoder, self).__init__()
        
        ### ENCODER
        self.hidden_1 = torch.nn.Linear(num_features, num_hidden_1)
        self.z_mean = torch.nn.Linear(num_hidden_1, num_latent)
        self.z_log_var = torch.nn.Linear(num_hidden_1, num_latent)
        
        
        ### DECODER
        self.linear_3 = torch.nn.Linear(num_latent, num_hidden_1)
        self.linear_4 = torch.nn.Linear(num_hidden_1, num_features)

    def reparameterize(self, z_mu, z_log_var):
        # Sample epsilon from standard normal distribution
        eps = torch.randn(z_mu.size(0), z_mu.size(1)).to(device)
        # note that log(x^2) = 2*log(x); hence divide by 2 to get std_dev
        z = z_mu + eps * torch.exp(z_log_var/2.) 
        return z
        

    def forward(self, x):
        
        ### ENCODER
        x = self.hidden_1(x)
        x = F.relu(x)
        z_mean = self.z_mean(x)
        z_log_var = self.z_log_var(x)
        encoded = self.reparameterize(z_mean, z_log_var)
        
        ### DECODER
        x = self.linear_3(encoded)
        x = F.relu(x)
        x = self.linear_4(x)
        decoded = F.sigmoid(x)
        
        return z_mean, z_log_var, encoded, decoded

    
torch.manual_seed(random_seed)
model = VariationalAutoencoder(num_features,
                               num_hidden_1,
                               num_latent)
model = model.to(device)
    

##########################
### COST AND OPTIMIZER
##########################

optimizer = torch.optim.Adam(model.parameters(), lr=learning_rate)<jupyter_output><empty_output><jupyter_text>Training<jupyter_code>for epoch in range(num_epochs):
    for batch_idx, (features, targets) in enumerate(train_loader):
        
        # don't need labels, only the images (features)
        features = features.view(-1, 28*28).to(device)

        ### FORWARD AND BACK PROP
        z_mean, z_log_var, encoded, decoded = model(features)

        # cost = reconstruction loss + Kullback-Leibler divergence
        kl_divergence = (0.5 * (z_mean**2 + 
                                torch.exp(z_log_var) - z_log_var - 1)).sum()
        pixelwise_bce = F.binary_cross_entropy(decoded, features, size_average=False)
        cost = kl_divergence + pixelwise_bce
        
        optimizer.zero_grad()
        cost.backward()
        
        ### UPDATE MODEL PARAMETERS
        optimizer.step()
        
        ### LOGGING
        if not batch_idx % 50:
            print ('Epoch: %03d/%03d | Batch %03d/%03d | Cost: %.4f' 
                   %(epoch+1, num_epochs, batch_idx, 
                     len(train_dataset)//batch_size, cost))<jupyter_output>Epoch: 001/005 | Batch 000/468 | Cost: 70690.5234
Epoch: 001/005 | Batch 050/468 | Cost: 26577.2266
Epoch: 001/005 | Batch 100/468 | Cost: 22768.0684
Epoch: 001/005 | Batch 150/468 | Cost: 19962.9766
Epoch: 001/005 | Batch 200/468 | Cost: 18513.4980
Epoch: 001/005 | Batch 250/468 | Cost: 18312.0352
Epoch: 001/005 | Batch 300/468 | Cost: 17719.9023
Epoch: 001/005 | Batch 350/468 | Cost: 17178.9941
Epoch: 001/005 | Batch 400/468 | Cost: 16102.8477
Epoch: 001/005 | Batch 450/468 | Cost: 16452.7461
Epoch: 002/005 | Batch 000/468 | Cost: 15541.4561
Epoch: 002/005 | Batch 050/468 | Cost: 15537.3926
Epoch: 002/005 | Batch 100/468 | Cost: 15945.7266
Epoch: 002/005 | Batch 150/468 | Cost: 15082.3652
Epoch: 002/005 | Batch 200/468 | Cost: 15397.8359
Epoch: 002/005 | Batch 250/468 | Cost: 14995.3613
Epoch: 002/005 | Batch 300/468 | Cost: 15054.1230
Epoch: 002/005 | Batch 350/468 | Cost: 15059.7988
Epoch: 002/005 | Batch 400/468 | Cost: 14945.8359
Epoch: 002/005 | Batch 450/468 | Cost: 14754.8281
[...]<jupyter_text>Evaluation<jupyter_code>%matplotlib inline
import matplotlib.pyplot as plt

##########################
### VISUALIZATION
##########################

n_images = 15
image_width = 28

fig, axes = plt.subplots(nrows=2, ncols=n_images, 
                         sharex=True, sharey=True, figsize=(20, 2.5))
orig_images = features[:n_images]
decoded_images = decoded[:n_images]

for i in range(n_images):
    for ax, img in zip(axes, [orig_images, decoded_images]):
        ax[i].imshow(img[i].detach().reshape((image_width, image_width)), cmap='binary')<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.14187192118226602, 'fraction_numerical': 0.07276565798733287, 'mean_word_length': 2.96319018404908, 'pattern_counts': {'":': 0, '<': 21, '<?xml version=': 0, '>': 21, 'https://': 5, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '5', 'dir': 'jupyter-structured-clean-dedup', 'id': '224028', 'license': 'MIT', 'n_tokens_mistral': 2727, 'n_tokens_neox': 2320, 'n_words': 539, 'path': 'code/model_zoo/pytorch_ipynb/autoencoder-var.ipynb', 'repo_name': 'Jules-Diez/deep-learning-book', 'size': '7138'}