<jupyter_start><jupyter_text>Learning dynamics for nonlinear autoencoder networks using ReLU activation.This notebook provides the code to produce Figure 6 in the paper: "Learning dynamics of linear denoising autoencoders". (ICML 2018)**NOTE:**The code provided below to rerun all the experiments may take some time to finish (and it is recommended to save the results to disk once completed). However, to just reproduce the plot in Figure 6 of the paper, the learning dynamics data is already available in the `data` folder and can simply be loaded in to create the plot.<jupyter_code># import required modules
import matplotlib.pyplot as plt
import numpy as np
import torch
from torch import nn
from torch.autograd import Variable
from torchvision import transforms
from torchvision.datasets import MNIST
from torchvision.datasets import CIFAR10

# custom imports
from src.nonlinear_ae_net.nonlinear_ae_net import create_model
from src.nonlinear_ae_net.nonlinear_ae_net import train_model<jupyter_output><empty_output><jupyter_text>--- MNIST ---  Load the data<jupyter_code>trans = transforms.Compose([transforms.ToTensor()])

# if not exist, download mnist dataset
train_set = MNIST(root="data", train=True, transform=trans, download=True)
x_train = train_set.train_data.numpy()
x_train = x_train.astype('float32') / 255.
x_train_mnist = x_train.reshape((len(x_train), np.prod(x_train.shape[1:])))
print(x_train_mnist.shape)<jupyter_output>(60000, 784)<jupyter_text>Train models<jupyter_code># save dynamics data
save = False

# D_in is input dimension;
# H is hidden dimension; D_out is output dimension.
N, D_in, H, D_out = 50000, 784, 256, 784

# Create input tensors.
x = Variable(torch.FloatTensor(x_train_mnist[:N]), requires_grad=False).cuda()

# specify reconstruction loss
loss_fn = nn.MSELoss()

# hyperparameters
learning_rate = 0.9
n_epoch = 2000
weight_decay = 0.0045
noise_level = 2

# dynamics without regularization
print('ReLU')
model_null = create_model(D_in, H, D_out, act_fn='relu')
dyns_relu_mnist = train_model(x, model_null, loss_fn, n_epoch, learning_rate)
print('ReLU - finished')

# dynamics with regularization
print('ReLU with regularization')
model_reg = create_model(D_in, H, D_out, act_fn='relu')
reg_dyns_relu_mnist = train_model(x, model_reg, loss_fn, n_epoch, learning_rate, weight_decay)
print('ReLU with regularization - finished')

# dynamics with gaussian noise
print('ReLU with noise')
model_noise = create_model(D_in, H, D_out, act_fn='relu', noise=noise_level)
noise_dyns_relu_mnist = train_model(x, model_noise, loss_fn, n_epoch, learning_rate)
print('ReLU with noise - finished')

if save:
    np.save('data/mnist_relu_none_highres', dyns_relu_mnist)
    np.save('data/mnist_relu_reg_highres', reg_dyns_relu_mnist)
    np.save('data/mnist_relu_noise_highres_1', noise_dyns_relu_mnist)<jupyter_output>ReLU
ReLU - finished
ReLU with regularization
ReLU with regularization - finished
ReLU with noise
ReLU with noise - finished<jupyter_text>--- CIFAR-10 ---  Load the data<jupyter_code>train_set = CIFAR10(root="data", train=True, transform=trans, download=True)
x_train = train_set.train_data
x_train = x_train.astype('float32')
x_train /= 255
x_train_cifar_10 = x_train.reshape((len(x_train), np.prod(x_train.shape[1:])))
print(x_train_cifar_10.shape)<jupyter_output>Files already downloaded and verified
(50000, 3072)<jupyter_text>Train models<jupyter_code># save dynamics data
save = False

# D_in is input dimension;
# H is hidden dimension; D_out is output dimension.
N, D_in, H, D_out = 30000, 3072, 512, 3072

# Create input Tensor
x = Variable(torch.FloatTensor(x_train_cifar_10[:N]), requires_grad=False).cuda()

# specify loss function
loss_fn = nn.MSELoss()

# hyperparameters
learning_rate = 0.9
n_epoch = 2000

# dynamics without regularization
print('ReLU')
model_null = create_model(D_in, H, D_out, act_fn='relu')
dyns_relu_cifar10 = train_model(x, model_null, loss_fn, n_epoch, learning_rate)
print('ReLU - finished')

# dynamics with regularization
print('ReLU with regularization')
model_reg = create_model(D_in, H, D_out, act_fn='relu')
reg_dyns_relu_cifar10 = train_model(x, model_reg, loss_fn, n_epoch, learning_rate, weight_decay)
print('ReLU with regularization - finished')

# dynamics with gaussian noise
print('ReLU with noise')
model_noise = create_model(D_in, H, D_out, act_fn='relu', noise=noise_level)
noise_dyns_relu_cifar10 = train_model(x, model_noise, loss_fn, n_epoch, learning_rate)
print('ReLU with noise - finished')

if save:
    np.save('data/cifar10_relu_none_highres', dyns_relu_cifar10)
    np.save('data/cifar10_relu_reg_highres', reg_dyns_relu_cifar10)
    np.save('data/cifar10_relu_noise_highres_1', noise_dyns_relu_cifar10)<jupyter_output>ReLU
ReLU - finished
ReLU with regularization
ReLU with regularization - finished
ReLU with noise
ReLU with noise - finished<jupyter_text>Plot results<jupyter_code># load saved dynamics data
load = True

if load:
    # mnist relu
    dyns_relu_mnist = np.load('data/mnist_relu_none_highres.npy')
    reg_dyns_relu_mnist = np.load('data/mnist_relu_reg_highres.npy')
    noise_dyns_relu_mnist = np.load('data/mnist_relu_noise_highres_1.npy')
    
    # cifar10 relu
    dyns_relu_cifar10 = np.load('data/cifar10_relu_none_highres.npy')
    reg_dyns_relu_cifar10 = np.load('data/cifar10_relu_reg_highres.npy')
    noise_dyns_relu_cifar10 = np.load('data/cifar10_relu_noise_highres_1.npy')

fig, (ax1, ax2) = plt.subplots(1, 2, figsize=(12, 4), sharey='row', sharex='col')
axes = [ax1, ax2]

# plot relu results mnist
x = range(21)
ax1.plot(dyns_relu_mnist[:,0], c='blue', label='AE')
ax1.plot(dyns_relu_mnist[:,1], c='blue')
ax1.plot(dyns_relu_mnist[:,2], c='blue')
ax1.plot(dyns_relu_mnist[:,3], c='blue')
ax1.plot(reg_dyns_relu_mnist[:,0], c='orange', label='WDAE')
ax1.plot(reg_dyns_relu_mnist[:,1], c='orange')
ax1.plot(reg_dyns_relu_mnist[:,2], c='orange')
ax1.plot(reg_dyns_relu_mnist[:,3], c='orange')
ax1.plot(noise_dyns_relu_mnist[:,0], c='green', label='DAE')
ax1.plot(noise_dyns_relu_mnist[:,1], c='green')
ax1.plot(noise_dyns_relu_mnist[:,2], c='green')
ax1.plot(noise_dyns_relu_mnist[:,3], c='green')
ax1.set_xticks(np.linspace(1, 200, 6))
ax1.set_xticklabels(np.linspace(0, 2000, 6, dtype=int))
ax1.set_xlabel('t (epoch)')
ax1.set_ylabel('Mapping')
ax1.set_title('MNIST')
ax1.legend(loc='lower right')

# plot relu results for cifar 10
ax2.plot(dyns_relu_cifar10[:,0], c='blue')
ax2.plot(dyns_relu_cifar10[:,1], c='blue')
ax2.plot(dyns_relu_cifar10[:,2], c='blue')
ax2.plot(dyns_relu_cifar10[:,3], c='blue')
ax2.plot(reg_dyns_relu_cifar10[:,0], c='orange')
ax2.plot(reg_dyns_relu_cifar10[:,1], c='orange')
ax2.plot(reg_dyns_relu_cifar10[:,2], c='orange')
ax2.plot(reg_dyns_relu_cifar10[:,3], c='orange')
ax2.plot(noise_dyns_relu_cifar10[:,0], c='green')
ax2.plot(noise_dyns_relu_cifar10[:,1], c='green')
ax2.plot(noise_dyns_relu_cifar10[:,2], c='green')
ax2.plot(noise_dyns_relu_cifar10[:,3], c='green')
ax2.set_xticks(np.linspace(1, 200, 6))
ax2.set_xticklabels(np.linspace(0, 2000, 6, dtype=int))
ax2.set_xlabel('t (epoch)')
ax2.set_title('CIFAR-10')
ax2.set_ylabel('')

plt.show()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.12844164919636616, 'fraction_numerical': 0.03508036338225017, 'mean_word_length': 7.1970217640320735, 'pattern_counts': {'":': 0, '<': 21, '<?xml version=': 0, '>': 21, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '6', 'dir': 'jupyter-structured-clean-dedup', 'id': '223391', 'license': 'MIT', 'n_tokens_mistral': 3016, 'n_tokens_neox': 2805, 'n_words': 565, 'path': 'Figure_6_nonlinear_dae_learning_dynamics.ipynb', 'repo_name': 'arnupretorius/lindaedynamics_icml2018', 'size': '7155'}