<jupyter_start><jupyter_text>Denoising AutoencoderBuild a stacked denoising autoencoder to work on fashion-MNIST data set. "Hands-On Machine Learning", by <NAME>, is the basis for much of the code. https://github.com/ageron/handson-ml2May also try a denoising convolutional autoencoder.<jupyter_code>import numpy as np
import datetime
import matplotlib as mpl
import matplotlib.pyplot as plt
import pandas as pd

import tensorflow as tf
from tensorflow import keras
import tensorboard

print('TensorFlow version: ', tf.__version__)
print('Keras version: ', keras.__version__)
print('Tensorboard version:', tensorboard.__version__)

%matplotlib inline<jupyter_output>TensorFlow version:  2.0.0
Keras version:  2.2.4-tf
Tensorboard version: 2.0.0<jupyter_text>Left align tables:<jupyter_code>%%html
<style>
table {float:left}
</style><jupyter_output><empty_output><jupyter_text>1.0 Load and Prepare Data<jupyter_code># load fashion MNIST
fashion_mnist = keras.datasets.fashion_mnist
(X_train_all, y_train_all), (X_test, y_test) = fashion_mnist.load_data()
# check the shape of the data sets
print('X_train_full shape:', X_train_all.shape)
print('y_train_full shape:', y_train_all.shape)
print('X_test shape:', X_test.shape)
print('y_test shape:', y_test.shape)
class_names = ["T-shirt/top", "Trouser", "Pullover", "Dress", "Coat",
               "Sandal", "Shirt", "Sneaker", "Bag", "Ankle boot"]
# split the data between train and validation sets, and scale

X_valid, X_train = X_train_all[:5000] / 255.0, X_train_all[5000:] / 255.0
y_valid, y_train = y_train_all[:5000], y_train_all[5000:]

# also scale the X_test
X_test = X_test / 255.0

print('X_valid shape:', X_valid.shape)
print('y_valid shape:', y_valid.shape)
print('X_train shape:', X_train.shape)
print('y_train shape:', y_train.shape)<jupyter_output>X_valid shape: (5000, 28, 28)
y_valid shape: (5000,)
X_train shape: (55000, 28, 28)
y_train shape: (55000,)<jupyter_text>2.0 Stacked Denoising AutoencoderBuild a stacked denoising autoencoder.This is a simple stacked autoencoder, except that noise is added at the first layer.First, we will define the "rounded accuracy" metric. Geron, in his book, says that "you might be tempted to use the accuracy metric, but it would not work properly, since this metric expects the labels to be either 0 or 1 for each pixel. You can easily work around this problem by creating a custom metric that computes the accuracy after rounding the targets and predictions to 0 or 1."<jupyter_code>def rounded_accuracy(y_true, y_pred):
    return keras.metrics.binary_accuracy(tf.round(y_true), tf.round(y_pred))
tf.random.set_seed(42)
np.random.seed(42)

denoising_encoder = keras.models.Sequential([
    keras.layers.Flatten(input_shape=[28, 28]),
    keras.layers.GaussianNoise(0.2),
    keras.layers.Dense(100, activation="selu"),
    keras.layers.Dense(30, activation="selu")
])
denoising_decoder = keras.models.Sequential([
    keras.layers.Dense(100, activation="selu", input_shape=[30]),
    keras.layers.Dense(28 * 28, activation="sigmoid"),
    keras.layers.Reshape([28, 28])
])
denoising_ae = keras.models.Sequential([denoising_encoder, denoising_decoder])
denoising_ae.compile(loss="binary_crossentropy", optimizer=keras.optimizers.SGD(lr=1.0),metrics=[rounded_accuracy])

denoising_encoder.summary()
denoising_decoder.summary()
history = denoising_ae.fit(X_train, X_train, epochs=10,
                           validation_data=[X_valid, X_valid])
def plot_reconstructions(model, index_list, X_valid):
    """Plot some original images, and their reconstructions
    
    Parameters
    ===========
    model : keras model
        Autoencoder model
        
    index_list : list
        List of indices. These indices correspond to the index of the X_valid images
        that will be shown
        
    X_valid : numpy array
        X_valid set

    """
    
    reconstructions = model.predict(X_valid)
    
    # get the length of index_list to set number of
    # images to plot
    n_images = len(index_list)

    # Plot a random sample
    fig, axes = plt.subplots(2, n_images,figsize=(n_images*1.5,3),dpi=150)
    # fig.suptitle('Digits for Sample %i' %num, size=15, x=0.2)

    for i in range(0, n_images):
        axes[0][i].imshow(np.reshape(X_valid[index_list[i],:,:],[28,28]), cmap='Greys_r')
        axes[0][i].axis('off')
        axes[0][i].set_title(str(index_list[i]))
        axes[1][i].imshow(np.reshape(reconstructions[index_list[i],:,:],[28,28]), cmap='Greys_r')
        axes[1][i].axis('off')
        
    plt.show()
# plot a random number of items
import random
index_list = random.sample(range(0,len(X_valid)), 5)

# add in noise to the images with keras Gaussian Noise layer
noise = keras.layers.GaussianNoise(0.2, dtype='float32')

plot_reconstructions(denoising_ae, index_list, noise(X_valid, training=True))<jupyter_output><empty_output><jupyter_text>Now we will add dropout for the first layer.<jupyter_code>tf.random.set_seed(42)
np.random.seed(42)

dropout_encoder = keras.models.Sequential([
    keras.layers.Flatten(input_shape=[28, 28]),
    keras.layers.Dropout(0.5),
    keras.layers.Dense(100, activation="selu"),
    keras.layers.Dense(30, activation="selu")
])
dropout_decoder = keras.models.Sequential([
    keras.layers.Dense(100, activation="selu", input_shape=[30]),
    keras.layers.Dense(28 * 28, activation="sigmoid"),
    keras.layers.Reshape([28, 28])
])
dropout_ae = keras.models.Sequential([dropout_encoder, dropout_decoder])
dropout_ae.compile(loss="binary_crossentropy", optimizer=keras.optimizers.SGD(lr=1.0),
                   metrics=[rounded_accuracy])

dropout_encoder.summary()
dropout_decoder.summary()
history = dropout_ae.fit(X_train, X_train, epochs=10,
                         validation_data=[X_valid, X_valid])
# plot a random number of items
import random
index_list = random.sample(range(0,len(X_valid)), 5)

# add dropout to the images with keras dropout layer
dropout = keras.layers.Dropout(0.5, dtype='float32')

plot_reconstructions(denoising_ae, index_list, dropout(X_valid, training=True))<jupyter_output><empty_output><jupyter_text>Denoising Convolutional AutoencoderBuild a denoising convolutional autoencoder.<jupyter_code>tf.random.set_seed(32)
np.random.seed(32)

noise_conv_encoder = keras.models.Sequential([
    keras.layers.Reshape([28, 28, 1], input_shape=[28, 28]),
    keras.layers.GaussianNoise(0.2),
    keras.layers.Conv2D(16, kernel_size=3, padding="SAME", activation="selu"),
    keras.layers.MaxPool2D(pool_size=2),
    keras.layers.Conv2D(32, kernel_size=3, padding="SAME", activation="selu"),
    keras.layers.MaxPool2D(pool_size=2),
    keras.layers.Conv2D(64, kernel_size=3, padding="SAME", activation="selu"),
    keras.layers.MaxPool2D(pool_size=2)
])
noise_conv_decoder = keras.models.Sequential([
    keras.layers.Conv2DTranspose(32, kernel_size=3, strides=2, padding="VALID", activation="selu",
                                 input_shape=[3, 3, 64]),
    keras.layers.Conv2DTranspose(16, kernel_size=3, strides=2, padding="SAME", activation="selu"),
    keras.layers.Conv2DTranspose(1, kernel_size=3, strides=2, padding="SAME", activation="sigmoid"),
    keras.layers.Reshape([28, 28])
])
noise_conv_ae = keras.models.Sequential([noise_conv_encoder, noise_conv_decoder])

noise_conv_ae.compile(loss="binary_crossentropy", optimizer=keras.optimizers.Adam(),metrics=[rounded_accuracy])

noise_conv_encoder.summary()
noise_conv_decoder.summary()
history = noise_conv_ae.fit(X_train, X_train, epochs=5,
                      validation_data=[X_valid, X_valid])
# plot a random number of items
import random
index_list = random.sample(range(0,len(X_valid)), 5)

# add in noise to the images with keras Gaussian Noise layer
noise = keras.layers.GaussianNoise(0.2, dtype='float32')

plot_reconstructions(noise_conv_ae, index_list, noise(X_valid, training=True))<jupyter_output><empty_output><jupyter_text>Lets compare the results from the denoising stacked autoencoder, and the denoising convolutional autoencoder.<jupyter_code># plot a random number of items
import random
index_list = random.sample(range(0,len(X_valid)), 5)

# add in noise to the images with keras Gaussian Noise layer
noise = keras.layers.GaussianNoise(0.2, dtype='float32')
data_input = noise(X_valid, training=True)



# plot original images
print("Original Images:")
n_images = len(index_list)
# Plot a random sample
fig, axes = plt.subplots(1, n_images,figsize=(n_images*1.5,3),dpi=150)

for i in range(0, n_images):
    axes[i].imshow(np.reshape(X_valid[index_list[i],:,:],[28,28]), cmap='Greys_r')
    axes[i].axis('off')
    axes[i].set_title(str(index_list[i]))

plt.show()

print("Denoising Autoencoder: ")
plot_reconstructions(denoising_ae, index_list, data_input)
print("Denoising Convoluional Autoencoder: ")
plot_reconstructions(noise_conv_ae, index_list, data_input)<jupyter_output>Original Images:
{'fraction_non_alphanumeric': 0.12712532372480576, 'fraction_numerical': 0.028487782907330256, 'mean_word_length': 5.33071988595866, 'pattern_counts': {'":': 0, '<': 29, '<?xml version=': 0, '>': 29, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '7', 'dir': 'jupyter-structured-clean-dedup', 'id': '173986', 'license': 'MIT', 'n_tokens_mistral': 3270, 'n_tokens_neox': 3063, 'n_words': 667, 'path': 'notebooks/3.0-denoising-autoencoder.ipynb', 'repo_name': 'tvhahn/autoencoder-sandbox', 'size': '8889'}