<jupyter_start><jupyter_text>CS109B Data Science 2: Advanced Topics in Data Science  Lecture 18: Autoencoders Example **Harvard University****Spring 2019****Instructors:** <NAME> and <NAME>Code and ideas taken/borrowed/adapted from Deep Learning From Basics to Practice, by <NAME>, https://dlbasics.com, http://glassner.com, Python utilities for saving and loading files, mostly images and Keras model weights<jupyter_code>#RUN THIS CELL 
import requests
from IPython.core.display import HTML
styles = requests.get("https://raw.githubusercontent.com/Harvard-IACS/2018-CS109A/master/content/styles/cs109.css").text
HTML(styles)
from keras.datasets import mnist
from keras.models import Sequential, Model
from keras.layers import Dense
import numpy as np
#import h5py
#from pathlib import Path
import matplotlib.pyplot as plt

from keras import backend as keras_backend
keras_backend.set_image_data_format('channels_last')
# I will be saving everything so I can re-use them 
save_files = True

import os, sys, inspect
current_dir = os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe())))
sys.path.insert(0, os.path.dirname(current_dir)) # path to parent dir
from DLBasics_Utilities import File_Helper
file_helper = File_Helper(save_files)
random_seed = 52
np.random.seed(random_seed)

# Read MNIST data. Of course we will not be using y_train and y_test
(X_train, y_train), (X_test, y_test) = mnist.load_data()

# Full size of the image 28x28 
N_pixels = X_train.shape[1]*X_train.shape[2]

# Convert to floats
X_train = keras_backend.cast_to_floatx(X_train)
X_test = keras_backend.cast_to_floatx(X_test)

# Normalize the range from [0,255] to [0,1]
X_train /= 255.
X_test /= 255.

# Reshape the data into a grid with one row per sample, each row 784 (28*28) pixels
X_train = X_train.reshape((len(X_train), N_pixels))
X_test = X_test.reshape((len(X_test), N_pixels))

print("X_train.shape = ",X_train.shape, " X_test.shape = ",X_test.shape)
def draw_predictions_set(predictions, filename=None):
    plt.figure(figsize=(8, 4))
    for i in range(5):
        plt.subplot(2, 5, i+1)
        plt.imshow(X_test[i].reshape(28, 28), vmin=0, vmax=1, cmap="gray")
        ax = plt.gca()
        ax.get_xaxis().set_visible(False)
        ax.get_yaxis().set_visible(False)
        plt.subplot(2, 5, i+6)
        plt.imshow(predictions[i].reshape(28, 28), vmin=0, vmax=1, cmap="gray")
        ax = plt.gca()
        ax.get_xaxis().set_visible(False)
        ax.get_yaxis().set_visible(False)
    plt.tight_layout()
    file_helper.save_figure(filename+'-predictions')
    plt.show()
# Build and train our first autoencoder
num_latent_vars = 20

modelAE1 = Sequential()
modelAE1.add(Dense(num_latent_vars, input_dim=N_pixels, activation='relu'))
modelAE1.add(Dense(N_pixels, activation='linear'))
modelAE1.compile(optimizer='adadelta', loss='mse')


weights_filename = "modelAE1-weights"
np.random.seed(52)
if not file_helper.load_model_weights(modelAE1, weights_filename):
    modelAE1.fit(X_train, X_train,
                   epochs=20, batch_size=128, shuffle=True,
                   verbose=2,
                   validation_data=(X_test, X_test))
    file_helper.save_model_weights(modelAE1, weights_filename)
predictions1 = modelAE1.predict(X_test)
draw_predictions_set(predictions1, 'FCC1')
# Build and train our first autoencoder
num_latent_vars = 20

modelAE1_s = Sequential()
modelAE1_s.add(Dense(num_latent_vars, input_dim=N_pixels, activation='relu'))
modelAE1_s.add(Dense(N_pixels, activation='sigmoid'))
modelAE1_s.compile(optimizer='adadelta', loss='binary_crossentropy')


weights_filename = "modelAE1-weights-sigmoid"
np.random.seed(52)
if not file_helper.load_model_weights(modelAE1_s, weights_filename):
    modelAE1_s.fit(X_train, X_train,
                   epochs=20, batch_size=128, shuffle=True,
                   verbose=2,
                   validation_data=(X_test, X_test))
    file_helper.save_model_weights(modelAE1_s, weights_filename)
predictions1 = modelAE1_s.predict(X_test)
draw_predictions_set(predictions1, 'FCC1-s')
from PIL import Image

filepath = file_helper.get_input_file_path("pavlos-gray-28-28.png")
im = Image.open(filepath) 
pix = im.load()
pavlos = np.zeros((1,784))
for y in range(28):
    for x in range(28):
        pavlos[0,(y*28)+x] = pix[x,y][0]/255.
        
predicted_pavlos = modelAE1_s.predict(pavlos)

plt.subplot(1, 2, 1)
plt.imshow(np.reshape(pavlos, (28, 28)), vmin=0, vmax=1, cmap="gray")
ax = plt.gca()
ax.get_xaxis().set_visible(False)
ax.get_yaxis().set_visible(False)
plt.subplot(1, 2, 2)
plt.imshow(np.reshape(predicted_pavlos, (28, 28)), vmin=0, vmax=1, cmap="gray")
ax = plt.gca()
ax.get_xaxis().set_visible(False)
ax.get_yaxis().set_visible(False)
plt.tight_layout()
file_helper.save_figure('Model-FCC-pavlos-pair')
plt.show()
# rebuild model with the Functional API so we can play with the decoder alone
from keras.layers import Input

num_latent_vars = 20

model_encoder_input = Input(shape=(784,))
model_encoder_512 = Dense(512, activation='relu')
model_encoder_256 = Dense(256, activation='relu')
model_encoder_latent = Dense(num_latent_vars, activation='relu')

model_decoder_256 = Dense(256, activation='relu')
model_decoder_512 = Dense(512, activation='relu')
model_decoder_out = Dense(784, activation='sigmoid')

model_encoder_step_1 = model_encoder_512(model_encoder_input)
model_encoder_step_2 = model_encoder_256(model_encoder_step_1)
model_encoder_output = model_encoder_latent(model_encoder_step_2)

model_decoder_step_1 = model_decoder_256(model_encoder_output)
model_decoder_step_2 = model_decoder_512(model_decoder_step_1)
model_decoder_output = model_decoder_out(model_decoder_step_2)

model_AE_F = Model(model_encoder_input, model_decoder_output)

model_encoder_only_model = Model(model_encoder_input, model_encoder_output)

model_decoder_only_input = Input(shape=(num_latent_vars,))
model_decoder_only_step_1 = model_decoder_256(model_decoder_only_input)
model_decoder_only_step_2 = model_decoder_512(model_decoder_only_step_1)
model_decoder_only_output = model_decoder_out(model_decoder_only_step_2)
model_decoder_only_model = Model(model_decoder_only_input, model_decoder_only_output)
model_AE_F.compile(optimizer='adadelta', loss='binary_crossentropy')

weights_filename = "model-AE-F-weights"
np.random.seed(52)
if not file_helper.load_model_weights(model_AE_F, weights_filename):
    model_AE_F.fit(X_train, X_train,
               epochs=20, batch_size=128, shuffle=True,
               verbose=2,
               validation_data=(X_test, X_test))
    file_helper.save_model_weights(model_AE_F, weights_filename)
# show the input data, its latent values, and the corresponding predicted images
np.random.seed(random_seed)
encoder_predictions = model_encoder_only_model.predict(X_test)

plt.figure(figsize=(8, 5))
latent_min = np.min(encoder_predictions[0:5])
latent_max = np.max(encoder_predictions[0:5])
for i in range(5):
    plt.subplot(3, 5, i+1)
    plt.imshow(X_test[i].reshape(28, 28), vmin=0, vmax=1, cmap="gray")
    ax = plt.gca()
    ax.get_xaxis().set_visible(False)
    ax.get_yaxis().set_visible(False)
    
    plt.subplot(3, 5, i+6)
    plt.bar(np.arange(len(encoder_predictions[i])), encoder_predictions[i])
    plt.xticks([], [])
    plt.ylim(latent_min, latent_max)
    
    plt.subplot(3, 5, i+11)
    decoder_model_input = np.resize(encoder_predictions[i], (1, len(encoder_predictions[i])))
    decoder_prediction = model_decoder_only_model.predict(decoder_model_input)
    plt.imshow(decoder_prediction.reshape(28, 28), vmin=0, vmax=1, cmap="gray")
    ax = plt.gca()
    ax.get_xaxis().set_visible(False)
    ax.get_yaxis().set_visible(False)
plt.tight_layout()
file_helper.save_figure("model-F-latents-and-output")
plt.show()
# show the latent values and the corresponding predicted images
# this makes big vertical spaces between the 
def show_latents_and_predictions(predictions, filename=None):
    plt.figure(figsize=(8, 4))
    latent_min = np.min(predictions[0:5])
    latent_max = np.max(predictions[0:5])
    for i in range(5):
        plt.subplot(2, 5, i+1)
        plt.bar(np.arange(len(predictions[i])), predictions[i])
        plt.xticks([], [])
        plt.ylim(latent_min, latent_max)

        plt.subplot(2, 5, i+6)
        decoder_model_input = np.resize(predictions[i], (1, len(predictions[i])))
        decoder_prediction = model_decoder_only_model.predict(decoder_model_input)
        plt.imshow(decoder_prediction.reshape(28, 28), vmin=0, vmax=1, cmap="gray")
        ax = plt.gca()
        ax.get_xaxis().set_visible(False)
        ax.get_yaxis().set_visible(False)
    plt.tight_layout()
    file_helper.save_figure(filename)
    plt.show()
# add [-noise to all values
encoder_predictions = np.array(model_encoder_only_model.predict(X_test))
np.random.seed(random_seed)
noise = 10
for i in range(encoder_predictions.shape[0]):
    for j in range(len(encoder_predictions[i])):
        encoder_predictions[i][j] += np.random.uniform(low=-noise, high=noise)
show_latents_and_predictions(encoder_predictions, 'NB2-MLP-AE5-prediction-latent-values-with-noise-1')<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.12156390318694557, 'fraction_numerical': 0.030445734311685468, 'mean_word_length': 5.3460736622654625, 'pattern_counts': {'":': 0, '<': 8, '<?xml version=': 0, '>': 8, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '110260', 'license': 'MIT', 'n_tokens_mistral': 3520, 'n_tokens_neox': 3319, 'n_words': 594, 'path': 'content/lectures/lecture18/notes/cs109b-lecture18-example.ipynb', 'repo_name': 'phoebewong/2019-CS109B', 'size': '9158'}