<jupyter_start><jupyter_text>Autoencoders [Keras]---- Author: <NAME>- GitHub: [github.com/diegoinacio](https://github.com/diegoinacio)- Notebook: [autoencoder_Keras.ipynb](https://github.com/diegoinacio/machine-learning-notebooks/blob/master/Deep-Learning-Models/autoencoder_Keras.ipynb)---Implementation of *Autoencoders* using Keras library.<jupyter_code>%matplotlib inline
import matplotlib
import matplotlib.pyplot as plt
import numpy as np

import tensorflow as tf
plt.rcParams['figure.figsize'] = (16, 8)<jupyter_output><empty_output><jupyter_text>Data exploration---<jupyter_code>fashion_mnist = tf.keras.datasets.fashion_mnist
[X_train, Y_train],[X_test, Y_test] = fashion_mnist.load_data()

print('X_train:', X_train.shape)
print('Y_train:', Y_train.shape)
print('X_test:', X_test.shape)
print('Y_test:', Y_test.shape)
# Label categories
objects = [
    'T-shirt/top', 'Trouser/pants',
    'Pullover shirt', 'Dress',
    'Coat', 'Sandal', 'Shirt',
    'Sneaker', 'Bag', 'Ankle', 'boot'
]

# Get dimensions
N1, N2 = X_train[0].shape
fig, AX = plt.subplots(3, 6, sharex=True, sharey=True)

np.random.seed(12345)
for ax in AX.ravel():
    rindex = np.random.randint(Y_train.size)
    ax.imshow(X_train[rindex])
    label = Y_train[rindex]
    ax.set_title(f'{objects[label]} ({label})')
plt.grid(False)
# data preparation
# scales, dimensions and dtypes
x_train, X_train = [X_train/255]*2
x_test, X_test = [X_test/255]*2

x_train = x_train.astype(np.float32).reshape(-1, N1*N2)
X_train = X_train.astype(np.float32).reshape(-1, N1, N2, 1)
x_test = x_test.astype(np.float32).reshape(-1, N1*N2)
X_test = X_test.astype(np.float32).reshape(-1, N1, N2, 1)

print('x_train:', x_train.shape)
print('X_train:', X_train.shape)
print('x_test:', x_test.shape)
print('X_test:', X_test.shape)<jupyter_output>x_train: (60000, 784)
X_train: (60000, 28, 28, 1)
x_test: (10000, 784)
X_test: (10000, 28, 28, 1)<jupyter_text>Shallow Autoencoder---![shallow autoencoder](sourceimages/autoencoder_shallow.png "Shallow Autoencoder")<jupyter_code># Number of neurons on the bottleneck hidden layer
neurons = 64

# m is the number of examples
# n_x is the input size 28x28=784
m, n_x = x_train.shape

# Model
encoder_s = tf.keras.Sequential([
    tf.keras.layers.Input(n_x),
    tf.keras.layers.Dense(neurons, activation='relu')
], name='Shallow-Encoder')

decoder_s = tf.keras.Sequential([
    tf.keras.layers.Input(neurons),
    tf.keras.layers.Dense(n_x, activation='sigmoid')
], name='Shallow-Decoder')


autoencoder_s = tf.keras.Sequential([
    encoder_s.input,
    encoder_s.layers[0],
    decoder_s.input,
    decoder_s.layers[0]
], name='Shallow-Autoencoder')

autoencoder_s.summary()
# Compile model
autoencoder_s.compile(
    optimizer='adadelta',
    loss='binary_crossentropy'
)

# Train model
for i in range(5):
    print(f'\nepochs: {i*100:04d} - {(i + 1)*100:04d}')
    autoencoder_s.fit(
        x_train, x_train,
        epochs=99,
        verbose=0
    )
    autoencoder_s.fit(
        x_train, x_train,
        epochs=1,
        validation_data=(x_test, x_test)
    )
fig, AX = plt.subplots(3, 6, figsize=(20, 10))

np.random.seed(1234)
for i in range(6):
    index = np.argwhere(Y_test == i)[:,0]
    index = np.random.choice(index)
    label = Y_test[index]
    
    AX[0][i].imshow(X_test[index][...,0])
    AX[0][i].set_title(f'{objects[label]} ({label})')
    if not i: AX[0][i].set_ylabel('Input', size=16)
    
    encoded = encoder_s.predict(x_test[index].reshape(1, -1))
    
    AX[1][i].imshow(encoded.reshape(8, 8))
    if not i: AX[1][i].set_ylabel('Encoded', size=16)
    
    decoded = decoder_s.predict(encoded)
    
    AX[2][i].imshow(decoded.reshape(N1, N2))
    if not i: AX[2][i].set_ylabel('Decoded', size=16)<jupyter_output><empty_output><jupyter_text>Deep Autoencoder---![deep autoencoder](sourceimages/autoencoder_deep.png "Deep Autoencoder")<jupyter_code># m is the number of examples
# n_x is the input size 28x28=784
m, n_x = x_train.shape

# Model
encoder_d = tf.keras.Sequential([
    tf.keras.layers.Input(n_x),
    tf.keras.layers.Dense(512, activation='relu'),
    tf.keras.layers.Dense(256, activation='relu'),
    tf.keras.layers.Dense(128, activation='relu'),
    tf.keras.layers.Dense(64, activation='relu')
], name='Deep-Encoder')

decoder_d = tf.keras.Sequential([
    tf.keras.layers.Input(64),
    tf.keras.layers.Dense(128, activation='relu'),
    tf.keras.layers.Dense(256, activation='relu'),
    tf.keras.layers.Dense(512, activation='relu'),
    tf.keras.layers.Dense(n_x, activation='sigmoid')
], name='Deep-Decoder')


autoencoder_d = tf.keras.Sequential([
    encoder_d.input,
    *encoder_d.layers,
    decoder_d.input,
    *decoder_d.layers
], name='Deep-Autoencoder')

autoencoder_d.summary()
# Compile model
autoencoder_d.compile(
    optimizer='adadelta',
    loss='binary_crossentropy'
)

# Train model
for i in range(5):
    print(f'\nepochs: {i*100:04d} - {(i + 1)*100:04d}')
    autoencoder_d.fit(
        x_train, x_train,
        epochs=99,
        verbose=0
    )
    autoencoder_d.fit(
        x_train, x_train,
        epochs=1
    )
fig, AX = plt.subplots(3, 6, figsize=(20, 10))

np.random.seed(1234)
for i in range(6):
    index = np.argwhere(Y_test == i)[:,0]
    index = np.random.choice(index)
    label = Y_test[index]
    
    AX[0][i].imshow(X_test[index][...,0])
    AX[0][i].set_title(f'{objects[label]} ({label})')
    if not i: AX[0][i].set_ylabel('Input', size=16)
    
    encoded = encoder_d.predict(x_test[index].reshape(1, -1))
    
    AX[1][i].imshow(encoded.reshape(8, 8))
    if not i: AX[1][i].set_ylabel('Encoded', size=16)
    
    decoded = decoder_d.predict(encoded)
    
    AX[2][i].imshow(decoded.reshape(N1, N2))
    if not i: AX[2][i].set_ylabel('Decoded', size=16)<jupyter_output><empty_output><jupyter_text>Convolutional Autoencoder---![convolutional autoencoder](sourceimages/autoencoder_convolutional.png "Convolutional Autoencoder")<jupyter_code># Avoid error: "Failed to get convolution algorithm. This is probably because cuDNN failed to initialize,
#               so try looking to see if a warning log message was printed above."
physical_devices = tf.config.experimental.list_physical_devices('GPU')
tf.config.experimental.set_memory_growth(physical_devices[0], True)
# Model
encoder_c = tf.keras.Sequential([
    
    tf.keras.layers.Input((N1, N2, 1)),
    tf.keras.layers.Convolution2D(16, kernel_size=(3,3), padding='same', activation='relu'),
    tf.keras.layers.MaxPooling2D(pool_size=(2,2), padding='same'),
    tf.keras.layers.Convolution2D(8, kernel_size=(3,3), padding='same', activation='relu'),
    tf.keras.layers.MaxPooling2D(pool_size=(2,2), padding='same'),
    tf.keras.layers.Convolution2D(3, kernel_size=(3,3), padding='same', activation='relu')
], name='Convolutional-Encoder')

decoder_c = tf.keras.Sequential([
    tf.keras.layers.Input((7, 7, 3)),
    tf.keras.layers.Convolution2D(8, kernel_size=(3,3), padding='same', activation='relu'),
    tf.keras.layers.UpSampling2D(size=(2, 2)),
    tf.keras.layers.Convolution2D(16, kernel_size=(3,3), padding='same', activation='relu'),
    tf.keras.layers.UpSampling2D(size=(2, 2)),
    tf.keras.layers.Convolution2D(1, kernel_size=(3,3), padding='same', activation='sigmoid')
], name='Convolutional-Decoder')


autoencoder_c = tf.keras.Sequential([
    encoder_c.input,
    *encoder_c.layers,
    decoder_c.input,
    *decoder_c.layers
], name='Convolutional-Autoencoder')

autoencoder_c.summary()
# Compile model
autoencoder_c.compile(
    optimizer='adadelta',
    loss='binary_crossentropy'
)

# Train model
for i in range(5):
    print(f'\nepochs: {i*100:04d} - {(i + 1)*100:04d}')
    autoencoder_c.fit(
        X_train, X_train,
        epochs=99,
        verbose=0
    )
    autoencoder_c.fit(
        X_train, X_train,
        epochs=1
    )
fig, AX = plt.subplots(3, 6, figsize=(20, 10))

np.random.seed(1234)
for i in range(6):
    index = np.argwhere(Y_test == i)[:,0]
    index = np.random.choice(index)
    label = Y_test[index]

    AX[0][i].imshow(X_test[index][...,0])
    AX[0][i].set_title(f'{objects[label]} ({label})')
    if not i: AX[0][i].set_ylabel('Input', size=16)

    encoded = encoder_c.predict(X_test[index][np.newaxis])
    
    AX[1][i].imshow(encoded[0]/encoded.max())
    if not i: AX[1][i].set_ylabel('Encoded', size=16)
    
    decoded = decoder_c.predict(encoded)
    
    AX[2][i].imshow(decoded.reshape(N1, N2))
    if not i: AX[2][i].set_ylabel('Decoded', size=16)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.16944770857814337, 'fraction_numerical': 0.035840188014101056, 'mean_word_length': 4.3731060606060606, 'pattern_counts': {'":': 0, '<': 21, '<?xml version=': 0, '>': 21, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '5', 'dir': 'jupyter-structured-clean-dedup', 'id': '261204', 'license': 'MIT', 'n_tokens_mistral': 3535, 'n_tokens_neox': 3335, 'n_words': 533, 'path': 'Deep-Learning-Models/autoencoder_Keras.ipynb', 'repo_name': 'XGBTrain5/07.ml_diegoinacio', 'size': '8516'}