<jupyter_start><jupyter_text>Autoencoder vs PCA for Dimensionality Reduction<jupyter_code>%matplotlib inline
import keras
import matplotlib.pyplot as plt
import numpy as np
import umap

from keras.datasets import mnist
from keras.layers import Dense
from keras.models import Model
from keras.models import Sequential
from keras.optimizers import Adam
from numpy.linalg import svd
from sklearn.manifold import TSNE
np.random.seed(1535383000)<jupyter_output><empty_output><jupyter_text>Dataset MNIST<jupyter_code>(X_train, y_train), (X_test, _) = mnist.load_data()

print(X_train.shape, X_test.shape)

plt.figure(figsize=(9,1))
for i in range(10):
    ax = plt.subplot(1, 10, i+1)
    plt.imshow(X_train[i,:], interpolation='nearest')
    plt.gray()
    plt.axis('off')
    
plt.tight_layout()

X_train = X_train.reshape(60000, 784) / 255
X_test = X_test.reshape(10000, 784) / 255<jupyter_output>(60000, 28, 28) (10000, 28, 28)<jupyter_text>PCA<jupyter_code>%%time

n_pc = 2

mu = X_train.mean(axis=0)
U, s, V = svd(X_train - mu, full_matrices=False)
Z_pca = np.dot(X_train - mu, V.T)
R_pca = np.dot(Z_pca[:, :n_pc], V[:n_pc, :]) + mu

n_rows, n_cols = R_pca.shape

#  MSE
err = np.sum((X_train - R_pca)**2) * (n_rows * n_cols)**-1

print(f'loss: {err:.4}')<jupyter_output>loss: 0.05595
CPU times: user 28.4 s, sys: 2.73 s, total: 31.2 s
Wall time: 10.7 s<jupyter_text>Autoencoder<jupyter_code>model = Sequential()
model.add(Dense(512, activation='elu', input_shape=(784, )))
model.add(Dense(128, activation='elu'))
model.add(Dense(2, activation='linear', name='bottleneck'))
model.add(Dense(128, activation='elu'))
model.add(Dense(512, activation='elu'))
model.add(Dense(784, activation='sigmoid'))
model.compile(loss='mean_squared_error', optimizer=Adam())

model.fit(X_train, X_train, batch_size=128, epochs=5, verbose=1, validation_data=(X_test, X_test))

encoder = Model(model.input, model.get_layer('bottleneck').output)

Z_ae = encoder.predict(X_train)
R_ae = model.predict(X_train)<jupyter_output>Train on 60000 samples, validate on 10000 samples
Epoch 1/5
60000/60000 [==============================] - 8s 138us/step - loss: 0.0568 - val_loss: 0.0494
Epoch 2/5
60000/60000 [==============================] - 8s 132us/step - loss: 0.0467 - val_loss: 0.0449
Epoch 3/5
60000/60000 [==============================] - 8s 130us/step - loss: 0.0439 - val_loss: 0.0432
Epoch 4/5
60000/60000 [==============================] - 8s 132us/step - loss: 0.0425 - val_loss: 0.0421
Epoch 5/5
60000/60000 [==============================] - 8s 128us/step - loss: 0.0413 - val_loss: 0.0410<jupyter_text>UMAP<jupyter_code>%%time

Z_umap = umap.UMAP(n_components=2, n_neighbors=5, min_dist=0.3, metric='correlation').fit_transform(X_train)<jupyter_output>CPU times: user 1min 26s, sys: 6.38 s, total: 1min 33s
Wall time: 1min 16s<jupyter_text>Embeddings<jupyter_code>plt.figure(figsize=(18,6))

plt.subplot(131)
plt.title('PCA')
plt.scatter(Z_pca[:5000,0], Z_pca[:5000,1], c=y_train[:5000], s=8, cmap='tab10')
plt.axis('off')

plt.subplot(132)
plt.title('Autoencoder')
plt.scatter(Z_ae[:5000,0], Z_ae[:5000,1], c=y_train[:5000], s=8, cmap='tab10')
plt.axis('off')

plt.subplot(133)
plt.title('UMAP')
plt.scatter(Z_umap[:5000,0], Z_umap[:5000,1], c=y_train[:5000], s=8, cmap='tab10')
plt.axis('off')

plt.tight_layout()<jupyter_output><empty_output><jupyter_text>Reconstructing from Embeddings<jupyter_code>R_models = (X_train, R_pca, R_ae)

plt.figure(figsize=(9, 3))
for i in range(10):
    for j in range(3):
        ax = plt.subplot(3, 10, 10*j+1+i)
        plt.imshow(R_models[j][i,:].reshape(28,28), interpolation='nearest')
        plt.gray()
        plt.axis('off')
        
plt.tight_layout()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.19860064585575887, 'fraction_numerical': 0.09122712594187299, 'mean_word_length': 6.013207547169811, 'pattern_counts': {'":': 0, '<': 25, '<?xml version=': 0, '>': 25, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '7', 'dir': 'jupyter-structured-clean-dedup', 'id': '268059', 'license': 'MIT', 'n_tokens_mistral': 1793, 'n_tokens_neox': 1526, 'n_words': 278, 'path': 'autoencoder-vs-pca-for-dimensionality-reduction/autoencoder-vs-pca-for-dimensionality-reduction.ipynb', 'repo_name': 'sebastianmarkow/blog-appendix', 'size': '3716'}