<jupyter_start><jupyter_text>This notebook covers deep embedded clustering (DEC) on face images. The paper is - https://arxiv.org/pdf/1511.06335.pdfThe source code is hugely influenced by https://github.com/XifengGuo/DEC-keras/blob/master/DEC.py<jupyter_code>from time import time
import numpy as np
import keras.backend as K
from keras.engine.topology import Layer, InputSpec
from keras.layers import Input, Dense, Flatten, Conv2D, MaxPooling2D, UpSampling2D, Lambda, RepeatVector, Reshape, Conv2DTranspose
from keras.models import Model
from keras.optimizers import SGD
from keras import callbacks
from keras.initializers import VarianceScaling
from sklearn.cluster import KMeans
from keras.layers import Input, Dense, Flatten, Conv2D, MaxPooling2D, UpSampling2D, Lambda, RepeatVector, Reshape, Conv2DTranspose
from keras.models import Model
from keras import backend as K
from sklearn.datasets import fetch_olivetti_faces
import matplotlib.pyplot as plt
from sklearn.model_selection import train_test_split
from keras.metrics import binary_crossentropy, kullback_leibler_divergence
from keras import regularizers
from keras.callbacks import TensorBoard, EarlyStopping
from keras.optimizers import SGD, Adam
data = fetch_olivetti_faces()
len(data.images)
images = [np.reshape(i, (64, 64, 1)) for i in data.images]
images = np.array(images)
print (images.shape)
input_img = Input(shape=(64, 64, 1))  # adapt this if using `channels_first` image data format

x = Conv2D(64, (3, 3), activation='relu', padding='same')(input_img)
#x = MaxPooling2D((2, 2), padding='same')(x)
x = Conv2D(64, (3, 3), activation='relu', padding='same')(x)
#x = MaxPooling2D((2, 2), padding='same')(x)
x = Conv2D(64, (3, 3), activation='relu', padding='same')(x)
encoded = MaxPooling2D((2, 2), padding='same')(x)

# at this point the representation is (4, 4, 8) i.e. 128-dimensional

x = Conv2D(64, (3, 3), activation='relu', padding='same')(encoded)
x = UpSampling2D((2, 2))(x)
x = Conv2D(64, (3, 3), activation='relu', padding='same')(x)
#x = UpSampling2D((2, 2))(x)
x = Conv2D(64, (3, 3), activation='relu',padding='same')(x)
#x = UpSampling2D((2, 2))(x)
decoded = Conv2D(1, (3, 3), activation='relu', padding='same')(x)

autoencoder = Model(input_img, decoded)
autoencoder.compile(optimizer=Adam(), loss='binary_crossentropy')
encoder = Model(input_img, Flatten()(encoded))
autoencoder.summary()
encoder.summary()
class ClusteringLayer(Layer):
    def __init__(self, n_clusters, weights=None, alpha=1.0, **kwargs):
        super(ClusteringLayer, self).__init__(**kwargs)
        self.n_clusters = n_clusters
        self.alpha = alpha
        self.initial_weights = weights
        self.input_spec = InputSpec(ndim=2)
    
    def build(self, input_shape):
        assert len(input_shape) == 2
        input_dim = input_shape[1]
        self.input_spec = InputSpec(dtype=K.floatx(), shape=(None, input_dim))
        self.clusters = self.add_weight((self.n_clusters, input_dim), initializer='glorot_uniform', name='clusters')
        if self.initial_weights is not None:
            self.set_weights(self.initial_weights)
            del self.initial_weights
        self.built = True
        
    def call(self, inputs, **kwargs):
        """ student t-distribution, as same as used in t-SNE algorithm.
                 q_ij = 1/(1+dist(x_i, u_j)^2), then normalize it.
        Arguments:
            inputs: the variable containing data, shape=(n_samples, n_features)
        Return:
            q: student's t-distribution, or soft labels for each sample. shape=(n_samples, n_clusters)
        """
        q = 1.0 / (1.0 + (K.sum(K.square(K.expand_dims(inputs, axis=1) - self.clusters), axis=2) / self.alpha))
        q **= (self.alpha + 1.0) / 2.0
        q = K.transpose(K.transpose(q) / K.sum(q, axis=1))
        return q

    def compute_output_shape(self, input_shape):
        assert input_shape and len(input_shape) == 2
        return input_shape[0], self.n_clusters

    def get_config(self):
        config = {'n_clusters': self.n_clusters}
        base_config = super(ClusteringLayer, self).get_config()
        return dict(list(base_config.items()) + list(config.items()))

n_clusters = 40
alpha = 1

clustering_layer = ClusteringLayer(n_clusters, alpha=alpha, name='clustering')(encoder.output)
DEC = Model(inputs=encoder.input, outputs=clustering_layer) 
DEC.summary()
def loss(y_true, y_pred):
    weight = y_pred ** 2 / K.sum(y_pred, axis=0)
    y_true = K.transpose(K.transpose(weight) / K.sum(weight, axis=1))
    return kullback_leibler_divergence(y_true, y_pred)
DEC.compile(optimizer=Adam(),loss=loss)
X_train, X_test, y_train, y_test = train_test_split(images, data.target, test_size=.2, random_state = 123, shuffle=True)
autoencoder.fit(X_train, X_train, epochs=20, validation_split=.2,callbacks=[EarlyStopping(monitor="val_loss",patience=5,mode='min')])
km = KMeans(n_clusters)
km.fit(encoder.predict(X_train))
DEC.get_layer('clustering').set_weights([km.cluster_centers_])
DEC.fit(X_train, y_train, epochs=20, validation_split=.2,callbacks=[EarlyStopping(monitor="val_loss",patience=5,mode='min')])
testpred = DEC.predict(X_test)
testpred = testpred.argmax(axis=1)
testpred
y_test
q = np.random.random((10,40))
q = K.constant(q)
weight = q ** 2 / K.sum(q, axis=0)
p = K.transpose(K.transpose(weight) / K.sum(weight, axis=1))
print (K.eval(kullback_leibler_divergence(p, q)))<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1369179600886918, 'fraction_numerical': 0.024390243902439025, 'mean_word_length': 4.770788912579957, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '288666', 'license': 'MIT', 'n_tokens_mistral': 1982, 'n_tokens_neox': 1883, 'n_words': 431, 'path': 'Clustering/DEC.ipynb', 'repo_name': 'victor7246/Practice', 'size': '5412'}