<jupyter_start><jupyter_text>Defining the auto-encoder<jupyter_code>autoencoder = tf.keras.models.Sequential([
    tf.keras.layers.Dense(512, activation='elu', input_shape=(784,)),
    tf.keras.layers.Dense(128,  activation='elu'),
    tf.keras.layers.Dense(10,    activation='linear', name="bottleneck"),
    tf.keras.layers.Dense(128,  activation='elu'),
    tf.keras.layers.Dense(512,  activation='elu'),
    tf.keras.layers.Dense(784,  activation='sigmoid')
])
autoencoder.compile(loss='mean_squared_error', optimizer = Adam())<jupyter_output>WARNING:tensorflow:From /home/mohsin/.local/lib/python3.6/site-packages/tensorflow/python/ops/init_ops.py:1251: calling VarianceScaling.__init__ (from tensorflow.python.ops.init_ops) with dtype is deprecated and will be removed in a future version.
Instructions for updating:
Call initializer instance with the dtype argument instead of passing it to the constructor<jupyter_text>Preparing the dataset.<jupyter_code>x_train = training_data.loc[:,1:].values
x_train = x_train / 255.0 # scaling

x_test = test_data.loc[:,1:].values
x_test = x_test / 255.0
trained_model = autoencoder.fit(x_train, x_train, batch_size=32, epochs=32, verbose=1, validation_data=(x_test, x_test))
encoder = tf.keras.Model(autoencoder.input, autoencoder.get_layer('bottleneck').output)<jupyter_output><empty_output><jupyter_text>Generating the gensim index  Preparing data for gensim indices<jupyter_code>training_encodings = encoder.predict(x_train)
test_encodings = encoder.predict(x_test)
indices = []
for item in training_encodings:
    indices.append([(x,y) for (x,y) in enumerate(list(item))])
    
for item in test_encodings:
    indices.append([(x,y) for (x,y) in enumerate(list(item))])<jupyter_output><empty_output><jupyter_text>Generating index<jupyter_code>index = gensim.similarities.Similarity("index",indices,10)<jupyter_output><empty_output><jupyter_text>Testing<jupyter_code>index.num_best = 25
predict_item = 7 #3 #863
index[encoder.predict(x_train[predict_item].reshape(1,-1)).ravel()][:3]
df = pd.concat([training_data,test_data])
df.index = [x for x in range(len(df))]
similar_classes = dict()
similar_classes['Label'] = []
similar_classes['Features'] = []
for i in [x[0] for x in index[encoder.predict(x_train[predict_item].reshape(1,-1)).ravel()]]:
    similar_classes['Label'].append(df.loc[i,0])
    similar_classes['Features'].append(df.loc[i,1:].values.reshape(28,28))
def plot_images(data_dictionary):
    for i in range(25):
        plt.subplot(5, 5, i+1)
        plt.title('Class = '+str(data_dictionary['Label'][i]))
        plt.imshow(data_dictionary['Features'][i])
print (f"Original Class is {df.loc[predict_item,0]}")
plt.figure(figsize=(15,20))
plot_images(similar_classes)
plt.show()<jupyter_output>Original Class is 2
{'fraction_non_alphanumeric': 0.1500539762504498, 'fraction_numerical': 0.02626844188557035, 'mean_word_length': 6.808988764044944, 'pattern_counts': {'":': 0, '<': 19, '<?xml version=': 0, '>': 19, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '5', 'dir': 'jupyter-structured-clean-dedup', 'id': '564264', 'license': 'MIT', 'n_tokens_mistral': 1048, 'n_tokens_neox': 988, 'n_words': 161, 'path': 'fashion-mnist-image-search/Feedforward network on Fashion MNIST.ipynb', 'repo_name': 'mohsin-ashraf/image-search', 'size': '2779'}