<jupyter_start><jupyter_text>Neural Networks for Fashion Image Classification and Visual Search BT5153 Applied Machine Learning Group Project Authors: <NAME> (A0206468Y) | <NAME> (A0206456E) | <NAME> (A0206507L) | <NAME> (A0206510X) | <NAME> (A0206512U)**All authors are graduate students of MSBA program Class of 2020 at NUS Business School. This Notebook searving for: Visual Search  1. Loading Kaggle dataset<jupyter_code>import os
os.environ['KAGGLE_USERNAME'] = "escapingzizi" # username from the json file
os.environ['KAGGLE_KEY'] = "<KEY>" # key from the json file
! kaggle datasets download -d paramaggarwal/fashion-product-images-small
! unzip 'fashion-product-images-small.zip' 
DATASET_PATH = "/content/myntradataset/"
print(os.listdir(DATASET_PATH))
from google.colab import drive
drive.mount('/content/gdrive')<jupyter_output>Go to this URL in a browser: https://accounts.google.com/o/oauth2/auth?client_id=947318989803-6bn6qk8qdgf4n4g3pfee6491hc0brc4i.apps.googleusercontent.com&redirect_uri=urn%3aietf%3awg%3aoauth%3a2.0%3aoob&response_type=code&scope=email%20https%3a%2f%2fwww.googleapis.com%2fauth%2fdocs.test%20https%3a%2f%2fwww.googleapis.com%2fauth%2fdrive%20https%3a%2f%2fwww.googleapis.com%2fauth%2fdrive.photos.readonly%20https%3a%2f%2fwww.googleapis.com%2fauth%2fpeopleapi.readonly

Enter your authorization code:
··········
Mounted at /content/gdrive<jupyter_text>2. Data Preparation before AutoEncodering<jupyter_code>import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import matplotlib.image as mping
import random
from multiprocessing import Pool
from keras_preprocessing.image import ImageDataGenerator
styles = pd.read_csv('styles.csv',error_bad_lines=False)
styles.head()
# get basic information of image
fname = random.choice([x for x in os.listdir('images')
               if os.path.isfile(os.path.join('images', x))])
image = mping.imread('images/' + fname)
plt.imshow(image)
print('image name = {}, image size = {}, image data type = {}'.format(fname, image.shape, image.dtype))
# mapping between images and styles.csv
from multiprocessing import Pool
   
p = Pool(5) # process 5 images simultaneously

image_name = []
for x in os.listdir('images'):
  if os.path.isfile:
    image_path = os.path.join('images', x)
    image_name.append(os.path.basename(image_path))
# convert list to integer
image_number = [i.split('.')[0] for i in image_name]
image_number = [int(i) for i in image_number]
image_number = pd.DataFrame(image_number, columns = ['id'])
# image_number.head(3)
# inner join styles dataframe and image_number in images folder
styles_image = pd.merge(styles, image_number, how='inner', on = 'id')
print('Inner joint styles and images dataframe size = {}'.format(styles_image.shape))
# add in image name column for ImageDataGenerator
styles_image['image_name'] = styles_image.apply(lambda row: str(row['id']) + '.jpg', axis = 1)
styles_image.head()<jupyter_output><empty_output><jupyter_text>2.1 Build DataGenerator<jupyter_code># defind image generator for batch imgae loading and labelling
# flow_from function parameters explanation can refer to: https://keras.io/preprocessing/image/

def data_generator(train_dataframe, sel_y_col, num_samples):
    '''
    this function is used to load image data,label is actually not used in model training
    '''

    batch_size = 64
    image_width = image.shape[0]
    image_height = image.shape[1]

    image_generator = ImageDataGenerator(rescale=1./255)
                                        #  validation_split = 0.1) we can't get the original image_name in original sequence

    training_generator = image_generator.flow_from_dataframe(
        dataframe = train_dataframe,
        directory = 'images',
        x_col = 'image_name', # image .jpg name
        y_col = sel_y_col, 
        target_size = (image_height,image_height), # original image size =(80,60,3) now make image_width and image_height the same!
        batch_size = batch_size, # default value
        shuffle = False,
        class_mode = 'categorical'
    )

    n_classes = len(training_generator.class_indices)
    print('total number of subset samples', num_samples)
    print('total number of class', n_classes)

    return training_generator, n_classes
# articleType data generators
training_generator, n_classes = data_generator(styles_image, 'articleType', articleType_samples)
training_generator.filenames[:10]
styles_image.iloc[:10,10]
batch_size = 64
training_samples = len(training_generator.filenames)
total_steps = training_samples // batch_size
print(training_samples, total_steps)<jupyter_output>44419 694<jupyter_text>AutoEncoder model is self-supervisored learnning. Input is image features Output is itself. Input and Output are the same.<jupyter_code>import math
# getting the X features from generator
def features_matrix(generator, batch_size):
    number_of_examples = len(generator.filenames)
    number_of_generator_calls = math.ceil(number_of_examples / (1.0 * batch_size))
    features_matrix = []
    for i in range(0,int(number_of_generator_calls)):
        features_matrix.extend(np.array(generator[i][0]))
    return features_matrix
train_X =  features_matrix(training_generator, batch_size)
print('training X',len(train_X), train_X[0].shape)
train_X = np.array(train_X)

print(train_X.shape)<jupyter_output>(44419, 60, 60, 3)<jupyter_text>3. AutoEncoder Architecture using Convolution Neural Networks Encoder and Decoder are two reverse representation of imgae features. We need to make sure the autoencoder output shape shall be the same as the input_img shape. As we apply the opposite convolution and pooling, the image width and height size shall be the same. Here we transform the original image size (80,60,3) into (60,60,3)<jupyter_code>from keras.layers import Input, Dense, Conv2D, MaxPooling2D, UpSampling2D
from keras.models import Model
from keras import backend as K

batch_size = 64
image_width = train_X.shape[1]
image_height = train_X.shape[2]
channels = train_X.shape[3]

input_img = Input(shape=(image_height, image_height, channels)) 

x = Conv2D(16, (3, 3), activation='relu', padding='same')(input_img)
x = MaxPooling2D((2, 2), padding='same')(x)
x = Conv2D(8, (3, 3), activation='relu', padding='same')(x)
x = MaxPooling2D((2, 2), padding='same')(x)
x = Conv2D(8, (3, 3), activation='relu', padding='same')(x)
conv_encoded = MaxPooling2D((2, 2), padding='same')(x)

# at this point the representation is (8, 8, 8) i.e. 512-dimensiona

x = Conv2D(8, (3, 3), activation='relu', padding='same')(conv_encoded)
x = UpSampling2D((2, 2))(x)
x = Conv2D(8, (3, 3), activation='relu', padding='same')(x)
x = UpSampling2D((2, 2))(x)
x = Conv2D(16, (3, 3), activation='relu')(x)
x = UpSampling2D((2, 2))(x)
conv_decoded = Conv2D(channels, (3, 3), activation='sigmoid', padding='same')(x)

conv_autoencoder = Model(input_img, conv_decoded) # set autoencoder
conv_encoder = Model(input_img, conv_encoded)  # set encoder

from keras import optimizers
adam = optimizers.Adam(lr=1e-5, beta_1=0.9, beta_2=0.999, epsilon=1e-08, decay=1e-6)
#rmsprop = optimizers.RMSprop(lr=1e-3)

conv_autoencoder.compile(optimizer=adam, loss="binary_crossentropy") # not categorical_crossentropy !!!!
# conv_encoder.compile(optimizer=adam, loss="binary_crossentropy")
conv_autoencoder.summary()<jupyter_output>Model: "model_7"
_________________________________________________________________
Layer (type)                 Output Shape              Param #   
=================================================================
input_4 (InputLayer)         (None, 60, 60, 3)         0         
_________________________________________________________________
conv2d_22 (Conv2D)           (None, 60, 60, 16)        448       
_________________________________________________________________
max_pooling2d_10 (MaxPooling (None, 30, 30, 16)        0         
_________________________________________________________________
conv2d_23 (Conv2D)           (None, 30, 30, 8)         1160      
_________________________________________________________________
max_pooling2d_11 (MaxPooling (None, 15, 15, 8)         0         
_________________________________________________________________
conv2d_24 (Conv2D)           (None, 15, 15, 8)         584       
___________________________________________________________[...]<jupyter_text>4. Model training and Generating Embedding Features  4.1 Training Convolutional Autoencoder<jupyter_code>import time
from keras.callbacks import EarlyStopping, ModelCheckpoint, ReduceLROnPlateau

start = time.time()
conv_autoencoder_history = conv_autoencoder.fit(train_X, train_X,
                                                epochs=50,
                                                batch_size=128,
                                                shuffle=True,
                                                validation_split=0.2)
elapsed = time.time() - start
print('Computation time = ' + str(round(elapsed,2)) + 's')
# plot train and valid loss history

def plot_loss(history):
    # plot train and valid loss history
    loss = history.history['loss']
    val_loss = history.history['val_loss']
    
    epoch_number = range(len(loss))
    plt.plot(epoch_number, loss, 'r', label='train loss')
    plt.plot(epoch_number, val_loss, 'b', label='val loss')
    plt.title('Train and Validation Loss')
    plt.legend()

    plt.show()
# plot train and valid loss history 
conv_autoencoder_loss_plot = plot_loss(conv_autoencoder_history)
# save trained model
path = '/content/gdrive/My Drive/Colab Notebooks/BT5153_AML_Group_Project/'
model_name = 'conv_autoencoder.h5'

conv_autoencoder.save(path+model_name)

# Recreate the exact same model, including its weights and the optimizer
# conv_autoencoder = keras.models.load_model('conv_autoencoder.h5')

# Show the model architecture
# conv_autoencoder.summary()<jupyter_output><empty_output><jupyter_text>4.2 Convolutional Autoencoder Embedding Features<jupyter_code># Generate embeddings using autoencoder model
encoded_imgs_train = conv_encoder.predict(train_X)
print(encoded_imgs_train.shape)
# flattern embedding features
output_shape_model = tuple([int(x) for x in encoded_imgs_train.shape[1:]])

encoded_imgs_train_flatten = encoded_imgs_train.reshape(-1, np.prod(output_shape_model))
print(encoded_imgs_train_flatten.shape)
# train dataset embedding feature dataframe
encoded_imgs_train_flatten_df = pd.DataFrame(encoded_imgs_train_flatten)
encoded_imgs_train_flatten_df['image_name'] = training_generator.filenames
cols = encoded_imgs_train_flatten_df.columns.tolist()
cols = cols[-1:] + cols[:-1]
encoded_imgs_train_flatten_df = encoded_imgs_train_flatten_df[cols]
encoded_imgs_train_flatten_df.head()<jupyter_output><empty_output><jupyter_text>5. Image Search using Cosine Similarity 5.1 Image Seach using Internal Images<jupyter_code>from sklearn.metrics.pairwise import pairwise_distances

def image_search(input_img_indx, top_n):
    input_img = encoded_imgs_train_flatten_df.iloc[input_img_indx,1:] # column 0 is image_name
    input_img = np.array(input_img).reshape(1, -1)
    cosine_sim = 1 - pairwise_distances(input_img, encoded_imgs_train_flatten_df.iloc[:,1:], metric='cosine')
    output_img_idx = [i[0] for i in sorted(enumerate(cosine_sim[0]), key=lambda x:x[1], reverse=True)][1:top_n+1]
    output_img_scores = [i[1] for i in sorted(enumerate(cosine_sim[0]), key=lambda x:x[1], reverse=True)][1:top_n+1]
    print('total number of cosine similarity scores',cosine_sim.shape)
    return encoded_imgs_train_flatten_df.iloc[output_img_idx,0], output_img_scores, output_img_idx
input_img_indx = 4026
top_n = 5
output_images, search_scores, output_img_idx = image_search(input_img_indx, top_n)
print(output_images)
print(search_scores)
print(output_img_idx)
print('Target image and the top5 recommended images')
fname = encoded_imgs_train_flatten_df.iloc[input_img_indx,0]
input_image = mping.imread('images/' + fname)
plt.imshow(input_image)

fig = plt.figure(figsize=(15,14))
for i in range(top_n-4, top_n+1):
    image = mping.imread('images/' + output_images.iloc[i-1])
    fig.add_subplot(1, top_n, i)
    plt.imshow(image)
plt.show()<jupyter_output>Target image and the top5 recommended images<jupyter_text>5.2 Evaluation of Image Search Model<jupyter_code># list top5 and target image features
output_img_idx.append(input_img_indx)
styles_image.iloc[output_img_idx,:]<jupyter_output><empty_output><jupyter_text>5.3 Image search using external images<jupyter_code>import cv2
import matplotlib.pyplot as plt
import matplotlib.image as mpimg

# original image size
image_height = image.shape[1]
path = '/content/gdrive/My Drive/Colab Notebooks/BT5153_AML_Group_Project/Images for visual search/'
my_image = "tshirt1.jpg" # blueshirt1.jpg, tshirt1.jpg, earing1.jpeg
fname = path + my_image
my_image = mpimg.imread(fname)
my_image = my_image * 1./255

resize_image = cv2.resize(my_image,(image_height, image_height))
plt.imshow(resize_image, interpolation='nearest')

resize_image = resize_image.reshape(1,resize_image.shape[0],resize_image.shape[1],resize_image.shape[2])
print(resize_image.shape)
# using trained autoencoder to generate encoding features
conv_encoded_image = conv_encoder.predict(resize_image)
print(conv_encoded_image.shape)
def my_image_search(my_image, top_n):
    input_img = my_image
    input_img = np.array(input_img).reshape(1, -1)
    cosine_sim = 1 - pairwise_distances(input_img, encoded_imgs_train_flatten_df.iloc[:,1:], metric='cosine')
    output_img_idx = [i[0] for i in sorted(enumerate(cosine_sim[0]), key=lambda x:x[1], reverse=True)][:top_n]
    output_img_scores = [i[1] for i in sorted(enumerate(cosine_sim[0]), key=lambda x:x[1], reverse=True)][:top_n]
    print('total number of cosine similarity scores',cosine_sim.shape)
    return encoded_imgs_train_flatten_df.iloc[output_img_idx,0], output_img_scores
my_output_images, my_search_scores = my_image_search(conv_encoded_image, 5)
print(my_output_images)
print(my_search_scores)
print('Target image and the top5 recommended images')
input_image = mping.imread(fname)
plt.imshow(input_image)

fig = plt.figure(figsize=(15,14))
for i in range(top_n-4, top_n+1):
    image = mping.imread('images/' + my_output_images.iloc[i-1])
    fig.add_subplot(1, top_n, i)
    plt.imshow(image)
plt.show()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1119022687609075, 'fraction_numerical': 0.029040139616055845, 'mean_word_length': 5.421335723890632, 'pattern_counts': {'":': 0, '<': 42, '<?xml version=': 0, '>': 42, 'https://': 2, 'lorem ipsum': 0, 'www.': 4, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 11, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '10', 'dir': 'jupyter-structured-clean-dedup', 'id': '392675', 'license': 'FSFAP', 'n_tokens_mistral': 4943, 'n_tokens_neox': 4694, 'n_words': 1054, 'path': 'BT5153_AML_ImageSearch_CNNAutoEncoder.ipynb', 'repo_name': 'swapnasamirshukla/BT5153-Term-Project', 'size': '14388'}