<jupyter_start><jupyter_text>Set file paths<jupyter_code>import os
import sys
sys.path.append('./modules')

###########################
# Specify if the model being tested is 'mac' or 'spoc'
pooling_type='mac'
# Length of feature vectors
len_features = 512
###########################

# For saving features
features_parent_dir='./featuresTest'
if not os.path.exists(features_parent_dir):
    os.makedirs(features_parent_dir)
features_db_path='featuresDb_' + pooling_type + '_images_real.npy'
features_test_path='featuresTest_' + pooling_type + '_images_real.npy'

# CSV file for train data 
csv_path_db_processed = '../real_images/df_final_all_resized_top14_folder_split_train.csv'
# Images directory for train data
imagesDir_db = '../real_images/train'
# CSV file for test data
csv_path_test_processed = '../real_images/df_final_test_top14.csv'
# Images directory for test data
imagesDir_test = '../real_images'

#For saving results
results_parent_dir = './results'
results_dir='./' + pooling_type + '_base-shape/siamese-margin-01'
if not os.path.exists(results_parent_dir+results_dir):
    os.makedirs(results_parent_dir+results_dir)
results_csv_path='/Test_Siamese_Top_1.csv'

# If loading model weights...
weights_path='../saved_models/mac-real-image-new-data-margin01_conv5_pca_32_1e-05_300_3_1600.h5'

# Column of the csv being considered
column_target = 'std_image'<jupyter_output><empty_output><jupyter_text>Load the model<jupyter_code>from Deep_Retrieval_Siamese_Architecture import deep_retrieval_siamese
from keras import optimizers
from keras.preprocessing import image

batch_size=1
## Load Deep Image Retrieval model
print('Loading Deep Image Retrieval model...')
model = deep_retrieval_siamese((224, 224,3), pooling_type)
## Freeze all layers
for layer in model.layers:
    layer.trainable = False
for layer in model.layers[3].layers:
    layer.trainable = False
## Unfreeze some of the layers
for layer in model.layers:
    layer.trainable = True
for layer in model.layers[3].layers[-5:]:
    layer.trainable = True
print('Loading the weights...')
model.load_weights(weights_path, )
print('Done!')<jupyter_output><empty_output><jupyter_text>Compute train and test features<jupyter_code>import numpy as np
%reload_ext autoreload
%autoreload 2
from modules_generating_results import compute_save_features
np.random.seed(0)
verbose=True
#Train features
features_database = compute_save_features(csv_path_db_processed,imagesDir_db,model,features_parent_dir,features_db_path,column_target,len_features,verbose)
#Test features
features_test = compute_save_features(csv_path_test_processed,imagesDir_test,model,features_parent_dir,features_test_path,column_target,len_features,verbose)<jupyter_output><empty_output><jupyter_text>Find most similar images of test images in Db, save test statistics and display average precision<jupyter_code>#############################
# number of top similar images retrieved 
no_top_similar_images = 5 
#############################

from modules_generating_results import compute_avg_precision
from modules_generating_results import save_similar_images
from modules_generating_results import compute_save_results
import pandas as pd
col_names=["query_image-name","query_image","retrieved_image","average_precision"]    
df_database=pd.read_csv(csv_path_db_processed)
df_test=pd.read_csv(csv_path_test_processed)
compute_save_results(no_top_similar_images,col_names,features_database,features_test,df_database,df_test,column_target,
                    results_parent_dir,results_dir,results_csv_path,save_images=False)

# Compute average precision
df_results=pd.read_csv(results_parent_dir+ results_dir+results_csv_path)
scores=df_results["average_precision"]
print("Top "+str(no_top_similar_images) + " accuracy is ",np.mean(scores))<jupyter_output><empty_output><jupyter_text>Classification table<jupyter_code>df_results=pd.read_csv(results_parent_dir+ results_dir+results_csv_path)
# Load predictions
preds=df_results["retrieved_image"]
# Load labels
df_test_labels=pd.read_csv('../real_images/df_final_test_top14.csv')
labels = df_test_labels[column_target]

# Considering top similar images
preds_top=[]
for i in range(len(preds)):
    if labels[i] in eval(preds[i]):
        preds_top.append(labels[i])
    else:
        preds_top.append(eval(preds[i])[0])
        
from sklearn.metrics import classification_report
print(classification_report(labels,preds_top))<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1156355455568054, 'fraction_numerical': 0.009898762654668167, 'mean_word_length': 7.927710843373494, 'pattern_counts': {'":': 0, '<': 21, '<?xml version=': 0, '>': 21, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '5', 'dir': 'jupyter-structured-clean-dedup', 'id': '610943', 'license': 'MIT', 'n_tokens_mistral': 1491, 'n_tokens_neox': 1429, 'n_words': 268, 'path': 'Test_gen_features_main.ipynb', 'repo_name': 'talalsoms/Deep-image-matching', 'size': '4445'}