<jupyter_start><jupyter_text>4. Predict-Image-Similar-FCNN-Binary[batch]For landmark-recognition-2019 algorithm validation  Run name<jupyter_code>import time

project_name = 'Google-LandMark-Rec2019'
model_name = 'Google-LandMark-Rec2019_3-Image-Similar-FCNN-Binary_20190511-093300_8406'
step_name = '4-Predict[batch]_%s' % model_name
time_str = time.strftime("%Y%m%d-%H%M%S", time.localtime())
run_name = step_name + '_' + time_str
print('run_name: ' + run_name)
t0 = time.time()<jupyter_output>run_name: 4-Predict[batch]_Google-LandMark-Rec2019_3-Image-Similar-FCNN-Binary_20190511-093300_8406_20190511-150948<jupyter_text>Important params<jupyter_code>import multiprocessing

cpu_amount = multiprocessing.cpu_count()
print('cpu_amount: ', cpu_amount)

small_debug_rows = 10
big_debug_rows = 20
predict_batch_size = 10
print('small_debug_rows:\t', small_debug_rows)
print('big_debug_rows:\t', big_debug_rows)
print('predict_batch_size:\t', predict_batch_size)<jupyter_output>cpu_amount:  4
small_debug_rows:	 10
big_debug_rows:	 20
predict_batch_size:	 10<jupyter_text>Import PKGs<jupyter_code>import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import matplotlib.image as mpimg
%matplotlib inline
from IPython.display import display

import os
import sys
import gc
import math
import shutil
import zipfile
import pickle
import h5py

from tqdm import tqdm

from sklearn.model_selection import train_test_split
from sklearn.metrics import confusion_matrix, accuracy_score
import keras
from keras.utils import Sequence
from keras.layers import *
from keras.models import *
from keras.applications import *
from keras.optimizers import *
from keras.regularizers import *
from keras.preprocessing.image import *
from keras.applications.inception_v3 import preprocess_input
import tensorflow as tf
import keras.backend.tensorflow_backend as KTF

config = tf.ConfigProto()
config.gpu_options.per_process_gpu_memory_fraction = 0.5
session = tf.Session(config=config)

KTF.set_session(session )<jupyter_output><empty_output><jupyter_text>Project folders<jupyter_code>cwd = os.getcwd()
feature_folder = os.path.join(cwd, 'feature')
input_folder = os.path.join(cwd, 'input')
output_folder = os.path.join(cwd, 'output')
model_folder = os.path.join(cwd, 'model')

org_train_folder = os.path.join(input_folder, 'org_train')
org_test_folder = os.path.join(input_folder, 'org_test')
train_folder = os.path.join(input_folder, 'data_train')
val_folder = os.path.join(input_folder, 'data_val')
test_folder = os.path.join(input_folder, 'data_test')
test_sub_folder = os.path.join(test_folder, 'test')

vgg16_feature_file = os.path.join(feature_folder, 'feature_wrapper_171023.h5')
train_csv_file = os.path.join(input_folder, 'train.csv')
test_csv_file = os.path.join(input_folder, 'test.csv')
sample_submission_folder = os.path.join(input_folder, 'sample_submission.csv')
print(vgg16_feature_file)
print(train_csv_file)
print(test_csv_file)
print(sample_submission_folder)<jupyter_output>/data/landmark-recognition-2019/feature/feature_wrapper_171023.h5
/data/landmark-recognition-2019/input/train.csv
/data/landmark-recognition-2019/input/test.csv
/data/landmark-recognition-2019/input/sample_submission.csv<jupyter_text>Functions<jupyter_code>def pickle_dump(data, file):
    with open(file, 'wb') as f:
        pickle.dump(data, f)

def pickle_load(file):
    with open(file, 'rb') as f:
        data = pickle.load(f)
    return data

a = list(range(10))
print(a)
demo_file = os.path.join(os.getcwd(), 'temp', 'pickle_demo.pkl')
print(demo_file)
pickle_dump(a, demo_file)
new_a = pickle_load(demo_file)
print(new_a)
%%time

def get_whole_classes_label(class_indices_data, item_classes_data):
    indices2class = {}
    for i, key in enumerate(class_indices_data.keys()):
#         print(key, class_indices_data[key])
        indices2class[class_indices_data[key]] = key
#         if i >= 5:
#             break
#     for i, key in enumerate(class_indices_data.keys()):
#         print(key, class_indices_data[key], '-->', class_indices_data[key], indices2class[class_indices_data[key]])
#         if i >= 10:
#             break
    print(item_classes_data.shape)
    whole_classes_data = np.zeros(item_classes_data.shape)
    for i, class_index in enumerate(item_classes_data):
        whole_classes_data[i] = indices2class[class_index]
#         print(i, class_index, '-->', indices2class[class_index])
#         if i >= 10:
#             break
    return whole_classes_data
# get_whole_classes_label(class_indices_train, item_classes_train)<jupyter_output>CPU times: user 0 ns, sys: 0 ns, total: 0 ns
Wall time: 14.1 µs<jupyter_text>Load feature<jupyter_code>!ls ./feature -hl
def load_feature(data_set_name, model_name, date_str, feature_amount):
#     data_set_name = 'train'
#     model_name = 'VGG19'
#     date_str = '171023'
    x_data_arr = []
    classes_data_arr = []
    index_data_arr = []
    for i in range(feature_amount):
        folder_name = 'data_%s_%02d' % (data_set_name, i)
        class_indices_file = os.path.join(cwd, 'feature', 'feature_{0}_{1}_{2}_class_indices.pkl'.format(model_name, folder_name, date_str))
        h5py_file_name = os.path.join(cwd, 'feature', 'feature_{0}_{1}_{2}.h5'.format(model_name, folder_name, date_str))
        if not os.path.exists(class_indices_file):
            print('File not exists', class_indices_file)
            continue
        if not os.path.exists(h5py_file_name):
            print('File not exists', h5py_file_name)
            continue
        print(class_indices_file)
        print(h5py_file_name)
        class_indices_data = pickle_load(class_indices_file)
        print(len(class_indices_data))

        with h5py.File(h5py_file_name, 'r') as h:
            item_x_data = np.array(h['x_data_%s_%02d' % (data_set_name, i)])
            item_classes_data = np.array(h['classes_data_%s_%02d' % (data_set_name, i)])
            item_index_data = np.array(h['index_data_%s_%02d' % (data_set_name, i)])
        print(item_x_data.shape)
        x_data_arr.append(item_x_data)
        item_y_data = get_whole_classes_label(class_indices_data, item_classes_data)
        print(item_y_data.shape, item_y_data[:10])
        classes_data_arr.append(item_y_data)
        index_data_arr.append(item_index_data)
    x_data = np.concatenate(x_data_arr, axis=0)
    y_data = np.concatenate(classes_data_arr, axis=0)
    idx_data = np.concatenate(index_data_arr, axis=0)
    print('*' * 60)
    print(x_data.shape)
    print(y_data.shape)
    print(idx_data.shape)
    return x_data, y_data, idx_data

x_train, y_train, idx_train = load_feature('train', 'VGG19', '171023', 20)
x_val, y_val, idx_val = load_feature('val', 'VGG19', '171023', 4)<jupyter_output>/data/landmark-recognition-2019/feature/feature_VGG19_data_val_00_171023_class_indices.pkl
/data/landmark-recognition-2019/feature/feature_VGG19_data_val_00_171023.h5
34160
(50000, 512)
(50000,)
(50000,) [ 1. 12. 20. 27. 27. 27. 27. 27. 27. 32.]
/data/landmark-recognition-2019/feature/feature_VGG19_data_val_01_171023_class_indices.pkl
/data/landmark-recognition-2019/feature/feature_VGG19_data_val_01_171023.h5
34196
(50000, 512)
(50000,)
(50000,) [ 9. 11. 24. 27. 27. 27. 27. 27. 27. 27.]
/data/landmark-recognition-2019/feature/feature_VGG19_data_val_02_171023_class_indices.pkl
/data/landmark-recognition-2019/feature/feature_VGG19_data_val_02_171023.h5
34264
(50000, 512)
(50000,)
(50000,) [ 1. 10. 11. 12. 27. 27. 27. 27. 27. 27.]
/data/landmark-recognition-2019/feature/feature_VGG19_data_val_03_171023_class_indices.pkl
/data/landmark-recognition-2019/feature/feature_VGG19_data_val_03_171023.h5
34155
(50000, 512)
(50000,)
(50000,) [ 1.  2. 10. 11. 13. 24. 27. 27. 27. 27.]
****************[...]<jupyter_text>Load model<jupyter_code>model_file = os.path.join(model_folder, '%s.h5' % model_name)
print(model_file)
model = load_model(model_file)
model.summary()<jupyter_output>__________________________________________________________________________________________________
Layer (type)                    Output Shape         Param #     Connected to                     
==================================================================================================
main_input (InputLayer)         (None, 512)          0                                            
__________________________________________________________________________________________________
library_input (InputLayer)      (None, 512)          0                                            
__________________________________________________________________________________________________
concatenate_1 (Concatenate)     (None, 1024)         0           main_input[0][0]                 
                                                                 library_input[0][0]              
__________________________________________________________________________________________________
dense_1 (D[...]<jupyter_text>Predict  single item<jupyter_code>%%time
idx = 0
item_main_x = np.array([x_train[idx]]*x_train.shape[0])
item_x_train = {
    'main_input': item_main_x,
    'library_input': x_train
}
y_proba = model.predict(item_x_train, batch_size=1024)
print(int(y_train[idx]), np.argmax(y_proba), '-->', int(y_train[np.argmax(y_proba)]), [int(y_train[item[0]]) for item in np.argsort(y_proba, axis=0)[-10:]])<jupyter_output>0 0 --> 0 [175067, 174106, 106318, 113460, 73879, 137248, 194505, 199923, 110488, 0]
CPU times: user 16 s, sys: 3.78 s, total: 19.8 s
Wall time: 49.7 s<jupyter_text>batch predict<jupyter_code>%%time
batch_libary_x = np.tile(x_train, (3, 1))
print(batch_libary_x.shape)
def get_batch_pred(batch_x_data, batch_libary_x, topn, batch_size=1024):
    print(batch_x_data.shape)
    print(libary_x.shape)
# def get_single_pred(idx, main_x, libary_x, topn, batch_size=1024):
    batch_main_x_arr = []
    for item_x_data in batch_x_data:
        expand_np = np.expand_dims(item_x_data, axis=0)
        tile_np = np.tile(expand_np, (libary_x.shape[0], 1))
        batch_main_x_arr.append(tile_np)
    batch_main_x = np.concatenate(batch_main_x_arr, axis=0)
    print(batch_main_x.shape)

    item_x = {
        'main_input': batch_main_x,
        'library_input': batch_libary_x
    }
#     y_proba = model.predict(item_x, batch_size=batch_size)
#     top1_pred = int(y_train[np.argmax(y_proba)])
#     topn_pred_arr = [int(y_train[item[0]]) for item in np.argsort(y_proba, axis=0)[-topn:]]
#     weighted_topn_pred_arr, class_score_arr = get_weight_topn(y_proba, topn, y_train)
#     weighted_top1_pred = weighted_topn_pred_arr[-1]
#     return top1_pred, topn_pred_arr, weighted_top1_pred, weighted_topn_pred_arr, class_score_arr

idx = 0
topn = 10
# train
print(get_batch_pred(x_train[:3], batch_libary_x, topn, batch_size=1024))
# val
# print(int(y_val[idx]), get_single_pred(idx, x_val, x_train, topn, batch_size=1024))
np.expand_dims(x_train[0], axis=0).shape
a = np.repeat(x_train[:2], 10000, axis=0)
print(a.shape)
np.tile?
orig = np.array([[1, 2], [3, 4], [5, 6]])
print(orig.shape)
print(orig)
print('*' * 80)
a = np.tile(orig, (3, 1))
print(a, a.shape)
x_train[:1].shape
assert 0<jupyter_output><empty_output><jupyter_text>topn<jupyter_code>topn = 10
amount = small_debug_rows
top1_count = 0
topn_count = 0
for idx in range(amount):
    item_main_x = np.array([x_train[idx]]*x_train.shape[0])
    item_x_train = {
        'main_input': item_main_x,
        'library_input': x_train
    }
    y_proba = model.predict(item_x_train, batch_size=1024)
    item_y_train = int(y_train[idx])
    top1_pred = int(y_train[np.argmax(y_proba)])
    topn_pred_arr = [int(y_train[item[0]]) for item in np.argsort(y_proba, axis=0)[-topn:]]
    if item_y_train == top1_pred:
        top1_count += 1
    if item_y_train in topn_pred_arr:
        topn_count += 1
    print(item_y_train, '-->', top1_pred, topn_pred_arr)

print('*' * 80)
print(top1_count, '%.2f' % (top1_count/amount))
print(topn_count, '%.2f' % (topn_count/amount))<jupyter_output><empty_output><jupyter_text>weighted topn<jupyter_code>def get_weight_topn(y_proba, topn, y_data):
    y_proba_argsorted = [(y_data[item[0]], y_proba[item[0]][0]) for item in np.argsort(y_proba, axis=0)[-topn:]]
    class_score_dict = {}
    for item_class, item_proba in y_proba_argsorted:
        if item_class in class_score_dict:
            class_score_dict[item_class] += item_proba
        else:
            class_score_dict[item_class] = item_proba
    class_score_arr = list(class_score_dict.items())
    class_score_arr = sorted(class_score_arr, key=lambda x: x[1])
    topn_pred_arr = [int(item[0]) for item in class_score_arr]
    class_score_arr = [round(item[1], 3) for item in class_score_arr]
    return topn_pred_arr, class_score_arr

y_proba = np.array([[0.1], [0.2], [0.3], [0.2], [0.2], [0.2]])
topn = 6
y_data = np.array([0, 0, 1, 1, 2, 2])
print(y_proba, type(y_proba))
print(topn)
print(y_data, type(y_data))
get_weight_topn(y_proba, topn, y_data)
topn = 10
amount = small_debug_rows
top1_count = 0
topn_1_count = 0
topn_count = 0
for idx in range(amount):
    item_main_x = np.array([x_train[idx]]*x_train.shape[0])
    item_x_train = {
        'main_input': item_main_x,
        'library_input': x_train
    }
    y_proba = model.predict(item_x_train, batch_size=1024)
    item_y_train = int(y_train[idx])
    top1_pred = int(y_train[np.argmax(y_proba)])
#     topn_pred_arr = [y_train[item[0]] for item in np.argsort(y_proba, axis=0)[-topn:]]
    topn_pred_arr, class_score_arr = get_weight_topn(y_proba, topn, y_train)
    topn_1_pred = topn_pred_arr[-1]
    if item_y_train == top1_pred:
        top1_count += 1
    if item_y_train == topn_1_pred:
        topn_1_count += 1
    if item_y_train in topn_pred_arr:
        topn_count += 1
    print(item_y_train, '-->', top1_pred, '\t', topn_1_pred, topn_pred_arr, class_score_arr)

print('*' * 80)
print(top1_count, '%.2f' % (top1_count/amount))
print(topn_1_count, '%.2f' % (topn_1_count/amount))
print(topn_count, '%.2f' % (topn_count/amount))<jupyter_output><empty_output><jupyter_text>encapsolution<jupyter_code>%%time
def get_single_pred(idx, main_x, libary_x, topn, batch_size=1024):
    time_arr = []
    time_arr.append(time.strftime("%Y%m%d-%H%M%S", time.localtime()))
#     item_main_x = np.array([main_x[idx]]*libary_x.shape[0])
    expand_np = np.expand_dims(main_x[idx], axis=0)
    item_main_x = np.tile(expand_np, (libary_x.shape[0], 1))
    time_arr.append(time.strftime("%Y%m%d-%H%M%S", time.localtime()))
    item_x = {
        'main_input': item_main_x,
        'library_input': libary_x
    }
    time_arr.append(time.strftime("%Y%m%d-%H%M%S", time.localtime()))
    y_proba = model.predict(item_x, batch_size=batch_size)
    time_arr.append(time.strftime("%Y%m%d-%H%M%S", time.localtime()))
    top1_pred = int(y_train[np.argmax(y_proba)])
    topn_pred_arr = [int(y_train[item[0]]) for item in np.argsort(y_proba, axis=0)[-topn:]]
    weighted_topn_pred_arr, class_score_arr = get_weight_topn(y_proba, topn, y_train)
    weighted_top1_pred = weighted_topn_pred_arr[-1]
    time_arr.append(time.strftime("%Y%m%d-%H%M%S", time.localtime()))
#     for item in time_arr:
#         print(item)
    return top1_pred, topn_pred_arr, weighted_top1_pred, weighted_topn_pred_arr, class_score_arr

idx = 0
topn = 10
# train
print(int(y_train[idx]), get_single_pred(idx, x_train, x_train, topn, batch_size=1024))
# val
# print(int(y_val[idx]), get_single_pred(idx, x_val, x_train, topn, batch_size=1024))
topn = 10
amount = big_debug_rows
top1_count = 0
topn_1_count = 0
topn_count = 0
for idx in range(amount):
    top1_pred, topn_pred_arr, weighted_top1_pred, weighted_topn_pred_arr, class_score_arr = get_single_pred(idx, x_train, x_train, topn, batch_size=1024)
    
    item_y_data = int(y_train[idx])
    if item_y_data == top1_pred:
        top1_count += 1
    if item_y_data == weighted_top1_pred:
        topn_1_count += 1
    if item_y_data in topn_pred_arr:
        topn_count += 1
    print(item_y_data, '-->', top1_pred, weighted_top1_pred, '\t', weighted_topn_pred_arr, class_score_arr)

print('*' * 80)
print(top1_count, '%.2f' % (top1_count/amount))
print(topn_1_count, '%.2f' % (topn_1_count/amount))
print(topn_count, '%.2f' % (topn_count/amount))
topn = 10
amount = big_debug_rows
top1_count = 0
topn_1_count = 0
topn_count = 0
for idx in range(amount):
    top1_pred, topn_pred_arr, weighted_top1_pred, weighted_topn_pred_arr, class_score_arr = get_single_pred(idx, x_val, x_train, topn, batch_size=1024)
    
    item_y_data = int(y_val[idx])
    if item_y_data == top1_pred:
        top1_count += 1
    if item_y_data == weighted_top1_pred:
        topn_1_count += 1
    if item_y_data in topn_pred_arr:
        topn_count += 1
    print(item_y_data, '-->', top1_pred, weighted_top1_pred, '\t', weighted_topn_pred_arr, class_score_arr)

print('*' * 80)
print(top1_count, '%.2f' % (top1_count/amount))
print(topn_1_count, '%.2f' % (topn_1_count/amount))
print(topn_count, '%.2f' % (topn_count/amount))
acc10000 = int(topn_count/amount*10000)
# acc10000 = 10000 # for test
# print(acc10000)
if acc10000 == 10000:
    run_name_acc = '%s_%05d' % (run_name, acc10000)
else:
    run_name_acc = '%s_%04d' % (run_name, acc10000)
print(run_name_acc)
print('Time elapsed: %.1fs' % (time.time() - t0))
print(run_name_acc)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.13342956120092378, 'fraction_numerical': 0.04942263279445727, 'mean_word_length': 4.647538311053147, 'pattern_counts': {'":': 0, '<': 42, '<?xml version=': 0, '>': 53, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 11, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '12', 'dir': 'jupyter-structured-clean-dedup', 'id': '39389', 'license': 'MIT', 'n_tokens_mistral': 7215, 'n_tokens_neox': 6643, 'n_words': 1100, 'path': 'landmark-recognition-2019/4. Image-Similar-FCNN-Binary-Predict[batch].ipynb', 'repo_name': 'StudyExchange/Kaggle', 'size': '17320'}