<jupyter_start><jupyter_text>3. Image-Similar-FCNN-BinaryFor landmark-recognition-2019 algorithm validation  Run name<jupyter_code>import time
project_name = 'Google-LandMark-Rec2019'
step_name = '3-Image-Similar-FCNN-Binary'
time_str = time.strftime("%Y%m%d-%H%M%S", time.localtime())
run_name = project_name + '_' + step_name + '_' + time_str
print('run_name: ' + run_name)
t0 = time.time()<jupyter_output>run_name: Google-LandMark-Rec2019_3-Image-Similar-FCNN-Binary_20190511-164020<jupyter_text>Important params<jupyter_code>import multiprocessing

cpu_amount = multiprocessing.cpu_count()
print('cpu_amount: ', cpu_amount)

batch_size = 16
print(batch_size)<jupyter_output>cpu_amount:  4
16<jupyter_text>Import PKGs<jupyter_code>import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import matplotlib.image as mpimg
%matplotlib inline
from IPython.display import display

import os
import sys
import gc
import math
import shutil
import zipfile
import pickle
import h5py
import random

# from tqdm import tqdm
from datetime import datetime

from sklearn.model_selection import train_test_split
from sklearn.metrics import confusion_matrix, accuracy_score
import keras
from keras.utils import Sequence
from keras.layers import *
from keras.models import *
from keras.applications import *
from keras.optimizers import *
from keras.regularizers import *
from keras.preprocessing.image import *
from keras.applications.inception_v3 import preprocess_input
import tensorflow as tf
import keras.backend.tensorflow_backend as KTF

config = tf.ConfigProto()
config.gpu_options.per_process_gpu_memory_fraction = 0.5
session = tf.Session(config=config)

KTF.set_session(session )<jupyter_output><empty_output><jupyter_text>Project folders<jupyter_code>cwd = os.getcwd()
feature_folder = os.path.join(cwd, 'feature')
input_folder = os.path.join(cwd, 'input')
output_folder = os.path.join(cwd, 'output')
model_folder = os.path.join(cwd, 'model')

org_train_folder = os.path.join(input_folder, 'org_train')
org_test_folder = os.path.join(input_folder, 'org_test')
train_folder = os.path.join(input_folder, 'data_train')
val_folder = os.path.join(input_folder, 'data_val')
test_folder = os.path.join(input_folder, 'data_test')
test_sub_folder = os.path.join(test_folder, 'test')

vgg16_feature_file = os.path.join(feature_folder, 'feature_wrapper_171023.h5')
train_csv_file = os.path.join(input_folder, 'train.csv')
test_csv_file = os.path.join(input_folder, 'test.csv')
sample_submission_folder = os.path.join(input_folder, 'sample_submission.csv')
print(vgg16_feature_file)
print(train_csv_file)
print(test_csv_file)
print(sample_submission_folder)<jupyter_output>/data/landmark-recognition-2019/feature/feature_wrapper_171023.h5
/data/landmark-recognition-2019/input/train.csv
/data/landmark-recognition-2019/input/test.csv
/data/landmark-recognition-2019/input/sample_submission.csv<jupyter_text>Functions<jupyter_code>def pickle_dump(data, file):
    with open(file, 'wb') as f:
        pickle.dump(data, f)

def pickle_load(file):
    with open(file, 'rb') as f:
        data = pickle.load(f)
    return data

a = list(range(10))
print(a)
demo_file = os.path.join(os.getcwd(), 'temp', 'pickle_demo.pkl')
print(demo_file)
pickle_dump(a, demo_file)
new_a = pickle_load(demo_file)
print(new_a)
%%time

def get_whole_classes_label(class_indices_data, item_classes_data):
    indices2class = {}
    for i, key in enumerate(class_indices_data.keys()):
#         print(key, class_indices_data[key])
        indices2class[class_indices_data[key]] = key
#         if i >= 5:
#             break
#     for i, key in enumerate(class_indices_data.keys()):
#         print(key, class_indices_data[key], '-->', class_indices_data[key], indices2class[class_indices_data[key]])
#         if i >= 10:
#             break
    print(item_classes_data.shape)
    whole_classes_data = np.zeros(item_classes_data.shape)
    for i, class_index in enumerate(item_classes_data):
        whole_classes_data[i] = indices2class[class_index]
#         print(i, class_index, '-->', indices2class[class_index])
#         if i >= 10:
#             break
    return whole_classes_data
# get_whole_classes_label(class_indices_train, item_classes_train)<jupyter_output>CPU times: user 0 ns, sys: 0 ns, total: 0 ns
Wall time: 6.91 µs<jupyter_text>Load feature<jupyter_code>!ls ./feature -hl
def load_feature(data_set_name, model_name, date_str, feature_amount):
#     data_set_name = 'train'
#     model_name = 'VGG19'
#     date_str = '171023'
    x_data_arr = []
    classes_data_arr = []
    index_data_arr = []
    for i in range(feature_amount):
        folder_name = 'data_%s_%02d' % (data_set_name, i)
        class_indices_file = os.path.join(cwd, 'feature', 'feature_{0}_{1}_{2}_class_indices.pkl'.format(model_name, folder_name, date_str))
        h5py_file_name = os.path.join(cwd, 'feature', 'feature_{0}_{1}_{2}.h5'.format(model_name, folder_name, date_str))
        if not os.path.exists(class_indices_file):
            print('File not exists', class_indices_file)
            continue
        if not os.path.exists(h5py_file_name):
            print('File not exists', h5py_file_name)
            continue
        print(class_indices_file)
        print(h5py_file_name)
        class_indices_data = pickle_load(class_indices_file)
        print(len(class_indices_data))

        with h5py.File(h5py_file_name, 'r') as h:
            item_x_data = np.array(h['x_data_%s_%02d' % (data_set_name, i)])
            item_classes_data = np.array(h['classes_data_%s_%02d' % (data_set_name, i)])
            item_index_data = np.array(h['index_data_%s_%02d' % (data_set_name, i)])
        print(item_x_data.shape)
        x_data_arr.append(item_x_data)
        item_y_data = get_whole_classes_label(class_indices_data, item_classes_data)
        print(item_y_data.shape, item_y_data[:10])
        classes_data_arr.append(item_y_data)
        index_data_arr.append(item_index_data)
    x_data = np.concatenate(x_data_arr, axis=0)
    y_data = np.concatenate(classes_data_arr, axis=0)
    idx_data = np.concatenate(index_data_arr, axis=0)
    print('*' * 60)
    print(x_data.shape)
    print(y_data.shape)
    print(idx_data.shape)
    return x_data, y_data, idx_data

x_train, y_train, idx_train = load_feature('train', 'VGG19', '171023', 40)
x_val, y_val, idx_val = load_feature('val', 'VGG19', '171023', 4)<jupyter_output>/data/landmark-recognition-2019/feature/feature_VGG19_data_val_00_171023_class_indices.pkl
/data/landmark-recognition-2019/feature/feature_VGG19_data_val_00_171023.h5
34160
(50000, 512)
(50000,)
(50000,) [ 1. 12. 20. 27. 27. 27. 27. 27. 27. 32.]
/data/landmark-recognition-2019/feature/feature_VGG19_data_val_01_171023_class_indices.pkl
/data/landmark-recognition-2019/feature/feature_VGG19_data_val_01_171023.h5
34196
(50000, 512)
(50000,)
(50000,) [ 9. 11. 24. 27. 27. 27. 27. 27. 27. 27.]
/data/landmark-recognition-2019/feature/feature_VGG19_data_val_02_171023_class_indices.pkl
/data/landmark-recognition-2019/feature/feature_VGG19_data_val_02_171023.h5
34264
(50000, 512)
(50000,)
(50000,) [ 1. 10. 11. 12. 27. 27. 27. 27. 27. 27.]
/data/landmark-recognition-2019/feature/feature_VGG19_data_val_03_171023_class_indices.pkl
/data/landmark-recognition-2019/feature/feature_VGG19_data_val_03_171023.h5
34155
(50000, 512)
(50000,)
(50000,) [ 1.  2. 10. 11. 13. 24. 27. 27. 27. 27.]
****************[...]<jupyter_text>ImageSequence<jupyter_code>class ImageSequence(Sequence):
    def __init__(self, x, y, batch_size, times_for_1_image, positive_rate):
        self.x = x
        self.y = y
        self.batch_size = batch_size
        self.times_for_1_image = times_for_1_image
        self.positive_rate = positive_rate
        
        self.len_x = self.x.shape[0]
        self.index = list(range(self.len_x))
        self.group = {}
        classes = list(set(self.y))
        classes.sort()
        self.classes = [item for item in classes]
        for c in self.classes:
            self.group[c] = []
        for i, y_i in enumerate(self.y):
            temp_arr = self.group[y_i]
            temp_arr.append(i)
            self.group[y_i] = temp_arr
        
    def __len__(self):
        # times_for_1_image: the times to train one image
        # 2: positive example and negative example
        return self.times_for_1_image * 2 * (math.ceil(self.len_x/self.batch_size))
    def __getitem__(self, idx):
        batch_main_x = np.zeros((self.batch_size, self.x.shape[1]))
        batch_libary_x = np.zeros((self.batch_size, self.x.shape[1]))
        batch_x = {}
        batch_y = [] # 0 or 1
        new_class = [item for item in self.classes]
        # is_positive = random.random() < self.positive_rate
        for i in range(self.batch_size):
            # prepare main image
            item_main_image_idx = random.choice(self.index) # random choice one image from all train images
            item_main_image_y = self.y[item_main_image_idx]
            
            # prepare libary image
            is_positive = random.random() < self.positive_rate
            if is_positive: # chioce a positive image as libary_x
                # choice one image from itself group
                item_libary_image_idx = random.choice(self.group[item_main_image_y]) # don't exclude item_main_image_idx, so it could choice a idx same to item_main_image_idx.
            else: # chioce a negative image as libary_x
                # choice group
                # new_class = copy.deepcopy(self.classes) # this method make generator slow
                if item_main_image_y in new_class:
                    new_class.remove(item_main_image_y)
                item_libary_image_group_num = random.choice(new_class)
                # choice one image from group
                item_libary_image_idx = random.choice(self.group[item_libary_image_group_num])
            # add item data to batch
            batch_main_x[i] = self.x[item_main_image_idx]
            batch_libary_x[i] = self.x[item_libary_image_idx]
            batch_y.append(int(is_positive))
        # concatenate array to np.array
        batch_x = {
            'main_input': batch_main_x,
            'library_input': batch_libary_x
        }
        batch_y = np.array(batch_y)
        return batch_x, batch_y

demo_sequence = ImageSequence(x_train[:200], y_train[:200], 128, 3, 0.1)
print(len(demo_sequence))
print(type(demo_sequence))

batch_index = 0
demo_batch = demo_sequence[batch_index]
demo_batch_x = demo_batch[0]
demo_batch_y = demo_batch[1]
print(type(demo_batch_x))
print(type(demo_batch_y))

demo_main_input = demo_batch_x['main_input']
demo_library_input = demo_batch_x['library_input']
print(demo_main_input.shape)
print(demo_library_input.shape)
print(demo_batch_y.shape)

# print(demo_main_input[0])
print(demo_batch_y)
train_sequence = ImageSequence(x_train, y_train, batch_size, 3, 0.5)
val_sequence = ImageSequence(x_val, y_val, batch_size, 3, 0.5)<jupyter_output><empty_output><jupyter_text>Model<jupyter_code>main_input = Input((x_train.shape[1],), dtype='float32', name='main_input')
library_input = Input((x_train.shape[1],), dtype='float32', name='library_input')

x = keras.layers.concatenate([main_input, library_input])
# x = Dense(x_train.shape[1]*2, activation='relu')(x)
x = Reshape((x_train.shape[1]*2, 1))(x)
x = BatchNormalization()(x)
x = Conv1D(2, 1, input_shape=(x_train.shape[1]*2, 1), activation='relu')(x)
# x = Conv1D(4, 1, input_shape=(x_train.shape[1]*2, 1), activation='relu')(x)
# x = Conv1D(8, 1, input_shape=(x_train.shape[1]*2, 1), activation='relu')(x)
# x = BatchNormalization()(x)
# x = Dropout(0.2)(x)
# x = Conv1D(8, 1, input_shape=(x_train.shape[1]*2, 1), activation='relu')(x)
# x = Conv1D(4, 1, input_shape=(x_train.shape[1]*2, 1), activation='relu')(x)
x = Conv1D(2, 1, input_shape=(x_train.shape[1]*2, 1), activation='relu')(x)
x = BatchNormalization()(x)
x = Flatten()(x)
# x = Dropout(0.2)(x)
# x = Dense(1024, activation='relu')(x)
# x = Dropout(0.2)(x)
# x = Dense(1024, activation='relu')(x)
# x = BatchNormalization()(x)
# x = Dropout(0.2)(x)
x = Dense(128, activation='relu')(x)
output = Dense(1, activation='sigmoid')(x)

model = Model(inputs=[main_input, library_input], outputs=[output])
model.compile(optimizer=Adam(lr=1e-5), loss='binary_crossentropy', metrics=['accuracy'])
model.summary()
hist = model.fit_generator(
    train_sequence, 
    steps_per_epoch=800, 
    epochs=1500, 
    verbose=1, 
    callbacks=None, 
    validation_data=val_sequence, 
    validation_steps=100, 
    class_weight=None, 
    max_queue_size=batch_size*16, 
    workers=cpu_amount, 
    use_multiprocessing=True, 
    shuffle=False, 
    initial_epoch=0
)
import matplotlib.pyplot as plt
%matplotlib inline

plt.plot(hist.history['loss'], color='b')
plt.plot(hist.history['val_loss'], color='r')
plt.show()
plt.plot(hist.history['acc'], color='b')
plt.plot(hist.history['val_acc'], color='r')
plt.show()
loss, acc = model.evaluate_generator(val_sequence, steps=100)
print(loss, acc)
acc10000 = int(acc*10000)
if acc10000 == 10000:
    run_name_acc = '%s_%05d' % (run_name, acc10000)
else:
    run_name_acc = '%s_%04d' % (run_name, acc10000)
print(run_name_acc)
def saveModel(model, run_name):
    cwd = os.getcwd()
    modelPath = os.path.join(cwd, 'model')
    if not os.path.isdir(modelPath):
        os.mkdir(modelPath)
    weigths_file = os.path.join(modelPath, run_name + '.h5')
    print(weigths_file)
    model.save(weigths_file)
saveModel(model, run_name_acc)
print('Time elapsed: %.1fs' % (time.time() - t0))
print(run_name_acc)<jupyter_output>Time elapsed: 32147.5s
Google-LandMark-Rec2019_3-Image-Similar-FCNN-Binary_20190511-164020_8393
{'fraction_non_alphanumeric': 0.1266072472238457, 'fraction_numerical': 0.044929865575686734, 'mean_word_length': 4.187192118226601, 'pattern_counts': {'":': 0, '<': 29, '<?xml version=': 0, '>': 32, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '8', 'dir': 'jupyter-structured-clean-dedup', 'id': '396554', 'license': 'MIT', 'n_tokens_mistral': 5451, 'n_tokens_neox': 5004, 'n_words': 876, 'path': 'landmark-recognition-2019/3. Image-Similar-FCNN-Binary.ipynb', 'repo_name': 'StudyExchange/Kaggle', 'size': '13688'}