<jupyter_start><jupyter_text>4. RepredictAbstract- 加载CNN model，并使用data_generator预测多次，然后ensemble  Run names<jupyter_code>import time

project_name = 'ic_furniture2018'
step_name = 'Repredict'
time_str = time.strftime("%Y%m%d_%H%M%S", time.localtime())
run_name = project_name + '_' + step_name + '_' + time_str
print('run_name: ' + run_name)
t0 = time.time()<jupyter_output>run_name: ic_furniture2018_Repredict_20180507_114547<jupyter_text>Improtant Params<jupyter_code>statistics_type = 'val_mean'

target_size = (224, 224) # default
batch_size = 32 # default

print(statistics_type, target_size, batch_size)<jupyter_output>val_mean (224, 224) 32<jupyter_text>Import PKGs<jupyter_code>import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import matplotlib.image as mpimg
%matplotlib inline
from IPython.display import display

import os
import gc
import math
import shutil
import zipfile
import pickle
import h5py
from PIL import Image

from tqdm import tqdm
from multiprocessing import cpu_count

from sklearn.model_selection import train_test_split
from sklearn.metrics import confusion_matrix, accuracy_score

cpu_amount = cpu_count()<jupyter_output>/home/ubuntu/anaconda3/envs/tensorflow_p36/lib/python3.6/site-packages/h5py/__init__.py:36: FutureWarning: Conversion of the second argument of issubdtype from `float` to `np.floating` is deprecated. In future, it will be treated as `np.float64 == np.dtype(float).type`.
  from ._conv import register_converters as _register_converters<jupyter_text>Project folders<jupyter_code>cwd = os.getcwd()
input_folder = os.path.join(cwd, 'input')
output_folder = os.path.join(cwd, 'output')
model_folder = os.path.join(cwd, 'model')
feature_folder = os.path.join(cwd, 'feature')
post_pca_feature_folder = os.path.join(cwd, 'post_pca_feature')
log_folder = os.path.join(cwd, 'log')
print('input_folder: \t\t\t%s' % input_folder)
print('output_folder: \t\t\t%s' % output_folder)
print('model_folder: \t\t\t%s' % model_folder)
print('feature_folder: \t\t%s' % feature_folder)
print('post_pca_feature_folder: \t%s' % post_pca_feature_folder)
print('log_folder: \t\t\t%s' % log_folder)

org_train_folder = os.path.join(input_folder, 'org_train')
org_val_folder = os.path.join(input_folder, 'org_val')
org_test_folder = os.path.join(input_folder, 'org_test')
train_folder = os.path.join(input_folder, 'data_train')
val_folder = os.path.join(input_folder, 'data_val')
test_folder = os.path.join(input_folder, 'data_test')
test_sub_folder = os.path.join(test_folder, 'test')

if not os.path.exists(post_pca_feature_folder):
    os.mkdir(post_pca_feature_folder)
    print('Create folder: %s' % post_pca_feature_folder)

train_json_file = os.path.join(input_folder, 'train.json')
val_json_file = os.path.join(input_folder, 'validation.json')
test_json_file = os.path.join(input_folder, 'test.json')
print('\ntrain_json_file: \t\t%s' % train_json_file)
print('val_json_file: \t\t\t%s' % val_json_file)
print('test_json_file: \t\t%s' % test_json_file)

train_csv_file = os.path.join(input_folder, 'train.csv')
val_csv_file = os.path.join(input_folder, 'validation.csv')
test_csv_file = os.path.join(input_folder, 'test.csv')
print('\ntrain_csv_file: \t\t%s' % train_csv_file)
print('val_csv_file: \t\t\t%s' % val_csv_file)
print('test_csv_file: \t\t\t%s' % test_csv_file)

sample_submission_csv_file = os.path.join(input_folder, 'sample_submission_randomlabel.csv')
print('\nsample_submission_csv_file: \t%s' % sample_submission_csv_file)<jupyter_output>input_folder: 			/data1/kaggle/imaterialist-challenge-furniture-2018/input
output_folder: 			/data1/kaggle/imaterialist-challenge-furniture-2018/output
model_folder: 			/data1/kaggle/imaterialist-challenge-furniture-2018/model
feature_folder: 		/data1/kaggle/imaterialist-challenge-furniture-2018/feature
post_pca_feature_folder: 	/data1/kaggle/imaterialist-challenge-furniture-2018/post_pca_feature
log_folder: 			/data1/kaggle/imaterialist-challenge-furniture-2018/log

train_json_file: 		/data1/kaggle/imaterialist-challenge-furniture-2018/input/train.json
val_json_file: 			/data1/kaggle/imaterialist-challenge-furniture-2018/input/validation.json
test_json_file: 		/data1/kaggle/imaterialist-challenge-furniture-2018/input/test.json

train_csv_file: 		/data1/kaggle/imaterialist-challenge-furniture-2018/input/train.csv
val_csv_file: 			/data1/kaggle/imaterialist-challenge-furniture-2018/input/validation.csv
test_csv_file: 			/data1/kaggle/imaterialist-challenge-furniture-2018/input/test.csv
[...]<jupyter_text>Preview data<jupyter_code>train_csv = pd.read_csv(train_csv_file)
print('train_csv.shape is {0}.'.format(train_csv.shape))
display(train_csv.head(2))

val_csv = pd.read_csv(val_csv_file)
print('val_csv.shape is {0}.'.format(val_csv.shape))
display(val_csv.head(2))

test_csv = pd.read_csv(test_csv_file)
print('test_csv.shape is {0}.'.format(test_csv.shape))
display(test_csv.head(2))

test_csv = pd.read_csv(test_csv_file)
print('test_csv.shape is {0}.'.format(test_csv.shape))
display(test_csv.head(2))

sample_submission_csv = pd.read_csv(sample_submission_csv_file)
print('sample_submission_csv.shape is {0}.'.format(sample_submission_csv.shape))
display(sample_submission_csv.head(2))
train_id = train_csv['image_id']
train_label_id = train_csv['label_id']

id_2_train_label_id_dict = dict(zip(train_id, train_label_id))
print('len(id_2_train_label_id_dict)=%d' % len(id_2_train_label_id_dict))

index = 0
print('id: %s, \tlandmark_id:%s' % (train_id[index], id_2_train_label_id_dict[train_id[index]]))
index = 1
print('id: %s, \tlandmark_id:%s' % (train_id[index], id_2_train_label_id_dict[train_id[index]]))

image_file = '%s_%s.jpg' % (train_id[index], id_2_train_label_id_dict[train_id[index]])
print(image_file)
val_id = val_csv['image_id']
val_label_id = val_csv['label_id']

id_2_val_label_id_dict = dict(zip(val_id, val_label_id))
print('len(id_2_val_label_id_dict)=%d' % len(id_2_val_label_id_dict))

index = 0
print('id: %s, \tlandmark_id:%s' % (val_id[index], id_2_val_label_id_dict[val_id[index]]))
index = 1
print('id: %s, \tlandmark_id:%s' % (val_id[index], id_2_val_label_id_dict[val_id[index]]))

image_file = '%s_%s.jpg' % (val_id[index], id_2_val_label_id_dict[val_id[index]])
print(image_file)
test_id = test_csv['image_id']

index = 0
print('id: %s' % (test_id[index]))
index = 1
print('id: %s' % (test_id[index]))

image_file = '%s.jpg' % (test_id[index])
print(image_file)<jupyter_output>id: 1
id: 2
2.jpg<jupyter_text>Data Generator<jupyter_code>from keras.preprocessing.image import ImageDataGenerator
from keras.callbacks import LearningRateScheduler, TensorBoard, EarlyStopping
train_gen = ImageDataGenerator(
    rotation_range=20,
    height_shift_range=0.2,
    width_shift_range=0.2,
    shear_range=0.2,
    zoom_range=0.2,
    channel_shift_range=20,
    horizontal_flip=True,
    vertical_flip=True,
    fill_mode='wrap'
)

val_gen = ImageDataGenerator(
    rotation_range=20,
    height_shift_range=0.2,
    width_shift_range=0.2,
    shear_range=0.2,
    zoom_range=0.2,
    channel_shift_range=20,
    horizontal_flip=True,
    vertical_flip=True,
    fill_mode='wrap'
)

test_gen = ImageDataGenerator(
    rotation_range=20,
    height_shift_range=0.2,
    width_shift_range=0.2,
    shear_range=0.2,
    zoom_range=0.2,
    channel_shift_range=20,
    horizontal_flip=True,
    vertical_flip=True,
    fill_mode='wrap'
)
data_train_path = os.path.join(input_folder, 'data_train')
data_val_path = os.path.join(input_folder, 'data_val')
data_test_path = os.path.join(input_folder, 'data_test')
# train_generator = train_gen.flow_from_directory(data_train_path, target_size, shuffle=True, batch_size=batch_size)
val_generator = val_gen.flow_from_directory(data_val_path, target_size, shuffle=False, batch_size=batch_size)
test_generator = test_gen.flow_from_directory(data_test_path, target_size, shuffle=False, batch_size=batch_size)
# print(train_generator.classes[:10])
print(val_generator.classes[:10])
print(test_generator.classes[:10])

print(test_generator.filenames[:10])
y_val = val_generator.classes<jupyter_output><empty_output><jupyter_text>Load model<jupyter_code>from sklearn.metrics import confusion_matrix

from keras.utils.np_utils import to_categorical # convert to one-hot-encoding
from keras.applications import imagenet_utils
from keras.models import *
from keras.layers import *
from keras.optimizers import *
from keras.applications import *
model_files = [
#     { 'model_name': 'ic_furniture2018_Train-Predict_Mix3model_20180415_053404_7941.h5'},
#     { 'model_name': 'ic_furniture2018_Train-Predict_Mix3model_20180415_061203_7987.h5'},
#     { 'model_name': 'ic_furniture2018_Train-Predict_Mix3model_20180415_070425_8115.h5'},
#     { 'model_name': 'ic_furniture2018_Train-Predict_Mix3model_20180415_071659_8120.h5'},
#     { 'model_name': 'ic_furniture2018_Train-Predict_Mix3model_20180415_072816_8131.h5'},
#     { 'model_name': 'ic_furniture2018_Train-Predict_Mix3model_20180415_075023_8175.h5'},
#     { 'model_name': 'ic_furniture2018_Train-Predict_Mix3model_20180415_080533_8136.h5'},
#     { 'model_name': 'ic_furniture2018_Train-Predict_Mix3model_20180415_081948_8214.h5'},
#     { 'model_name': 'ic_furniture2018_TrainPredict_Mix3model_20180429_100055_8077.h5'},
#     { 'model_name': 'ic_furniture2018_TrainPredict_Mix3model_20180429_102052_8077.h5'},
#     { 'model_name': 'ic_furniture2018_TrainPredict_Mix3model_20180429_103656_8060.h5'},
#     { 'model_name': 'ic_furniture2018_TrainPredict_Mix3model_20180429_110955_8066.h5'},
    
    { 'model_name': 'ic_furniture2018_TrainPredict_FineTune1_20180505_091959_8284.h5', 'image_size': 224},
    { 'model_name': 'ic_furniture2018_TrainPredict_FineTune_20180417_170937_8341.h5', 'image_size': 299},
    { 'model_name': 'ic_furniture2018_TrainPredict_FineTune_20180418_130955_8303.h5', 'image_size': 299},
#     { 'model_name': 'ic_furniture2018_TrainPredict_FineTune_20180419_141628_2393.h5', 'image_size': 450},
#     { 'model_name': 'ic_furniture2018_TrainPredict_FineTune_20180419_141628_8255.h5', 'image_size': 450},
    { 'model_name': 'ic_furniture2018_TrainPredict_FineTune_20180422_041707_8300.h5', 'image_size': 224},
    { 'model_name': 'ic_furniture2018_TrainPredict_FineTune_20180422_163907_8367.h5', 'image_size': 300},
    { 'model_name': 'ic_furniture2018_TrainPredict_FineTune_20180423_134043_8426.h5', 'image_size': 300},
    { 'model_name': 'ic_furniture2018_TrainPredict_FineTune_20180424_143948_8278.h5', 'image_size': 200},
    { 'model_name': 'ic_furniture2018_TrainPredict_FineTune_20180427_122315_8392.h5', 'image_size': 300},
]


# load_model(filepath)

for model_file in model_files:
    if os.path.exists(os.path.join(model_folder, model_file['model_name'])):
        print('File exists: %s' % model_file['model_name'])
    else:
        print('***File do not exists: %s' % model_file['model_name'])
def predict_acc(model, generator, epoch=10, y_data=None):
    print('|', end='')
    y_data_probas = []
    y_data_accs = []
    for i in range(epoch):
        print('*', end='')
        epoch_start = time.time()
        y_data_proba_temp = model.predict_generator(
            generator,
            max_queue_size=128,
            workers=cpu_amount,
            use_multiprocessing=True
        )
        if y_data is not None:
            y_data_pred_temp = np.argmax(y_data_proba_temp, -1)
            acc = accuracy_score(y_data, y_data_pred_temp)
            y_data_accs = y_data_accs + acc
        y_data_probas.append(y_data_proba_temp)
        print(' %.1fs ' % (time.time() - epoch_start), end='')
    print('|', end='')
    print(len(y_data_probas), end=' ')
    print(y_data_probas[0].shape, end=' ')
    return y_data_probas, y_data_accs
def predict(model_files, gen, data_path, is_test=False, epoch=1, batch_size=32):
    y_data_probas = []
    for model_dict in model_files:
        print(model_dict)
        model_file = os.path.join(model_folder, model_dict['model_name'])
        target_size = (model_dict['image_size'], model_dict['image_size'])
        generator = gen.flow_from_directory(data_path, target_size, shuffle=False, batch_size=batch_size)
        y_data = None
        if is_test is not True:
            y_data = generator.classes
        model = load_model(model_file, custom_objects={'imagenet_utils': imagenet_utils})
        try:
            y_data_proba_single, y_data_acc_single = predict_acc(model, generator, epoch, y_data)
        except Exception as ex:
            print(ex)
            continue
        print(y_data_acc_single)
        y_data_probas = y_data_probas + y_data_proba_single
    return y_data_probas

import pickle
generator_file = os.path.join(model_folder, "generator.dat")
print(generator_file)
pickle.dump(val_gen, open(generator_file, "wb"), True)
val_gen1 = pickle.load(open(generator_file, "rb"))

y_val_probas = predict(model_files, val_gen1, data_val_path, False, 1, batch_size)<jupyter_output>/data1/kaggle/imaterialist-challenge-furniture-2018/model/generator.dat
{'model_name': 'ic_furniture2018_TrainPredict_FineTune1_20180505_091959_8284.h5', 'image_size': 224}
Found 6301 images belonging to 128 classes.
|* 49.9s |1 (6301, 128) []
{'model_name': 'ic_furniture2018_TrainPredict_FineTune_20180417_170937_8341.h5', 'image_size': 299}
Found 6301 images belonging to 128 classes.
|* 66.6s |1 (6301, 128) []
{'model_name': 'ic_furniture2018_TrainPredict_FineTune_20180418_130955_8303.h5', 'image_size': 299}
Found 6301 images belonging to 128 classes.
|* 67.9s |1 (6301, 128) []
{'model_name': 'ic_furniture2018_TrainPredict_FineTune_20180422_041707_8300.h5', 'image_size': 224}
Found 6301 images belonging to 128 classes.
|* 60.8s |1 (6301, 128) []
{'model_name': 'ic_furniture2018_TrainPredict_FineTune_20180422_163907_8367.h5', 'image_size': 300}
Found 6301 images belonging to 128 classes.
|* 69.1s |1 (6301, 128) []
{'model_name': 'ic_furniture2018_TrainPredict_FineTune_20180423_134043_8[...]<jupyter_text>Get basic statistical data<jupyter_code>ensemble_res = {}
print(len(y_val_probas))
print(y_val_probas[0].shape)
def get_acc(y_pred, y_proba):
    max_indexes = np.argmax(y_proba, -1)
    return accuracy_score(y_pred ,max_indexes)
%%time
def get_mean(probas):
    probas_newaxises = []
    for p in probas:
        print('*', end='')
        probas_newaxises.append(p[:, :, np.newaxis])
    probas_newaxis = np.concatenate(probas_newaxises, axis=-1)
    probas_mean = np.mean(probas_newaxis, axis=-1)
    print('probas_mean.shape: \t', probas_mean.shape)
    return probas_mean

ensemble_res['val_mean'] = get_acc(y_val ,get_mean(y_val_probas))
# y_test_proba_mean = get_mean(y_test_probas)

# print(ensemble_res)
%%time
def get_max(probas):
    probas_newaxises = []
    for p in probas:
        print('*', end='')
        probas_newaxises.append(p[:, :, np.newaxis])
    probas_newaxis = np.concatenate(probas_newaxises, axis=-1)
    probas_max = np.max(probas_newaxis, axis=-1)
    print('probas_max.shape: \t', probas_max.shape)
    return probas_max

ensemble_res['val_max'] = get_acc(y_val ,get_max(y_val_probas))
# y_test_proba_max = get_max(y_test_probas)

# print(ensemble_res)
for i, key in enumerate(ensemble_res.keys()):
    if i % 2 == 0:
        print('%s  \t%.4f' % (key, ensemble_res[key]), end='\t')
    else:
        print('%s  \t%.4f' % (key, ensemble_res[key]))<jupyter_output>val_mean  	0.8605	val_max  	0.8405<jupyter_text>Predict<jupyter_code>y_test_probas = predict(model_files, test_gen, data_test_path, False, 5, batch_size)
print(len(y_test_probas))
print(y_test_probas[0].shape)
y_test_proba_mean = get_mean(y_test_probas)
y_test_proba_max = get_max(y_test_probas)
%%time

if statistics_type == 'val_mean':
    max_indexes = np.argmax(y_test_proba_mean, -1)
elif statistics_type == 'val_min':
    max_indexes = np.argmax(y_test_proba_min, -1)
elif statistics_type == 'val_max':
    max_indexes = np.argmax(y_test_proba_max, -1)
else: 
    # statistics_type == 'val_median'
    max_indexes = np.argmax(y_test_proba_median, -1)
# print(xg_test_pred.shape)

test_filenames = test_generator.filenames
test_dict = {}
for pair in zip(test_filenames, max_indexes):
    image_name, indx = pair[0], int(pair[1])
    image_name = image_name.split('/')[-1]
    image_id = int(image_name.split('.')[0])
#     print(pair[0], image_name, image_id, indx, indx+1, type(image_id), type(indx))
    test_dict[image_id] = indx + 1

#确认图片的id是否能与ImageDataGenerator()对应上
for name in test_filenames[:10]:
    image_name = name.split('/')[-1]
    image_id = int(image_name.split('.')[0])
#     print('%s\t%s\t%s' % (name, image_id, test_dict[image_id]))
display(sample_submission_csv.head(2))
%%time
len_sample_submission_csv = len(sample_submission_csv)
print('len(len_sample_submission_csv)=%d' % len_sample_submission_csv)
count = 0
for i in range(len_sample_submission_csv):
    image_id = int(sample_submission_csv.iloc[i, 0])
    if image_id in test_dict:
        pred_label = test_dict[image_id]
#         print('%s\t%s' % (image_id, pred_label))
        sample_submission_csv.iloc[i, 1] = pred_label
    else:
#         print('%s\t%s' % (image_id, 20))
        sample_submission_csv.iloc[i, 1] = 20 # 属于20的类最多，所以全都设置成这个类，可能会比设置成其他得到的结果好
    count += 1
    if count % 1000 == 0:
        print(int(count/1000), end=' ')
display(sample_submission_csv.head(2))
print(list(set(sample_submission_csv['predicted'])))
run_name_acc = run_name + '_' + str(int(ensemble_res[statistics_type]*10000)).zfill(4)
print(run_name_acc)
pred_file = os.path.join(output_folder, 'pred_%s.csv' % run_name_acc)
sample_submission_csv.to_csv(pred_file, index=None)
print('Time cost: %.2f s' % (time.time() - t0))
print(statistics_type)

print(run_name_acc)
print('Done !')<jupyter_output>Time cost: 12959.18 s
val_mean
ic_furniture2018_Repredict_20180507_114547_8604
Done !
{'fraction_non_alphanumeric': 0.12886569007128024, 'fraction_numerical': 0.061177527080877814, 'mean_word_length': 5.938473520249222, 'pattern_counts': {'":': 0, '<': 29, '<?xml version=': 0, '>': 29, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '9', 'dir': 'jupyter-structured-clean-dedup', 'id': '437901', 'license': 'MIT', 'n_tokens_mistral': 7781, 'n_tokens_neox': 7066, 'n_words': 996, 'path': 'imaterialist-challenge-furniture-2018/4. Repredict.ipynb', 'repo_name': 'StudyExchange/Kaggle', 'size': '17817'}