<jupyter_start><jupyter_text>Use dataset __7__ as test set<jupyter_code>from inception_v4 import InceptionV4
from keras.applications.densenet import DenseNet201
from keras.layers import (GlobalAveragePooling2D, Dense, Conv2D, Flatten,
                          Dropout, concatenate, Input, Activation, normalization, GlobalAveragePooling1D, Conv1D)
from keras.optimizers import SGD
from keras.callbacks import ModelCheckpoint, TensorBoard
from keras.models import Model, load_model, model_from_json, Input
import os
import cv2
import random
import numpy as np
import matplotlib.pyplot as plt
from utils import SaveModelOnMAE_tail, generate_generator, read_image_and_K_from_dir, read_48_points


%matplotlib inline
os.environ["CUDA_VISIBLE_DEVICES"] = "0"
random.seed(7)
net = 'DenseNet201'
with_K = 'tail'
idx_test = [17, 13, 20, 21]


# Dataset
dir_path = './data/csv_files'
train_paths, train_labels, train_K, test_paths, test_labels, test_K = read_image_and_K_from_dir(dir_path, idx_test=idx_test)

test_paths_48, test_labels_48, test_K_48 = read_48_points(
    dir_path='./data/csv_files', dir_points='./data/48_points', idx_test=idx_test
)
test_paths_48, test_labels_48, test_K_48 = np.squeeze(test_paths_48).tolist(), np.squeeze(test_labels_48).tolist(), np.squeeze(test_K_48).tolist()

indexes_rand = list(range(len(train_K)))
random.shuffle(indexes_rand)
train_paths, train_labels, train_K = np.asarray(train_paths)[indexes_rand].tolist(), np.asarray(train_labels)[indexes_rand].tolist(), np.asarray(train_K)[indexes_rand].tolist()

idx_validate = np.loadtxt('./data/validation_selection_index.txt').astype(int)
validate_paths, validate_labels, validate_K = np.asarray(test_paths)[idx_validate].tolist(), np.asarray(test_labels)[idx_validate].tolist(), np.asarray(test_K)[idx_validate].tolist()

indexes_rand = list(range(len(test_K)))
random.shuffle(indexes_rand)
test_paths, test_labels, test_K = np.asarray(test_paths)[indexes_rand].tolist(), np.asarray(test_labels)[indexes_rand].tolist(), np.asarray(test_K)[indexes_rand].tolist()

print('\t\tpath\tlabel\tK')
print('train:\t\t{}\t{}\t{}'.format(len(train_paths), len(train_labels), len(train_K)))
print('val:\t\t{}\t{}\t{}'.format(len(validate_paths), len(validate_labels), len(validate_K)))
print('test:\t\t{}\t{}\t{}'.format(len(test_paths), len(test_labels), len(test_K)))
print('test_48:\t{}\t{}\t{}'.format(len(test_paths_48), len(test_labels_48), len(test_K_48)))
model = load_model('./acceptable_weights/weights_tail_K/DenseNet201_tail_K_MAE0.324.hdf5')
# upgrade keras 2.1.5 -> 2.2.4
base_model = DenseNet201(weights='imagenet', include_top=False, input_shape=(150, 150, 3))
flat = GlobalAveragePooling2D()(base_model.output)
flat_len = 1
for i in range(1, len(flat.shape)):
    flat_len *= int(flat.shape[i])
K_len = min(flat_len // 1, 2048)
input_K = Input((K_len, 1))
# K_flow = Activation(activation='linear')(input_K)
K_flow = Conv1D(K_len//3, 3, activation='relu')(input_K)
K_flow = GlobalAveragePooling1D()(K_flow)
x = concatenate([flat, K_flow])
x = Dense(1024, activation='relu')(x)
x = Dropout(0.4)(x)
x = Dense(512, activation='relu')(x)
x = Dropout(0.4)(x)
x = Dense(1, activation='relu')(x)

model = Model(inputs=[*base_model.inputs, input_K], outputs=x, name=net)
model.compile(loss='mean_absolute_error', optimizer='adam')
model.summary()
batch_size_train = 4
epochs_per_iteration = 250000
batch_size_validate = len(validate_labels) // 4

save_dir_weights = './weights/weights_' + with_K + '_K/'
if not os.path.exists(save_dir_weights):
    os.mkdir(save_dir_weights)

checkpointer_best = ModelCheckpoint(
    filepath=save_dir_weights + net + '.hdf5',
    monitor='val_loss', verbose=1, save_best_only=True, mode='min'
)
saver_MAE = SaveModelOnMAE_tail()
print('Training ...')
train_generator = generate_generator(train_paths, train_labels, train_K, batch_size_train, net=net, with_K=with_K, K_len=K_len)
validate_generator = generate_generator(validate_paths, validate_labels, validate_K, batch_size_validate, net=net, with_K=with_K, K_len=K_len)
history = model.fit_generator(
    train_generator, steps_per_epoch = np.ceil(len(train_paths)/batch_size_train/epochs_per_iteration),
    epochs=epochs_per_iteration * 3,
    validation_data=validate_generator, validation_steps=2,
    verbose=1, callbacks=[checkpointer_best, saver_MAE]
)
vl, l = history.history['val_loss'], history.history['loss']
vl, l = np.asarray(vl), np.asarray(l)
vl[vl > 1], l[l > 1] = 1, 1

plt.plot(vl, 'r')
plt.plot(l, 'b')
plt.legend(['val_loss', 'loss'])
plt.show()
# test
K_len = 1920
losses = [0.303, 0.308, 0.311, 0.313, 0.326, 0.329, 0.335]
for idx, loss in enumerate(losses[:1]):
    print('Loading {}/{} model: {}'.format(idx+1, len(losses), './weights/weights_tail_K/DenseNet201_tail_K_MAE'+str(loss)+'.hdf5'))
    model = load_model('./weights/weights_tail_K/DenseNet201_tail_K_MAE'+str(loss)+'.hdf5')
    generator_for_test = generate_generator(test_paths_48, test_labels_48, test_K_48, len(test_labels_48), net=net, with_K=with_K, K_len=K_len)
    X_test, Y_test = generator_for_test.__next__()
    preds = np.squeeze(model.predict(X_test))
    fg, (ax0, ax1) = plt.subplots(1, 2, figsize=(20, 6))
    ax0.plot(Y_test, 'b')
    ax0.plot(preds, 'r')
    ax0.legend(['real_values', 'preds'])
    ax1.plot(np.abs(preds - Y_test))
    MAE = round(np.mean(abs(preds - Y_test)), 3)
    ax1.set_title('On test_48 set, MAE = {}'.format(MAE))
    print(np.unique(preds).shape)
    plt.show()
import pandas as pd
label_pred = np.hstack([np.asarray(range(Y_test.shape[0])).reshape(-1, 1), Y_test.reshape(-1, 1), preds.reshape(-1, 1)])
label_pred = pd.DataFrame(label_pred)
label_pred.to_csv(
    os.path.join('preds', 'preds_' + with_K + '_K', '_'.join([net, with_K, 'K_MAE', str(MAE) + '.txt'])),
    index=None,
    header=['index', 'label', 'prediction']
)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1596925704526046, 'fraction_numerical': 0.03313407344150299, 'mean_word_length': 6.674967234600262, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 8, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '602517', 'license': 'MIT', 'n_tokens_mistral': 2403, 'n_tokens_neox': 2264, 'n_words': 369, 'path': 'main_tail.ipynb', 'repo_name': 'ZhengPeng7/Temperature_regression', 'size': '5855'}