<jupyter_start><jupyter_text>Import + utilities<jupyter_code>import gc
import json
import tensorflow as tf

from tensorflow.keras.callbacks import EarlyStopping
from tensorflow.keras.models import Model
from tensorflow.keras.layers import (
    ZeroPadding2D,
    Dense,
    GlobalAveragePooling2D,
    AveragePooling2D,
    Input,
    Dropout,
)

from tensorflow_addons.optimizers import RectifiedAdam, Lookahead
from tensorflow_addons.activations import mish
from concurrent.futures import ProcessPoolExecutor as PoolExecutor

import pandas as pd
import numpy as np
from sklearn.metrics import roc_auc_score, accuracy_score

np.random.seed(0)

import matplotlib.pyplot as plt
from matplotlib.pyplot import imshow

%matplotlib inline
classnames = None
with (DATASET_FOLDER / "prep_data" / str(IMAGE_SIZE) / "classnames.json").open() as fp:
    classnames = np.array(json.load(fp))
OUTPUT_DIM = len(classnames)
LOSS = "binary_crossentropy" if OUTPUT_DIM == 2 else "categorical_crossentropy"
METRIC = "AUC" if OUTPUT_DIM == 2 else "accuracy"
def plot_metric(history, metric):
    # Plot training & validation loss values
    plt.plot(history.history[metric])
    plt.plot(history.history[f"val_{metric}"])
    plt.title(f"Model {metric}")
    plt.ylabel(f"{metric}")
    plt.xlabel("Epoch")
    plt.legend(["Train", "Test"], loc="upper left")
    plt.show()<jupyter_output><empty_output><jupyter_text>Preparing datasets<jupyter_code># https://www.tensorflow.org/tutorials/load_data/images
file_list = None
with (DATASET_FOLDER / "prep_data" / str(IMAGE_SIZE) / "file_list.json").open() as fp:
    file_list = json.load(fp)
file_list["train"][:10]
def build_process_path(classnames):
    def process_path(file_path):
        label = tf.strings.split(file_path, os.path.sep)[-2]
        label = label == classnames
        # load the raw data from the file as a string
        img = tf.io.read_file(file_path)
        img = tf.io.decode_compressed(img, "GZIP")
        # convert the compressed string to a 3D uint8 tensor
        img = tf.image.decode_jpeg(img, channels=NB_CHANNEL)
        return img, label

    return process_path
classnames
process_path = build_process_path(classnames)<jupyter_output><empty_output><jupyter_text>output_shapes=(tf.TensorShape((None, None, None)), tf.TensorShape((1, )))<jupyter_code>AUTOTUNE = tf.data.experimental.AUTOTUNE


def build_dataset(file_list, process_path, *, repeat, batch_size, prefetch):
    dataset = tf.data.Dataset.from_tensor_slices(file_list)
    if repeat:
        dataset = dataset.repeat()

    dataset = dataset.map(process_path, num_parallel_calls=AUTOTUNE)
    if batch_size is not None:
        dataset = dataset.batch(batch_size)

    if prefetch is not None:
        dataset = dataset.prefetch(prefetch)

    return dataset
dataset_train = build_dataset(
    file_list["train"],
    process_path,
    repeat=True,
    batch_size=BATCH_SIZE,
    prefetch=PREFETCH,
)
dataset_valid = build_dataset(
    file_list["valid"],
    process_path,
    repeat=True,
    batch_size=BATCH_SIZE,
    prefetch=PREFETCH,
)
dataset_test = build_dataset(
    file_list["test"], process_path, repeat=False, batch_size=BATCH_SIZE, prefetch=None
)
for image, label in dataset_train.take(1):
    print("Image shape: ", image.numpy().shape)
    print("Label: ", label.numpy().shape)<jupyter_output><empty_output><jupyter_text>Images Example<jupyter_code>def show_image(image, image_size, one_channel=False):
    if one_channel:
        imshow(image.reshape(IMAGE_SIZE, IMAGE_SIZE))
    else:
        imshow(image)
for image, label in dataset_train.take(1):
    print("Label: ", label[0].numpy())
    show_image(image[0].numpy(), IMAGE_SIZE, ONE_CHANNEL)
for image, label in dataset_valid.take(1):
    print("Label: ", label[0].numpy())
    show_image(image[0].numpy(), IMAGE_SIZE, ONE_CHANNEL)<jupyter_output><empty_output><jupyter_text>Training model (transfer learning)<jupyter_code>steps_per_epoch = np.ceil(len(file_list["train"]) / BATCH_SIZE)
steps_per_epoch_val = np.ceil(len(file_list["valid"]) / BATCH_SIZE)<jupyter_output><empty_output><jupyter_text>Creating model, using existing one<jupyter_code>activation = mish
optimizer = Lookahead(RectifiedAdam(), sync_period=6, slow_step_size=0.5)
nb_channel = 1 if ONE_CHANNEL else 3

# Base model have precise input shape
# In order to match, I do Pooling + Padding (can be asymetric) to match it

POOLING = int(np.ceil(IMAGE_SIZE / TARGET_SIZE))
PADDING = np.floor(TARGET_SIZE - np.floor(IMAGE_SIZE / POOLING))
PADDING_ASYM = int(PADDING % 2)

PADDING = int(np.floor(PADDING / 2))


# Let's divide the size to be smaller than input of base_model
# inputs = AveragePooling2D(pool_size=(POOLING, POOLING))(inputs)
# Let's pad to adapt to precise shape
# inputs = ZeroPadding2D(
#    padding=((PADDING, PADDING + PADDING_ASYM), (PADDING, PADDING + PADDING_ASYM),)
# )(inputs)

# This is the input of our new model
inputs = Input(shape=(IMAGE_SIZE, IMAGE_SIZE, nb_channel))
input_layer = inputs


# Now, we can use PRE TRAINED model
base_model = PRETRAINED_MODEL(
    input_tensor=inputs,
    input_shape=(TARGET_SIZE, TARGET_SIZE, nb_channel),
    weights=None,  # "imagenet",
    # weights="noisy-student",
    # weights=None,
    include_top=False,
)

x = (
    base_model.output
    if OUTPUT_LAYER is None or OUTPUT_LAYER >= -1
    else base_model.layers[OUTPUT_LAYER].output
)
# add a global spatial average pooling layer
x = GlobalAveragePooling2D()(x)
# let's add a fully-connected layer
# x = Dense(1024, activation=activation, kernel_initializer="he_normal")(x)
# x = Dense(512, activation=activation, kernel_initializer="he_normal")(x)

x = Dense(1024, activation=activation, kernel_initializer="he_normal")(x)
# x = Dropout(0.2)(x)
x = Dense(512, activation=activation, kernel_initializer="he_normal")(x)
# x = Dropout(0.2)(x)
x = Dense(128, activation=activation, kernel_initializer="he_normal")(x)
# x = Dropout(0.2)(x)


# x = Dense(256, activation=activation, kernel_initializer="he_normal")(x)
# x = Dropout(0.2)(x)
# x = Dense(128, activation=activation, kernel_initializer="he_normal")(x)
# x = Dropout(0.2)(x)
# and a logistic layer -- let's say we have 200 classes
predictions = Dense(OUTPUT_DIM, activation="softmax")(x)

# this is the model we will train
model = Model(inputs=input_layer, outputs=predictions)

# first: train only the top layers (which were randomly initialized)
# i.e. freeze all base_model layers
for layer in base_model.layers:
    layer.trainable = False

# compile the model (should be done *after* setting layers to non-trainable)
es = EarlyStopping(
    monitor="val_loss",
    verbose=1,
    mode="min",
    patience=patience_1,
    restore_best_weights=True,
)
# We need to recompile the model for these modifications to take effect
es.set_model(model)
model.compile(optimizer=optimizer, loss=LOSS)
model.summary()
# let's visualize layer names and layer indices to see how many layers
# we should freeze:
for i, layer in enumerate(model.layers):
    print(i, layer.name)<jupyter_output><empty_output><jupyter_text>Fit new layers<jupyter_code># train the model on the new data for a few epochs
history_1 = model.fit(
    dataset_train,
    callbacks=[es],
    epochs=epochs_1,
    steps_per_epoch=steps_per_epoch,
    validation_data=dataset_valid,
    validation_steps=steps_per_epoch_val,
)
# at this point, the top layers are well trained and we can start fine-tuning
# convolutional layers from inception V3. We will freeze the bottom N layers
# and train the remaining top layers.
plot_metric(history_1, "loss")<jupyter_output><empty_output><jupyter_text>truth_test = []for _, labels in dataset_test:    truth_test.append(np.argmax(labels, axis=1))truth_test = np.hstack(truth_test)truth_testtruth_valid = []for i, (_, labels) in enumerate(dataset_valid):    truth_valid.append(np.argmax(labels, axis=1))    if i >= steps_per_epoch_val - 1:        breaktruth_valid = np.hstack(truth_valid)truth_valid.shape<jupyter_code>preds_valid = model.predict(dataset_valid, steps=steps_per_epoch_val)
preds_valid.shape
if OUTPUT_DIM > 2:
    print(
        f"Accuracy valid: {accuracy_score(truth_valid, np.argmax(preds_valid, axis=1))}"
    )
if OUTPUT_DIM == 2:
    print(f"ROC AUC valid: {roc_auc_score(truth_valid, preds_valid[:, 1])}")
preds_test = model.predict(dataset_test)
preds_test.shape
if OUTPUT_DIM > 2:
    print(f"Accuracy test: {accuracy_score(truth_test, np.argmax(preds_test, axis=1))}")
if OUTPUT_DIM == 2:
    print(f"ROC AUC test: {roc_auc_score(truth_test, preds_test[:, 1])}")<jupyter_output><empty_output><jupyter_text>Unfreeze and fit more/all layers<jupyter_code># Let's freeze the whole model
for layer in model.layers:
    layer.trainable = False
# Now, we can unfreeze everything, or only some block
for layer in model.layers[FROM_LAYER_RETRAIN:]:
    layer.trainable = True
# Let's build an optimizer
optimizer = Lookahead(RectifiedAdam(), sync_period=6, slow_step_size=0.5)
es = EarlyStopping(
    monitor="val_loss",
    verbose=1,
    mode="min",
    patience=patience_2,
    restore_best_weights=True,
)
# We need to recompile the model for these modifications to take effect
es.set_model(model)
model.compile(optimizer=optimizer, loss=LOSS)
# we train our model again (this time fine-tuning the top 2 inception blocks
# alongside the top Dense layers
history_2 = model.fit(
    dataset_train,
    callbacks=[es],
    epochs=epochs_2,
    steps_per_epoch=steps_per_epoch,
    validation_data=dataset_valid,
    validation_steps=steps_per_epoch_val,
)
plot_metric(history_2, "loss")<jupyter_output><empty_output><jupyter_text>Evaluation<jupyter_code>preds_valid = model.predict(dataset_valid, steps=steps_per_epoch_val)
preds_valid.shape
if OUTPUT_DIM > 2:
    print(
        f"Accuracy valid: {accuracy_score(truth_valid, np.argmax(preds_valid, axis=1))}"
    )
if OUTPUT_DIM == 2:
    print(f"ROC AUC valid: {roc_auc_score(truth_valid, preds_valid[:, 1])}")
preds_test = model.predict(dataset_test)
preds_test.shape
if OUTPUT_DIM > 2:
    print(f"Accuracy test: {accuracy_score(truth_test, np.argmax(preds_test, axis=1))}")
if OUTPUT_DIM == 2:
    print(f"ROC AUC test: {roc_auc_score(truth_test, preds_test[:, 1])}")
# Open payments 13min, 0.9338321957865205 (batch 256)
# cat-in-the-dat-ii 0.7643929251726093 1h10 (batch 256)
# RL ROC AUC test: 0.9430279072306678 mobile net patience 20
# BNP cardif (image size : 96) ROC AUC test: 0.7168011573597879 1h16
# give-me-some-credit ROC AUC test: 0.8591709381962962 2h4
# Open payments 1h10
# ROC AUC valid: 0.9432323040315754
# ROC AUC test: 0.9345809216081824
# EfficientNetB0
# EfficientNetB2 2h => ROC AUC valid: 0.9357130861335854 ROC AUC test: 0.9330937133279599
# "santander-customer-satisfaction"
# 0.8164865598696714 => target size 96, whole re train
# 0.8141824599511267 => target size 224, whole re train
# ROC AUC valid: 0.8268400760249797 => 160 => 160, whole, batch 32, 1layer 1024 -> 512 -> 128, no dropout, earlystoping eevry time

# give me some credit
# 0.8462134942186483 => target size 160, whole train (batch 128, 2layers 1024, 128)

# ROC AUC valid: 0.8348917439829162
# RL
# ROC AUC valid: 0.892118469133795 => 160, whole, batch 64, 1layer 1024
# ROC AUC valid: 0.9233534348199217 => 160 => 160, whole, batch 32, 1layer 1024 -> 512 -> 128, no dropout
# ROC AUC valid: 0.9465346534653467 => 160 => 160, whole, batch 32, 1layer 1024 -> 512 -> 128, no dropout, earlystoping eevry time

# Open payment
# ROC AUC valid: 0.9186458210299415 => 96 =>160, whole, batch 64, 1layer 1024
# ROC AUC valid: 0.8778868370932499 => 96 => 96, whole, batch 64, 1layer 1024
# ROC AUC valid: 0.8917381493730192 => 96 => 224, whole, batch 64, 1layer 1024
# ROC AUC valid: 0.9090374872044725 => 96 => 96, whole, batch 64, 1layer 1024 -> 512 -> 128
# ROC AUC valid: 0.886417393797122 => 96 => 160, whole, batch 64, 1layer 1024 -> 512 -> 128
# ROC AUC valid: 0.9045819676568436 => 96 => 96, whole, batch 64, 1layer 1024 -> 128, no dropout
# ROC AUC valid: 0.9045819676568436 => 96 => 96, whole, batch 64, 1layer 1024 -> 128, no dropout
# ROC AUC valid: 0.9406275221953189 => 96 => 96, whole, batch 32, 1layer 1024 -> 512 -> 128, no dropout
# ROC AUC valid: 0.9399110034154216 => 160 => 160, whole, batch 32, 1layer 1024 -> 512 -> 128, no dropout, earlystoping eevry time
# ROC AUC valid: 0.8811034128677376 => 96 => 96, whole, batch 128, 1layer 1024 -> 512 -> 128, no dropout, earlystoping eevry time

# Bank marketing
# ROC AUC valid: 0.7970734141661526 => 160 => 160, whole, batch 32, 1layer 1024 -> 512 -> 128, no dropout

# Albert
# ROC AUC valid: 0.7500980687987842 => 288 => 160, whole, batch 32, 1layer 1024 -> 512 -> 128, no dropout

# bnp-cardif
# ROC AUC valid: 0.7206667869818926 => 384 => 96, whole, batch 32, 1layer 1024 -> 512 -> 128, no dropout

# "santander-customer-satisfaction"
# 0.833417731838137 => target size 96, whole train
# 0.8170226029745679 => target size 224, whole train
# ROC AUC test: 0.8348549041045967 => 160 => 160, whole, batch 32, 1layer 1024 -> 512 -> 128, no dropout, earlystoping eevry time

# give me some credit
# 0.8453497574694486 => target size 160, whole train (batch 128, 2layers 1024, 128)
# ROC AUC test: 0.8447058873195916

# RL
# ROC AUC test: 0.9051288159651395 => 160, whole, batch 64, 1layer 1024
# ROC AUC test: 0.9128674518211912 => 160 => 160, whole, batch 32, 1layer 1024 -> 512 -> 128, no dropout
# ROC AUC test: 0.9456874816987527 => 160 => 160, whole, batch 32, 1layer 1024 -> 512 -> 128, no dropout, earlystoping eevry time
# ROC AUC test: 0.9399110034154216 => 160 => 160, whole, batch 32, 1layer 1024 -> 512 -> 128, no dropout, earlystoping eevry time

# Open payment
# ROC AUC test: 0.916907759155486 => 96 =>160, whole, batch 64, 1layer 1024
# ROC AUC test: 0.8931101859362467 => 96 => 96, whole, batch 64, 1layer 1024
# ROC AUC test: 0.8938387451368033 => 96 => 224, whole, batch 64, 1layer 1024
# ROC AUC test: 0.9067445823812874 => 96 => 96, whole, batch 64, 1layer 1024 -> 512 -> 128
# ROC AUC test: 0.8954549557710788 => 96 => 160, whole, batch 64, 1layer 1024 -> 512 -> 128
# ROC AUC test: 0.9062895529860363 => 96 => 96, whole, batch 64, 1layer 1024 -> 128, no dropout
# ROC AUC test: 0.9403517762951931 => 96 => 96, whole, batch 32, 1layer 1024 -> 512 -> 128, no dropout

# Bank marketing
# ROC AUC valid: 0.7959000291791145 => 160 => 160, whole, batch 32, 1layer 1024 -> 512 -> 128, no dropout

# Albert
# ROC AUC test: 0.7487660412524685 => 288 => 160, whole, batch 32, 1layer 1024 -> 512 -> 128, no dropout

# bnp-cardif
# ROC AUC test: 0.725934546476426 => 384 => 96, whole, batch 32, 1layer 1024 -> 512 -> 128, no dropout<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.11640608998429712, 'fraction_numerical': 0.10309278350515463, 'mean_word_length': 4.821939586645469, 'pattern_counts': {'":': 0, '<': 41, '<?xml version=': 0, '>': 161, 'https://': 1, 'lorem ipsum': 0, 'www.': 1, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '10', 'dir': 'jupyter-structured-clean-dedup', 'id': '523378', 'license': 'MIT', 'n_tokens_mistral': 6373, 'n_tokens_neox': 5119, 'n_words': 1387, 'path': 'train-from-file-generic-clean.ipynb', 'repo_name': 'Hartorn/thc-net', 'size': '14647'}