<jupyter_start><jupyter_text>Import Library<jupyter_code>import os
os.environ["CUDA_VISIBLE_DEVICES"]="0"
import random
import numpy as np
import pandas as pd
import tensorflow as tf
import tensorflow_addons as tfa
import matplotlib.pyplot as plt
%matplotlib inline

from glob import glob
from tqdm import tqdm
from datetime import datetime

from tensorflow.keras.preprocessing.image import ImageDataGenerator
from tensorflow.keras.callbacks import ReduceLROnPlateau, ModelCheckpoint
from sklearn.model_selection import train_test_split
from tensorflow.keras import Model, Input
from tensorflow.keras.layers import Dense, Dropout, GlobalAveragePooling2D, Conv2D, Flatten
from tensorflow.keras.applications import MobileNetV2<jupyter_output><empty_output><jupyter_text>Set Seed<jupyter_code>SEED = 42

def seed_everything(seed):
    random.seed(seed)
    os.environ['PYTHONHASHSEED'] = str(seed)
    np.random.seed(seed)
    tf.random.set_seed(seed)

seed_everything(SEED)<jupyter_output><empty_output><jupyter_text>Config<jupyter_code>DATA_PATH = f'../Datasets/kfood/'
CKPT_PATH = f'./checkpoints/metric_learning_model.h5'
LOG_PATH = './logs/metric_model/' + datetime.now().strftime("%Y%n%d-%H%M%S")

size = 224
batch_size = 512

classes = 150

learning_rate = 1e-2
wd = 0.0005
max_lr = 1e-2
min_lr = 5e-5
cycle_len = 20

EPOCHS = 100
use_pretrained = True<jupyter_output><empty_output><jupyter_text>Load Data<jupyter_code># featurewise_center=True,
# featurewise_std_normalization=True,
# rotation_range=20,
# width_shift_range=0.2,
# height_shift_range=0.2,
# horizontal_flip=True,
# vertical_flip=True,
datagen = ImageDataGenerator(
    rescale=1./255.,
)

datagenerator = datagen.flow_from_directory(
    DATA_PATH,
    target_size=(size, size),
    batch_size=batch_size,
    class_mode='sparse',
    shuffle=True, 
)<jupyter_output>Found 150513 images belonging to 150 classes.<jupyter_text>Load Model<jupyter_code>base_model = MobileNetV2(
        input_shape=(size, size, 3),
        include_top=False,
        weights='imagenet',
    )
base_model.trainable = False

def MetricLearning(base_model):
    x = inputs = Input([size, size, 3])
    x = base_model(x)
    x = Conv2D(32, 3, padding='same', activation='relu')(x)
    x = Flatten()(x)
    x = Dense(512, activation='elu')(x)
    x = Dropout(0.2)(x)
    x = Dense(512, activation='elu')(x)
    x = Dropout(0.2)(x)
    output = Dense(256, activation=None)(x)
    return Model(inputs=inputs, outputs=output)

with tf.device('/device:GPU:0'):
    model = MetricLearning(base_model)
    if use_pretrained:
        model.load_weights(CKPT_PATH)
model.summary()<jupyter_output>Model: "functional_1"
_________________________________________________________________
Layer (type)                 Output Shape              Param #   
=================================================================
input_2 (InputLayer)         [(None, 224, 224, 3)]     0         
_________________________________________________________________
mobilenetv2_1.00_224 (Functi (None, 7, 7, 1280)        2257984   
_________________________________________________________________
conv2d (Conv2D)              (None, 7, 7, 32)          368672    
_________________________________________________________________
flatten (Flatten)            (None, 1568)              0         
_________________________________________________________________
dense (Dense)                (None, 512)               803328    
_________________________________________________________________
dropout (Dropout)            (None, 512)               0         
______________________________________________________[...]<jupyter_text>Compile<jupyter_code>optimizer = tf.keras.optimizers.SGD(learning_rate)

model.compile(
    optimizer=optimizer,
    loss=tfa.losses.TripletSemiHardLoss(
        distance_metric='angular'
    )
)
ckpt = tf.keras.callbacks.ModelCheckpoint(
    filepath=CKPT_PATH,
    save_weights_only=False,
    monitor='loss',
    mode='min',
    save_best_only=True,
    verbose=1
)

def trainfle_fn(x):
    return 1. / (2.**(x - 1))
clr_f = tfa.optimizers.CyclicalLearningRate(
    initial_learning_rate=(max_lr+min_lr)/2,
    maximal_learning_rate=max_lr,
    step_size=cycle_len,
    scale_fn=trainfle_fn
)
lr_scheduler = tf.keras.callbacks.LearningRateScheduler(clr_f)

early_stop = tf.keras.callbacks.EarlyStopping(
    monitor='loss',
    min_delta=0,
    patience=30,
    verbose=2,
    mode='auto',
    baseline=None,
    restore_best_weights=True
)

tensorboard = tf.keras.callbacks.TensorBoard(
    log_dir=LOG_PATH,
    histogram_freq=1
)

callbacks = [ckpt, lr_scheduler, early_stop, tensorboard]<jupyter_output><empty_output><jupyter_text>Training<jupyter_code>history = model.fit(
    datagenerator,
    epochs=EPOCHS,
    callbacks=callbacks
)<jupyter_output>Epoch 1/100
  1/294 [..............................] - ETA: 0s - loss: 0.9870WARNING:tensorflow:From /home/sungjin/anaconda3/envs/Pasta/lib/python3.7/site-packages/tensorflow/python/ops/summary_ops_v2.py:1277: stop (from tensorflow.python.eager.profiler) is deprecated and will be removed after 2020-07-01.
Instructions for updating:
use `tf.profiler.experimental.stop` instead.
  4/294 [..............................] - ETA: 5:55 - loss: 0.9875<jupyter_text>Classification Model<jupyter_code>metric_model = MetricLearning(base_model)
metric_model.load_weights(CKPT_PATH)
metric_model.trainable = False

def Classification(base_model):
    x = inputs = Input([size, size, 3])
    x = base_model(x)
    x = Dense(256, activation='relu')(x)
    x = Dropout(0.2)(x)
    output = Dense(classes, activation='softmax')(x)
    return Model(inputs=inputs, outputs=output)

with tf.device('/device:GPU:0'):
    c_model = Classification(metric_model)
c_model.summary()<jupyter_output>Model: "functional_17"
_________________________________________________________________
Layer (type)                 Output Shape              Param #   
=================================================================
input_10 (InputLayer)        [(None, 224, 224, 3)]     0         
_________________________________________________________________
functional_15 (Functional)   (None, 256)               3823968   
_________________________________________________________________
dense_21 (Dense)             (None, 256)               65792     
_________________________________________________________________
dropout_13 (Dropout)         (None, 256)               0         
_________________________________________________________________
dense_22 (Dense)             (None, 150)               38550     
=================================================================
Total params: 3,928,310
Trainable params: 104,342
Non-trainable params: 3,823,968
_____________________________________[...]<jupyter_text>Compile<jupyter_code>c_model.compile(
    optimizer=tfa.optimizers.AdamW(learning_rate, wd),
        loss=tf.keras.losses.SparseCategoricalCrossentropy(),
        metrics=['accuracy']
)
C_CKPT_PATH = f'./checkpoints/classification_model.h5'
C_LOG_PATH = './logs/classification_model/' + datetime.now().strftime("%Y%n%d-%H%M%S")
ckpt = tf.keras.callbacks.ModelCheckpoint(
    filepath=C_CKPT_PATH,
    save_weights_only=False,
    monitor='val_accuracy',
    mode='max',
    save_best_only=True,
    verbose=1
)

def trainfle_fn(x):
    return 1. / (2.**(x - 1))
clr_f = tfa.optimizers.CyclicalLearningRate(
    initial_learning_rate=(max_lr+min_lr)/2,
    maximal_learning_rate=max_lr,
    step_size=cycle_len,
    scale_fn=trainfle_fn
)
lr_scheduler = tf.keras.callbacks.LearningRateScheduler(clr_f)

early_stop = tf.keras.callbacks.EarlyStopping(
    monitor='val_accuracy',
    min_delta=0,
    patience=30,
    verbose=2,
    mode='auto',
    baseline=None,
    restore_best_weights=True
)

tensorboard = tf.keras.callbacks.TensorBoard(
    log_dir=LOG_PATH,
    histogram_freq=1
)

callbacks = [ckpt, lr_scheduler, early_stop, tensorboard]
c_datagen = ImageDataGenerator(
    rescale=1./255.,
    validation_split=0.2
)

c_train_datagenerator = c_datagen.flow_from_directory(
    DATA_PATH,
    target_size=(size, size),
    batch_size=batch_size,
    class_mode='sparse',
    shuffle=True,
    subset='training',
)

c_val_datagenerator = c_datagen.flow_from_directory(
    DATA_PATH,
    target_size=(size, size),
    batch_size=batch_size,
    class_mode='sparse',
    shuffle=True,
    subset='validation',
)
history = c_model.fit(
    c_train_datagenerator,
    epochs=EPOCHS,
    callbacks=callbacks,
    validation_data=c_val_datagenerator,
    verbose=1,
)<jupyter_output>Epoch 1/100
236/236 [==============================] - ETA: 0s - loss: 2.1121 - accuracy: 0.4223
Epoch 00001: val_accuracy improved from -inf to 0.45232, saving model to ./checkpoints/classification_model.h5
236/236 [==============================] - 588s 2s/step - loss: 2.1121 - accuracy: 0.4223 - val_loss: 1.9516 - val_accuracy: 0.4523
Epoch 2/100
236/236 [==============================] - ETA: 0s - loss: 2.0857 - accuracy: 0.4252
Epoch 00002: val_accuracy improved from 0.45232 to 0.46097, saving model to ./checkpoints/classification_model.h5
236/236 [==============================] - 587s 2s/step - loss: 2.0857 - accuracy: 0.4252 - val_loss: 1.9412 - val_accuracy: 0.4610
Epoch 3/100
236/236 [==============================] - ETA: 0s - loss: 2.1017 - accuracy: 0.4209
Epoch 00003: val_accuracy did not improve from 0.46097
236/236 [==============================] - 590s 2s/step - loss: 2.1017 - accuracy: 0.4209 - val_loss: 1.9806 - val_accuracy: 0.4496
Epoch 4/100
236/236 [============[...]
{'fraction_non_alphanumeric': 0.14994269042409086, 'fraction_numerical': 0.05032822757111598, 'mean_word_length': 4.293987865416437, 'pattern_counts': {'":': 0, '<': 32, '<?xml version=': 0, '>': 32, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '9', 'dir': 'jupyter-structured-clean-dedup', 'id': '311779', 'license': 'Apache-2.0', 'n_tokens_mistral': 3497, 'n_tokens_neox': 3108, 'n_words': 549, 'path': 'training.ipynb', 'repo_name': 'PaaS-TA-Contest-DietBox/FoodClassification', 'size': '9597'}