<jupyter_start><jupyter_text>Identify Dog BreedsThis notebook will be use to test different pre-trained architectures, fine-tunning and mainly to study how convolution neural nets will handle the data to make a prediction.  Create a folder to save models and other necessary files<jupyter_code>%rm -r saved_models
%mkdir saved_models<jupyter_output><empty_output><jupyter_text>ImportsHere I import all the stuff that I will use in this notebook<jupyter_code># Install gdown
!pip install gdown

import os # Iterate through the directories/files
import gdown # Download my trained models from GDrive
import numpy as np
import pandas as pd # Work with the labels.csv
import tensorflow as tf # Preprocess data, train and inference models
import tensorflow_hub as hub # Get the pretrained models
from tensorflow.keras.applications import MobileNet # MobileNetV1
from timeit import default_timer as timer # Compute the train time
from tqdm import tqdm # Fancy progress bars made easy 
from IPython.display import FileLink # Download files from kaggle<jupyter_output><empty_output><jupyter_text>Load DataFrameLoad all the CSV with the name of the images for training<jupyter_code>df = pd.read_csv("../input/dog-breed-identification/labels.csv")

df<jupyter_output><empty_output><jupyter_text>The id is the name of the images on `train` folder, perhaps they doesn't have the .jpg extension. Here I add it.<jupyter_code># Adicionar extensão às imagens
df["id"] = df["id"].apply(lambda x: str(x)+".jpg")<jupyter_output><empty_output><jupyter_text>Available ModelsAn easy way to have the models and the correct input sizes<jupyter_code># Dict with all the models from Tf Hub those I used on my tests
ALL_HUB_MODELS = {
    "InceptionV3": {
        "url": "https://tfhub.dev/google/imagenet/inception_v3/feature_vector/4",
        "size": 299
    },
    "EfficientNetB2":{
        "url": "https://tfhub.dev/google/efficientnet/b2/feature-vector/1",
        "size": 260
    },
    "EfficientNetB2_Trainable_Tf2":{
        "url": "https://tfhub.dev/tensorflow/efficientnet/b2/feature-vector/1",
        "size": 260
    },
    "EfficientNetB5":{
        "url": "https://tfhub.dev/google/efficientnet/b5/feature-vector/1",
        "size": 456
    },
    "EfficientNetB5_Trainable_Tf2":{
        "url": "https://tfhub.dev/tensorflow/efficientnet/b5/feature-vector/1",
        "size": 456
    },
    "MobileNetV2":{
        "url": "https://tfhub.dev/google/tf2-preview/mobilenet_v2/feature_vector/4",
        "size": 224
    },
}

ALL_TF_MODELS = {

    "MobileNet": {
        "model": MobileNet(input_shape=(224,224,3), include_top=False),
        "size": 224
    }
}<jupyter_output><empty_output><jupyter_text>ConstantsHere I define all the constants used on the notebook<jupyter_code># Choose the model to use
#CUR_MODEL = ALL_HUB_MODELS["EfficientNetB2_Trainable_Tf2"]
CUR_MODEL = ALL_TF_MODELS["MobileNet"]

# Base directory
BASE_DIR = os.path.abspath(os.path.dirname("."))

# A small way to tell the tensorflow that I want to use AUTOTUNE
AUTOTUNE = tf.data.experimental.AUTOTUNE

# Batch size for training and validation subset
BATCH_SIZE = 64

# Number of epochs
EPOCHS = 20

# Portion of train data used in validation
VAL_SPLIT = .30

# This will be used to know how many epochs remain
# after a break in training
EPOCHS_CHANGED = EPOCHS

# URL for Tf Hub model
#MODULE_HANDLE = CUR_MODEL["url"]
BASE_MODEL = CUR_MODEL["model"]

# Tuple with the "correct" size of the input (images)
# for the chosen model
IMG_SIZE=(CUR_MODEL["size"] , CUR_MODEL["size"] )


# Boolean to know if we already did the warmup step
WARMUP_DONE = False<jupyter_output><empty_output><jupyter_text>Classes and functions to help in my adventureHere will be all the function and classes that will helping me processing and handling data and models.<jupyter_code>class SaveModel(tf.keras.callbacks.Callback):
    """
        For each epoch tgis callback will called saving the model
        and writing the current epoch on a file

    """

    def __init__(self, warmup):
        super(SaveModel, self).__init__()

        self.warmup = warmup
    
    def on_epoch_end(self, epoch, logs=None):
        
        self.model.save("saved_models/warm_up.h5")

        with open("saved_models/epochs.txt", "a+") as f:
            
            # If it is the first epoch lets verify it is
            # a warmup or the real train
            if epoch == 0:
                if self.warmup:
                    f.write("Warmup:\n")

                elif not self.warmup:
                    f.write("Train:\n")
                    
            f.write(str(epoch+1)+"\n")


            
def write_2_file(filename, content):
    """
        Function that write text on a given file


        Parameters
        -----------
        filename: str
            Path to the file
        
        content: str
            Text to write on file
    """
    
    with open(filename, "a+") as f:
        f.write(content)
    
    return


def scheduler(epoch, lr):
    """
        Função que a cada 2 epochs diminui o learning rate
        para uma aprendizagem mais lenta
        Function that decrease the learning rate by a factor of 1/2
        with a 2 epochs step

        Parameters
        -----------
        epoch: int
            Current epoch
        
        lr: float
            Current learning rate

        Return
        --------
        return: float
            New learning rate
    """
    print(lr)

    if epoch % 10 == 0 and epoch != 0:
        lr *= 1./2
    
    return lr


def train(model, train_gen, steps, val_gen, val_steps, epochs, callback, warmup):

    """
        Train the model

        Parameters
        -----------
        model: Keras Model
            Model created with Keras
        
        train_gen: Keras ImageDataGenerator
            Object that will create new transformed images
            based on the original, to train
        
        steps: int
            Steps per epoch during train

        val_gen: Keras ImageDataGenerator
            Object that will create new transformed images
            based on the original, to validate
        
        val_steps: int
            Steps per epoch during validation

        epochs: int
            Number of epochs
        
        callback: Keras Callback
            Object with the callback
        
        warmup: bool
            Define if it is the warmup train or not
        

        Return
        --------
        return: dict
            Dict with all the metrics computed during the train
    """

    # List of callbacks, the SaveModel callback is always set
    callbacks = [SaveModel(warmup)]

    if callback is not None:
        callbacks.append(callback)

    # Train and save the computed metrics
    hist = model.fit(
        train_gen,
        epochs=epochs,
        steps_per_epoch=steps,
        validation_data=val_gen,
        validation_steps=val_steps,
        callbacks=callbacks
    )

    return hist<jupyter_output><empty_output><jupyter_text>DataLoaderLet's preprocess and prepare all the data for training<jupyter_code># Arguments for the data generator and data flow
# Here we define the portion of split data and rescale the RGB values
# from 0-255 to 0-1
datagen_kwargs = dict(rescale=1./255, validation_split=VAL_SPLIT)
dataflow_kwargs = dict(target_size=IMG_SIZE, batch_size=BATCH_SIZE,
                       interpolation="bilinear")
# Image Generator for validation (here we don't apply any transformation)
val_datagen = tf.keras.preprocessing.image.ImageDataGenerator(**datagen_kwargs)


# Image flow for validation (here we load and prepare our images from the DataFrame)
val_gen = val_datagen.flow_from_dataframe(
    df,
    directory="../input/dog-breed-identification/train",
    x_col="id",
    y_col="breed",
    subset="validation",
    save_format="jpg",
    shuffle=False,
    **dataflow_kwargs
)
# Image Generator for training (here we have DataAugmentation)
train_datagen = tf.keras.preprocessing.image.ImageDataGenerator(
    rotation_range=45,
    horizontal_flip=True,
    width_shift_range=0.4,
    height_shift_range=0.4,
    shear_range=0.4,
    zoom_range=0.4,
    **datagen_kwargs
)


# Image Flow for training (again we prepare and load the images for training)
train_gen = train_datagen.flow_from_dataframe(
    df,
    directory="../input/dog-breed-identification/train",
    x_col="id",
    y_col="breed",
    subset="training",
    shuffle=True,
    **dataflow_kwargs
)
#Verify if we want to use my custom decay learning rate callback
resp = ""
callback = None

while resp != "1" and resp != "2" and resp != "3":
    resp = input("No callback/scheduler [1/2]: ")

    if resp == "2":
        callback = tf.keras.callbacks.LearningRateScheduler(scheduler)

        
# Verify if we want to warm up before the real train
resp = ""

while resp != "y" and resp != "n":
    resp = input("Warm up or not [y/n]: ")<jupyter_output><empty_output><jupyter_text>Here we calculate the number of steps per epoch using the formula: $$\left \lfloor{\frac{number\ images}{batch\ size}}\right \rfloor$$<jupyter_code>steps_per_epoch = train_gen.samples // train_gen.batch_size
val_steps = val_gen.samples // train_gen.batch_size

steps_per_epoch<jupyter_output><empty_output><jupyter_text>Model creationLet's import our feature extractor model and add the new high level layers<jupyter_code># Main Model
model = tf.keras.Sequential([
    tf.keras.layers.InputLayer(input_shape=IMG_SIZE + (3,)),
    hub.KerasLayer(MODULE_HANDLE, trainable=False),
    tf.keras.layers.Dense(512),
    tf.keras.layers.Dropout(0.5),
    tf.keras.layers.Dense(len(train_gen.class_indices), activation="softmax")
])


model.compile(
    optimizer=tf.keras.optimizers.SGD(learning_rate=0.001),
    loss=tf.keras.losses.CategoricalCrossentropy(),
    metrics=["accuracy"]
)<jupyter_output><empty_output><jupyter_text>Train<jupyter_code># Verify if we want to train
isTrain = ""

while isTrain != "y" and isTrain != "n":
    isTrain = input("Do you want to train? [y/n]:")


if isTrain == "y":
    
    # Verify if we have selected to warmup
    if resp == "y":

        # VVerify if warmup was interrupted
        if os.path.isfile("saved_models/epochs.txt"):
            with open("saved_models/epochs.txt", "r") as f:

                lines = f.read().splitlines()

                if "Train:" in lines:
                    # Warmup Done
                    WARMUP_DONE = True

                # Update number of epochs
                last_epoch = lines[-1]
                EPOCHS_CHANGED = EPOCHS - int(last_epoch)


        print("WARMUP BEGUN!")

        # Start counter for compute execution time
        start = timer()

        # If the warmup are not done, let's continue the warmup
        if WARMUP_DONE == False:

            # If we have the model saved, load it
            if os.path.isfile("saved_models/warm_up.h5"):
                model = tf.keras.models.load_model("saved_models/warm_up.h5",
                                           custom_objects={"KerasLayer": hub.KerasLayer})

            # Train the model
            train(model, train_gen, steps_per_epoch, val_gen, val_steps, EPOCHS_CHANGED, callback, True)

            # Save the model
            model.save("saved_models/warm_up.h5")
        
        # End the timer
        end = timer()

        # Write the execution time 
        write_2_file("saved_models/timer.txt", "WarmUp: "+str(end-start))

        print("WARMUP ENDED\n")
        
        ## Start the Real Train
        
        # Load again the saved model
        model = tf.keras.models.load_model("saved_models/warm_up.h5",
                                           custom_objects={"KerasLayer": hub.KerasLayer})

        # All layers are trainable now
        model.trainable = True


        # Define an exponencial decay for decrease the learning rate for each
        # step
        """
        lr_scheduler = tf.keras.optimizers.schedules.ExponentialDecay(
            0.05,
            decay_steps=10000000,
            decay_rate=0.7,
            staircase=True
        )"""
        
        model.compile(
            optimizer=tf.keras.optimizers.SGD(learning_rate=0.01, momentum=0.9, nesterov=True),
            loss=tf.keras.losses.CategoricalCrossentropy(),
            metrics=["accuracy"]
        )

        print("MAIN TRAIN\n")
        
        # Start counter for compute execution time
        start = timer()

        # If the main train was interrupt we will calculate
        # the remain epochs to continue the train
        if WARMUP_DONE:
            EPOCHS = EPOCHS_CHANGED
        
        
        # Early stop after 5 epochs without improvements, at least
        # improvements of 0.001 decreases on validation loss
        callback = tf.keras.callbacks.EarlyStopping(monitor="val_loss", mode="min", patience=20, min_delta=0.001)
        
        # Train the model
        hist = train(model, train_gen, steps_per_epoch, val_gen, val_steps, EPOCHS*5, callback, False)
        
        # End the timer
        end = timer()

        # Save the execution time on the file
        write_2_file("saved_models/timer.txt", " Train: "+str(end-start))

        print("MAIN TRAIN ENDED\n")


    # Main Train only
    else:
        print("NORMAL TRAIN")
        
        # Start the timer for execution time
        start = timer()

        # Treinar the model
        hist = train(model, train_gen, steps_per_epoch, val_gen, val_steps, EPOCHS, None, False)
        
        # End the timer
        end = timer()

        print("NORMAL TRAIN ENDED")
if isTrain == "y":
    
    # Create a dict with the mean of each metric per epoch
    metrics ={

        "acc": np.mean( hist.history["accuracy"] ), 
        "val_acc": np.mean( hist.history["val_accuracy"] ),
        "loss": np.mean( hist.history["loss"] ),
        "val_loss": np.mean( hist.history["val_loss"] )
    }


    # Show each metric mean
    for k, v in metrics.items():
        print("{}: {:.3f} -".format(k, v), end=" ")

    # Show execution time
    print("\n\nExecution time: ", end-start)<jupyter_output><empty_output><jupyter_text>Download of the modelGenerate a link for direct download of the saved model in h5 format<jupyter_code>%cd /kaggle/working
FileLink(r'saved_models/warm_up.h5')<jupyter_output><empty_output><jupyter_text>EvaluateLet's prepare the .csv for submission<jupyter_code>## Optional (uncomment to use)
## If you have a .h5 model save on your Google Drive, just make it public, copy
## the id of it and past it here

#url = "https://drive.google.com/uc?id=your_id"
#gdown.download(url, "saved_models/warm_up.h5", quiet=False)<jupyter_output><empty_output><jupyter_text>Loading only the model will not work when we try to predict. To fix the problem I load the model save the weights, create a new model based on the inputs and outputs layers (they are connected to other intermediate layers) and load the weights to use in the new model<jupyter_code>model = tf.keras.models.load_model("saved_models/warm_up.h5",
                                       custom_objects={"KerasLayer": hub.KerasLayer})


model.save_weights("saved_models/cpkt")

model = tf.keras.Model(inputs=model.inputs, outputs=model.outputs)

model.load_weights("saved_models/cpkt")
# Path for directory with the test images
BASE_PATH = "../input/dog-breed-identification/test/"

# Get all file names (with extension now)
test_imgs = os.walk(BASE_PATH).__next__()[2]

# Create a list with all the possible classes
cols = list(train_gen.class_indices.keys())

# Append the column id to identify the image
cols = ["id"] + cols

# Now create a DataFrame with the cols
df_pred = pd.DataFrame(columns=cols)


# Infer the model with more than 10k images (take a coffee or maybe ... ten)
for test_img in tqdm(test_imgs):
    
    # Get the id
    name = test_img.split(".")[0]
    
    # Preprocess the image to be valid to pass into the model
    img = tf.keras.preprocessing.image.load_img(BASE_PATH+test_img, target_size=IMG_SIZE)
    img = tf.keras.preprocessing.image.img_to_array(img)
    img = tf.expand_dims(img, 0)
    rescale_layer = tf.keras.layers.experimental.preprocessing.Rescaling(1./255)
    img = rescale_layer(img)
    
    # Inference
    out = model.predict(img)[0]
    
    # Create a dict to fill with the computed probabilities
    # for each breed
    
    out_dict = {k:0 for k in cols}
    out_dict["id"] = name
    
    for i in range(len(out)):
        for k,v in train_gen.class_indices.items():
            if v == i:
                out_dict[k] = out[i]
    
    # Append the dict (row) to our DataFrame
    df_pred = df_pred.append(out_dict, ignore_index=True)
# Write to a .csv file to make the submission 
df_pred.to_csv("submission.csv", index=False)
%pip install kaggle

!kaggle competitions submit -c dog-breed-identification -f submission.csv -m " "<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.09745286921576739, 'fraction_numerical': 0.009573902251639973, 'mean_word_length': 2.6572293062459478, 'pattern_counts': {'":': 35, '<': 57, '<?xml version=': 0, '>': 57, 'https://': 7, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 1, 'word_list_counts': {'cursed_substrings.json': 10, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '14', 'dir': 'jupyter-structured-clean-dedup', 'id': '560980', 'license': 'MIT', 'n_tokens_mistral': 5364, 'n_tokens_neox': 5153, 'n_words': 1509, 'path': 'English/Dog Breed Identification/dog-breed-identification-with-tensorflow-keras.ipynb', 'repo_name': 'TigaxMT/Ground-Truth-vs-Prediction', 'size': '16921'}