<jupyter_start><jupyter_text>Putting It All Together: A Realistic ExampleIn this section we're going to work through a realistic example of a deep learning workflow. We'll be working with a smallish dataset featuring different kinds of flowers from Kaggle. We're going to apply data augmentation to synthetically expand the size of our dataset. And we'll attempt transfer learning using networks pretrained on the ImageNet dataset, which includes some flower species already.<jupyter_code># All of this should look familiar from previous notebooks:
import matplotlib.pyplot as plt
import numpy as np
import os
from PIL import Image, ImageOps

from keras.applications.mobilenet_v2 import MobileNetV2, preprocess_input
from keras.layers import Dense, GlobalAveragePooling2D, Dropout
from keras.models import Model, Sequential
from keras.preprocessing.image import ImageDataGenerator, image as keras_image
from keras.utils import to_categorical

# Our function to load an image from a path and fix it up, but 
# modified slightly to accomodate MobileNetV2.
def load_maintain_aspect_ratio(input_image_path, target_size):
    image = Image.open(input_image_path)
    width, height = image.size
    
    w_pad = 0
    h_pad = 0
    bonus_h_pad = 0
    bonus_w_pad = 0
    
    if width > height:
        pix_diff = (width - height)
        h_pad = pix_diff // 2
        bonus_h_pad = pix_diff % 2 # If the difference was odd, add one pixel on one side.
    elif height > width:
        pix_diff = (height - width)
        w_pad = pix_diff // 2
        bonus_w_pad = pix_diff % 2 # If the difference was odd, add one pixel on one side.
    # else: image is already square. Both pads stay 0
    
    image = ImageOps.expand(image, (w_pad, h_pad, w_pad+bonus_w_pad, h_pad+bonus_h_pad))
    image = image.resize(target_size)
    
    # Get the image data as a numpy array.
    image_data = np.array(image.getdata()).reshape(image.size[0], image.size[1], 3)
    
    # The preprocess function from MobileNetV2
    # It expects a numpy array with RGB values between 0-255
    return preprocess_input(image_data)


# Our recurring plot function.
def plot_training_history(history, model):
    figure = plt.figure()

    plt.subplot(1, 2, 1)
    plt.plot(history.history['accuracy'])
    plt.plot(history.history['val_accuracy'])
    plt.title('model accuracy')
    plt.ylabel('accuracy')
    plt.xlabel('epoch')
    plt.tight_layout()

    plt.subplot(1, 2, 2)
    plt.plot(history.history['loss'])
    plt.plot(history.history['val_loss'])
    plt.title('model loss')
    plt.ylabel('loss')
    plt.xlabel('epoch')
    plt.tight_layout()

    figure.tight_layout()
    plt.show()
# Some constants:
# 224x224 is MobileNetV2's default, so lets stick with it
image_size = 224
batch_size = 32
validation_split = 0.2

# Start small, for the sake of learning speed.
num_epochs = 5

# The dataset is too large to reasonably redistribute as part of this repository, so you will
# have to download it separately from: https://www.kaggle.com/alxmamaev/flowers-recognition/

# The download as a flowers folder, this variable should point to the 
# loccation of that folder. Inside that folder there should be 5 folders
# each named for the type of flower.
flower_dataset_directory = 'flowers_dataset/flowers/'

# The image classes
classes = {
    'daisy': 0, 
    'dandelion': 1, 
    'rose': 2,
    'sunflower': 3,
    'tulip': 4
}

# Process all of the images into an array
images = []
labels = []
for subdir in classes.keys():
    current_location = os.path.join(flower_dataset_directory, subdir)
    print(f'Processing {subdir}')
    sub_dir_count = 0
    for file in os.listdir(current_location):
        try:
            image = load_maintain_aspect_ratio(os.path.join(current_location, file), (image_size, image_size))
            images.append(image)
            labels.append(classes[subdir])
            sub_dir_count += 1
        except:
            print(f'Failed to load image: {subdir}/{file}. Ignored it.')
    
    print(f'Found {sub_dir_count} images of type {subdir}')

# Just double check.
assert len(images) == len(labels)

# This is a little bit crude, but we'll just randomly select each image/label pair
# to be in the validation set based on our validation split. We could take greater
# care here to ensure that the right amount are represented from each class
# but it will probably be okay...
x_train = []
y_train = []

x_validation = []
y_validation = []

for image, label in zip(images, labels):
    if np.random.random() > validation_split:
        x_train.append(image)
        y_train.append(label)
    else:
        x_validation.append(image)
        y_validation.append(label)

# Properly format the images into a np array
x_train = np.array(x_train)
x_validation = np.array(x_validation)

# Make the labels one-hot encoded:
y_train = to_categorical(y_train, len(classes))
y_validation = to_categorical(y_validation, len(classes))
            
print(f'Loaded {len(images)}')
print(f'Training size: {len(x_train)}, validation size: {len(x_validation)}')
# Lots of possible augmentation to the training data
# hopefully allowing us to avoid overfitting
train_generator = ImageDataGenerator(    
    rotation_range=360,
    width_shift_range=0.2,
    height_shift_range=0.2,
    shear_range=10, 
    zoom_range=0.5, 
    fill_mode='constant', 
    cval=0.0
)

# Don't transform the validation images.
validation_generator = ImageDataGenerator()

# Fit them both, best practice
train_generator.fit(x_train)
validation_generator.fit(x_validation)
# Lets do some sanity checking:
print(x_train.shape)
print(x_validation.shape)
print(y_train.shape)
print(y_validation.shape)

# View a couple, validations are never augmented
for _ in range(3):
    plt.imshow(next(validation_generator.flow(x_validation))[0])
    plt.show()
    
# But training data is
for _ in range(3):
    plt.imshow(next(train_generator.flow(x_train))[0])
    plt.show()
# Loading our pretrained mobilenet 
base_model = MobileNetV2(weights='imagenet', include_top=False, input_shape=(image_size, image_size, 3))

# Make a very simple new classifier
old_top = base_model.output
old_top = GlobalAveragePooling2D()(old_top)
new_top = Dense(len(classes), activation='softmax')(old_top)
model = Model(inputs=base_model.input, outputs=new_top)

# We have a small amount of data, but the data is pretty similar
# to imagenet, which does train on many flower images, so we can
# expect the existing weights to be pretty good. Freeze all. 
for layer in base_model.layers:
    layer.trainable = False

# Go for it!
model.compile(optimizer='adam', loss='categorical_crossentropy', metrics=['accuracy'])
history = model.fit_generator(train_generator.flow(x_train, y_train, batch_size=batch_size), 
                    steps_per_epoch=len(x_train) // batch_size, 
                    epochs=num_epochs,
                    validation_data=validation_generator.flow(x_validation, y_validation),
                    validation_steps=len(x_validation) // batch_size
)

plot_training_history(history, model)
# Things are going great! Lets unfreeze some of our model's layers and see if "forgetting" 
# some of the stuff our network learned about dogs, buildings, and waterbottles can
# improve our results further... 

# This number was chosen specifically for MobileNetV2, it is the 
# start of the 15th block. 
for layer in model.layers[134:]:
    layer.trainable = True

# Recompile to ensure the layers get set to trainable
model.compile(optimizer='adam', loss='categorical_crossentropy', metrics=['accuracy'])
history = model.fit_generator(train_generator.flow(x_train, y_train, batch_size=batch_size), 
                    steps_per_epoch=len(x_train) // batch_size, 
                    epochs=num_epochs,
                    validation_data=validation_generator.flow(x_validation, y_validation),
                    validation_steps=len(x_validation) // batch_size
)

plot_training_history(history, model)<jupyter_output>Epoch 1/5
109/109 [==============================] - 574s 5s/step - loss: 0.6596 - accuracy: 0.7687 - val_loss: 1.8947 - val_accuracy: 0.8221
Epoch 2/5
109/109 [==============================] - 326s 3s/step - loss: 0.4691 - accuracy: 0.8315 - val_loss: 5.8411 - val_accuracy: 0.6611
Epoch 3/5
109/109 [==============================] - 339s 3s/step - loss: 0.4343 - accuracy: 0.8422 - val_loss: 3.4338 - val_accuracy: 0.7368
Epoch 4/5
109/109 [==============================] - 334s 3s/step - loss: 0.4229 - accuracy: 0.8528 - val_loss: 5.5191 - val_accuracy: 0.7368
Epoch 5/5
109/109 [==============================] - 318s 3s/step - loss: 0.4541 - accuracy: 0.8476 - val_loss: 0.9193 - val_accuracy: 0.7728<jupyter_text>Right on, things still look okay. The dip in validation accuracy is definitely concerning though. After this maybe we could get an even bigger dataset and buy some cloud compute time to train the model for longer... But if we continue to see the validation accuracy decline (overfitting) then we NEED to try something to set it right. More data, more augmentation of the data would both be good ideas. We could also try an alternate model or adjust our classification layers — but that should probably be **in addition** to more data and more data.<jupyter_code># I ran this one overnight, just for fun, and to see if we started to overfit
history = model.fit_generator(train_generator.flow(x_train, y_train, batch_size=batch_size), 
                    steps_per_epoch=len(x_train) // batch_size, 
                    epochs=30,
                    validation_data=validation_generator.flow(x_validation, y_validation),
                    validation_steps=len(x_validation) // batch_size
)

plot_training_history(history, model)<jupyter_output>Epoch 1/30
109/109 [==============================] - 334s 3s/step - loss: 0.3392 - accuracy: 0.8768 - val_loss: 0.8876 - val_accuracy: 0.7200
Epoch 2/30
109/109 [==============================] - 316s 3s/step - loss: 0.3245 - accuracy: 0.8839 - val_loss: 2.4309 - val_accuracy: 0.7067
Epoch 3/30
109/109 [==============================] - 309s 3s/step - loss: 0.3548 - accuracy: 0.8802 - val_loss: 1.2468 - val_accuracy: 0.7885
Epoch 4/30
109/109 [==============================] - 336s 3s/step - loss: 0.3184 - accuracy: 0.8907 - val_loss: 3.0005 - val_accuracy: 0.6827
Epoch 5/30
109/109 [==============================] - 301s 3s/step - loss: 0.2973 - accuracy: 0.8965 - val_loss: 1.0696 - val_accuracy: 0.7656
Epoch 6/30
109/109 [==============================] - 287s 3s/step - loss: 0.2773 - accuracy: 0.9060 - val_loss: 0.4120 - val_accuracy: 0.8269
Epoch 7/30
109/109 [==============================] - 302s 3s/step - loss: 0.2667 - accuracy: 0.9069 - val_loss: 0.9461 - val_accuracy: 0.8209[...]
{'fraction_non_alphanumeric': 0.12611524163568774, 'fraction_numerical': 0.04358736059479554, 'mean_word_length': 3.761504424778761, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 10, 'https://': 1, 'lorem ipsum': 0, 'www.': 1, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '144023', 'license': 'Unlicense', 'n_tokens_mistral': 3725, 'n_tokens_neox': 3327, 'n_words': 1035, 'path': '04-convolutional-neural-networks/04-a-realistic-CNN-pipeline.ipynb', 'repo_name': 'rekil156/intro-to-deep-learning', 'size': '10760'}