<jupyter_start><jupyter_text>CNN implementation from scratchIn this notebook we try to define and train a standard CNN from scratch, evaluating then the results on the test set  Dataset downloadIn this section the dataset is downloaded from *Kaggle*, unzipped and well formatted.<jupyter_code>import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import json
import os
from tqdm import tqdm, tqdm_notebook
import random

import tensorflow as tf
from tensorflow.keras.models import Sequential, Model
from tensorflow.keras.layers import *
from tensorflow.keras.optimizers import *
from tensorflow.keras.applications import *
from tensorflow.keras.callbacks import *
from tensorflow.keras.initializers import *
from tensorflow.keras.preprocessing.image import ImageDataGenerator

from numpy.random import seed
import pickle
seed(1)
tf.random.set_seed(1)
! pip install -q kaggle

from google.colab import files
_ = files.upload()

! mkdir -p ~/.kaggle
! cp kaggle.json ~/.kaggle/
! chmod 600 ~/.kaggle/kaggle.json
! kaggle datasets download -d ikarus777/best-artworks-of-all-time
! unzip best-artworks-of-all-time.zip<jupyter_output>[1;30;43mOutput streaming troncato alle ultime 5000 righe.[0m
  inflating: resized/resized/Henri_Rousseau_61.jpg  
  inflating: resized/resized/Henri_Rousseau_62.jpg  
  inflating: resized/resized/Henri_Rousseau_63.jpg  
  inflating: resized/resized/Henri_Rousseau_64.jpg  
  inflating: resized/resized/Henri_Rousseau_65.jpg  
  inflating: resized/resized/Henri_Rousseau_66.jpg  
  inflating: resized/resized/Henri_Rousseau_67.jpg  
  inflating: resized/resized/Henri_Rousseau_68.jpg  
  inflating: resized/resized/Henri_Rousseau_69.jpg  
  inflating: resized/resized/Henri_Rousseau_7.jpg  
  inflating: resized/resized/Henri_Rousseau_70.jpg  
  inflating: resized/resized/Henri_Rousseau_8.jpg  
  inflating: resized/resized/Henri_Rousseau_9.jpg  
  inflating: resized/resized/Henri_de_Toulouse-Lautrec_1.jpg  
  inflating: resized/resized/Henri_de_Toulouse-Lautrec_10.jpg  
  inflating: resized/resized/Henri_de_Toulouse-Lautrec_11.jpg  
  inflating: resized/resized/Henri_de_Toulouse-Lautrec_12.j[...]<jupyter_text>Data preprocessing<jupyter_code>artists = pd.read_csv('artists.csv')
# Sort artists by number of paintings
artists = artists.sort_values(by=['paintings'], ascending=False)

# Create a dataframe with artists having more than 200 paintings
artists_top = artists[artists['paintings'] >= 200].reset_index()
artists_top = artists_top[['name', 'paintings']]
artists_top['class_weight'] = artists_top.paintings.sum() / (artists_top.shape[0] * artists_top.paintings)
#artists_top = artists_top.loc[artists_top['paintings'] >200]
artists_top
# Set class weights - assign higher weights to underrepresented classes
class_weights = artists_top['class_weight'].to_dict()
class_weights<jupyter_output><empty_output><jupyter_text>Next step is to solve a problem: the downloaded dataset present two directory containing the same paintings but they have different name. What we do in the next cells is to create a single directory called 'Albrecht_Durer' containing one copy of each painting, and then we delete all unuseful directories.<jupyter_code>updated_name = "Albrecht_Dürer".replace("_", " ")
artists_top.iloc[4, 0] = updated_name
images_dir = 'images/images'
artists_dirs = os.listdir(images_dir)
artists_top_name = artists_top['name'].str.replace(' ', '_').values
! mv '/content/images/images/Albrecht_Du╠Иrer' /content/images/images/Albrecht_Dürer
! rm -R '/content/images/images/Albrecht_DuтХа├кrer' 
! rm -R '/content/resized'<jupyter_output><empty_output><jupyter_text>Dataset splitting: train, test and validation Constants:<jupyter_code>import pathlib
import os

IMAGE_DIR = '/content/images/'
TRAIN_DIR = pathlib.Path(os.path.join(IMAGE_DIR, 'train'))
TEST_DIR = pathlib.Path(os.path.join(IMAGE_DIR, 'test'))

IMAGE_HEIGHT = 224
IMAGE_WIDTH = 224
BATCH_SIZE = 128
RANDOM_SEED = 42
VALIDATION_SPLIT = 0.10
EPOCHS=15<jupyter_output><empty_output><jupyter_text>The directories are moved and transformed in order to obtain the train and the test directories:<jupyter_code>import numpy as np
import shutil

rootdir= '/content/images/images' #path of the original folder
classes = os.listdir(rootdir)

for i, c in enumerate(classes, start=1):
  if c not in artists_top_name.tolist():
    shutil.rmtree(rootdir + '/' + c)
    continue
  if not os.path.exists(rootdir + '/train/' + c):
    os.makedirs(rootdir + '/train/' + c)
  if not os.path.exists(rootdir + '/test/' + c):  
    os.makedirs(rootdir + '/test/' + c)

  source = os.path.join(rootdir, c)
  allFileNames = os.listdir(source)

  np.random.shuffle(allFileNames)

  test_ratio = 0.10
  train_FileNames, test_FileNames = np.split(np.array(allFileNames),
                                                        [int(len(allFileNames)* (1 - test_ratio))])

  train_FileNames = [source+'/'+ name for name in train_FileNames.tolist()]
  test_FileNames = [source+'/' + name for name in test_FileNames.tolist()]

  for name in train_FileNames:
    shutil.copy(name, rootdir +'/train/' + c)

  for name in test_FileNames:
    shutil.copy(name, rootdir +'/test/' + c)
! mv /content/images/images/train /content/images
! mv /content/images/images/test /content/images
! rm -r /content/images/images
import tensorflow as tf

training_images = tf.keras.preprocessing.image_dataset_from_directory(
    TRAIN_DIR, labels='inferred', label_mode='categorical',
    class_names=None, color_mode='rgb', batch_size=BATCH_SIZE, image_size=(IMAGE_HEIGHT,
    IMAGE_WIDTH), shuffle=True, seed=RANDOM_SEED, validation_split=VALIDATION_SPLIT, subset='training',
    interpolation='bilinear', follow_links=False
)

val_images = tf.keras.preprocessing.image_dataset_from_directory(
    TRAIN_DIR, labels='inferred', label_mode='categorical',
    class_names=None, color_mode='rgb', batch_size=BATCH_SIZE, image_size=(IMAGE_HEIGHT,
    IMAGE_WIDTH), shuffle=True, seed=RANDOM_SEED, validation_split=VALIDATION_SPLIT, subset='validation',
    interpolation='bilinear', follow_links=False
)

test_images = tf.keras.preprocessing.image_dataset_from_directory(
    TEST_DIR, labels='inferred', label_mode='categorical',
    class_names=None, color_mode='rgb', batch_size=BATCH_SIZE, image_size=(IMAGE_HEIGHT,
    IMAGE_WIDTH), shuffle=True, seed=RANDOM_SEED, interpolation='bilinear', follow_links=False
)
NUM_CLASSES = len(training_images.class_names)<jupyter_output><empty_output><jupyter_text>CNN definition<jupyter_code>import tensorflow as tf
from tensorflow import keras as ks
from tensorflow.keras import layers
import pathlib
class standardCNN:

    def __init__(self, train=None, test=None, val=None, classes=11, name='standardCNN'):
        # set datasets
        self.training_set = train
        self.val_set = val
        self.test_set = test
        self.name=name 
        
        # model
        inputs = ks.Input(shape=(224, 224, 3))
        x = layers.Rescaling(1. / 255)(inputs)
        x = layers.Conv2D(filters=32, kernel_size=(3, 3), padding="same", activation="relu")(x)
        x = layers.MaxPooling2D(pool_size=(4, 4), strides=4)(x)
        x = layers.Conv2D(filters=64, kernel_size=(3, 3), padding="same", activation="relu")(x)
        x = layers.MaxPooling2D(pool_size=(2, 2))(x)
        x = layers.Conv2D(filters=128, kernel_size=(3, 3), padding="same", activation="relu")(x)
        x = layers.Flatten()(x)
        outputs = layers.Dense(classes, activation='softmax', name='predictions')(x)
        self.model = ks.Model(inputs=inputs, outputs=outputs)

        # utils
        self.history = None

        self.dir_name ="/content/drive/Shareddrives/CIDL/project/models/CNN"
        if not os.path.exists(self.dir_name):
          os.makedirs(self.dir_name)
        self.save_path = os.path.join(self.dir_name, name+'.h5')

        self.callbacks_list = [
                  ks.callbacks.ModelCheckpoint(
                      filepath=self.save_path,
                      monitor="val_loss",
                      verbose=1,
                      save_best_only=True) #saves the best model in terms of the metric in monitor
                  ]

    def compile_and_fit(self, optimizer='adam', loss='categorical_crossentropy', learning_rate=None, epochs=EPOCHS):
        if learning_rate is not None:
            optimizer.learning_rate = learning_rate

        self.model.compile(
            optimizer=optimizer,
            loss=loss,
            metrics=['accuracy']
        )

        self.history = self.model.fit(
             self.training_set,
             epochs=epochs,
             validation_data=self.val_set,
             callbacks=self.callbacks_list
        )   

    def plot_model(self, model_name):
        ks.utils.plot_model(self.model, to_file=str(model_name) +".jpg", show_shapes=True)
    
    def train_more(self, epochs=EPOCHS):
      self.new_history = self.model.fit(
             self.training_set,
             epochs=epochs,
             validation_data=self.val_set,
             callbacks=self.callbacks_list
        )
      for key in self.history.history.keys():
        self.history.history[key].extend(self.new_history.history[key])

    def load_best_model(self):
      self.model = ks.models.load_model(self.save_path)

    def save_history(self):
      with open(os.path.join(self.dir_name, self.name), 'wb') as file_pi:
        pickle.dump(self.history.history, file_pi)

    def load_history(self):
      with open(os.path.join(self.dir_name, self.name), 'rb') as file_pi:
        history = pickle.load(file_pi)
      return history  

    def evaluate(self, test=None):
      if test == None:
        test = self.test_set
      test_loss, test_acc = self.model.evaluate(self.test_set)
      print(f"Test accuracy: {test_acc:.3f}, test loss: {test_loss:.3f}")

    def summary(self):
        self.model.summary()
cnn_model = standardCNN(train=training_images, val=val_images, test=test_images, classes=NUM_CLASSES)
cnn_model.summary()<jupyter_output>Model: "model"
_________________________________________________________________
 Layer (type)                Output Shape              Param #   
=================================================================
 input_1 (InputLayer)        [(None, 224, 224, 3)]     0         
                                                                 
 rescaling (Rescaling)       (None, 224, 224, 3)       0         
                                                                 
 conv2d (Conv2D)             (None, 224, 224, 32)      896       
                                                                 
 max_pooling2d (MaxPooling2D  (None, 56, 56, 32)       0         
 )                                                               
                                                                 
 conv2d_1 (Conv2D)           (None, 56, 56, 64)        18496     
                                                                 
 max_pooling2d_1 (MaxPooling  (None, 28, 28, 64)       0     [...]<jupyter_text>Model representation<jupyter_code>cnn_model.plot_model(model_name="standardCNN")<jupyter_output><empty_output><jupyter_text>Sanity check: overfit a bunch of dataIf our network is not able to overfit a reduced amount of data, it will never be able to fit the training data<jupyter_code>training_images_reduced = tf.keras.preprocessing.image_dataset_from_directory(
    TRAIN_DIR, labels='inferred', label_mode='categorical',
    class_names=None, color_mode='rgb', batch_size=BATCH_SIZE, image_size=(IMAGE_HEIGHT,
    IMAGE_WIDTH), shuffle=True, seed=RANDOM_SEED, validation_split=0.8, subset='training',
    interpolation='bilinear', follow_links=False
)
cnn_model_to_overfit = standardCNN(train=training_images_reduced, val=val_images, test=test_images, classes=NUM_CLASSES)
cnn_model_to_overfit.compile_and_fit()<jupyter_output>Epoch 1/20
6/6 [==============================] - ETA: 0s - loss: 2.5260 - accuracy: 0.1877
Epoch 00001: val_loss improved from inf to 2.18354, saving model to /content/drive/Shareddrives/CIDL/project/models/CNN/standardCNN.h5
6/6 [==============================] - 21s 1s/step - loss: 2.5260 - accuracy: 0.1877 - val_loss: 2.1835 - val_accuracy: 0.2725
Epoch 2/20
6/6 [==============================] - ETA: 0s - loss: 2.0772 - accuracy: 0.2857
Epoch 00002: val_loss improved from 2.18354 to 2.04322, saving model to /content/drive/Shareddrives/CIDL/project/models/CNN/standardCNN.h5
6/6 [==============================] - 8s 636ms/step - loss: 2.0772 - accuracy: 0.2857 - val_loss: 2.0432 - val_accuracy: 0.3146
Epoch 3/20
6/6 [==============================] - ETA: 0s - loss: 1.8812 - accuracy: 0.3669
Epoch 00003: val_loss improved from 2.04322 to 1.83089, saving model to /content/drive/Shareddrives/CIDL/project/models/CNN/standardCNN.h5
6/6 [==============================] - 7s 629ms/step - [...]<jupyter_text>Train network on the whole dataset<jupyter_code>cnn_model.compile_and_fit()
cnn_model.save_history()<jupyter_output><empty_output><jupyter_text>Visualize results In this section, we compare the results of the 3 tested classifier analyzing their behavior in terms of accuracy<jupyter_code>history = standardCNN().load_history()
name='standard CNN'

acc = history['accuracy']
val_acc = history['val_accuracy']
loss = history['loss']
val_loss = history['val_loss']
best_loss_epoch = np.argmin(val_loss)
best_loss = np.min(val_loss)
best_acc = val_acc[best_loss_epoch]

epochs = range(len(acc))

plt.plot(epochs, acc, 'b.', label='Training accuracy')
plt.plot(epochs, val_acc, 'g.', label='Validation accuracy')
plt.title('Training and validation accuracy on '+name)
plt.legend(loc='upper right')

plt.figure()

plt.plot(epochs, loss, 'b', label='Training loss')
plt.plot(epochs, val_loss, 'g', label='Validation loss')
plt.plot(best_loss_epoch, best_loss, 'ro', label='Selected model')
plt.title('Training and validation loss on '+name)
plt.legend(loc='upper right')

plt.show()
print(f"Lowest validation loss has been reached at epoch {best_loss_epoch:d} with validation accuracy of {best_acc:.3f}")<jupyter_output><empty_output><jupyter_text>Evaluate against test set Once trained our classifier, we pick the most performing one in the validation set and we test it against the test set<jupyter_code>cnn_model.load_best_model()
cnn_model.evaluate(test_images)<jupyter_output>4/4 [==============================] - 3s 77ms/step - loss: 0.6542 - accuracy: 0.7861
Test accuracy: 0.786, test loss: 0.654<jupyter_text>Considerations The network reaches 57% of validation accuracy, but then overfits quickly. This could be caused mainly by 3 factors:*   Lack of data*   The spatial extent of the last feature map is still quite large, so the fully connected layer has a high number of parameters to train, about 50 times the available raw training images*   Lack of regularization techniques  Deeper network and dropout Here we try to deepen the network, in order to reduce the dimensionality of the feature vector. The flattened vector is then passed to a fully connexted network that exploits Dropout.  Model definition<jupyter_code>class DropoutCNN(standardCNN):
  def __init__(self, train=None, test=None, val=None, classes=11, name='DropoutCNN'):
    super().__init__(train, test, val, classes, name)
    
    inputs = ks.Input(shape=(224, 224, 3))
    x = layers.Rescaling(1. / 255)(inputs)
    x = layers.Conv2D(filters=32, kernel_size=(3, 3), padding="same", activation="relu")(x)
    x = layers.MaxPooling2D(pool_size=(4, 4), strides=4)(x)
    x = layers.Conv2D(filters=64, kernel_size=(3, 3), padding="same", activation="relu")(x)
    x = layers.MaxPooling2D(pool_size=(2, 2))(x)
    x = layers.Conv2D(filters=128, kernel_size=(3, 3), padding="same", activation="relu")(x)
    x = layers.MaxPooling2D(pool_size=(2, 2))(x)
    x = layers.Conv2D(filters=256, kernel_size=(3, 3), padding="same", activation="relu")(x)
    x = layers.MaxPooling2D(pool_size=(2, 2))(x)
    x = layers.Flatten()(x)
    x = layers.Dense(32, activation='relu')(x)
    x = layers.Dropout(0.2)(x)
    outputs = layers.Dense(classes, activation='softmax', name='predictions')(x)
    self.model = ks.Model(inputs=inputs, outputs=outputs)

dropout_model = DropoutCNN(train=training_images, val=val_images, test=test_images, classes=NUM_CLASSES)
dropout_model.plot_model(model_name="DropoutCNN")
dropout_model.summary()<jupyter_output>Model: "model_1"
_________________________________________________________________
 Layer (type)                Output Shape              Param #   
=================================================================
 input_2 (InputLayer)        [(None, 224, 224, 3)]     0         
                                                                 
 rescaling_1 (Rescaling)     (None, 224, 224, 3)       0         
                                                                 
 conv2d_3 (Conv2D)           (None, 224, 224, 32)      896       
                                                                 
 max_pooling2d_2 (MaxPooling  (None, 56, 56, 32)       0         
 2D)                                                             
                                                                 
 conv2d_4 (Conv2D)           (None, 56, 56, 64)        18496     
                                                                 
 max_pooling2d_3 (MaxPooling  (None, 28, 28, 64)       0   [...]<jupyter_text>Train network on the whole dataset<jupyter_code>dropout_model.compile_and_fit()
dropout_model.train_more(epochs=5)
dropout_model.save_history()<jupyter_output><empty_output><jupyter_text>Visualize results In this section, we compare the results of the 3 tested classifier analyzing their behavior in terms of accuracy<jupyter_code>history = DropoutCNN().load_history()
name='standard CNN with dropout'

acc = history['accuracy']
val_acc = history['val_accuracy']
loss = history['loss']
val_loss = history['val_loss']
best_loss_epoch = np.argmin(val_loss)
best_loss = np.min(val_loss)
best_acc = val_acc[best_loss_epoch]

epochs = range(len(acc))

plt.plot(epochs, acc, 'b.', label='Training accuracy')
plt.plot(epochs, val_acc, 'g.', label='Validation accuracy')
plt.title('Training and validation accuracy on '+name)
plt.legend(loc='upper right')

plt.figure()

plt.plot(epochs, loss, 'b', label='Training loss')
plt.plot(epochs, val_loss, 'g', label='Validation loss')
plt.plot(best_loss_epoch, best_loss, 'ro', label='Selected model')
plt.title('Training and validation loss on '+name)
plt.legend(loc='upper right')

plt.show()
print(f"Lowest validation loss has been reached at epoch {best_loss_epoch:d} with validation accuracy of {best_acc:.3f}")<jupyter_output><empty_output><jupyter_text>Evaluate against test set Once trained our classifier, we pick the most performing one in the validation set and we test it against the test set<jupyter_code>dropout_model.load_best_model()
dropout_model.evaluate(test_images)<jupyter_output>4/4 [==============================] - 5s 103ms/step - loss: 1.1516 - accuracy: 0.6244
Test accuracy: 0.624, test loss: 1.152<jupyter_text>Considerations The regularization given by the dropout enhanced the validation accuracy, however the reduction of parameters made the network struggling in learning from the training set, especially at the beginning  Bigger network and early stopping In this block we try to increase the network size, in order to verify if it is possible to increase validation accuracy. We also equip the network with an early stopping mechanism, in order to block training if no improvement is detected  Model definition<jupyter_code>class BiggerCNN(standardCNN):
  def __init__(self, train=None, test=None, val=None, classes=11, name='BiggerCNN'):
    super().__init__(train, test, val, classes, name)
    inputs = ks.Input(shape=(224, 224, 3))
    x = layers.Rescaling(1. / 255)(inputs)
    x = layers.Conv2D(filters=32, kernel_size=(3, 3), padding="same", activation="relu")(x)
    x = layers.MaxPooling2D(pool_size=(2, 2), strides=4)(x)
    x = layers.Conv2D(filters=64, kernel_size=(3, 3), padding="same", activation="relu")(x)
    x = layers.MaxPooling2D(pool_size=(2, 2), strides=4)(x)
    x = layers.Conv2D(filters=128, kernel_size=(3, 3), padding="same", activation="relu")(x)
    x = layers.MaxPooling2D(pool_size=(2, 2))(x)
    x = layers.Conv2D(filters=256, kernel_size=(3, 3), padding="same", activation="relu")(x)
    x = layers.Flatten()(x)
    x = layers.Dense(64, activation='relu')(x)
    x = layers.Dropout(0.2)(x)
    outputs = layers.Dense(classes, activation='softmax', name='predictions')(x)
    self.model = ks.Model(inputs=inputs, outputs=outputs)

    self.callbacks_list.append(ks.callbacks.EarlyStopping(monitor='val_loss', patience=10, restore_best_weights=True))

bigger_model = BiggerCNN(train=training_images, val=val_images, test=test_images, classes=NUM_CLASSES)
bigger_model.plot_model(model_name="BiggerCNN")
bigger_model.summary()<jupyter_output>Model: "model_4"
_________________________________________________________________
 Layer (type)                Output Shape              Param #   
=================================================================
 input_5 (InputLayer)        [(None, 224, 224, 3)]     0         
                                                                 
 rescaling_4 (Rescaling)     (None, 224, 224, 3)       0         
                                                                 
 conv2d_13 (Conv2D)          (None, 224, 224, 32)      896       
                                                                 
 max_pooling2d_10 (MaxPoolin  (None, 56, 56, 32)       0         
 g2D)                                                            
                                                                 
 conv2d_14 (Conv2D)          (None, 56, 56, 64)        18496     
                                                                 
 max_pooling2d_11 (MaxPoolin  (None, 14, 14, 64)       0   [...]<jupyter_text>Train network on the whole dataset<jupyter_code>bigger_model.compile_and_fit(epochs = 100)
bigger_model.save_history()<jupyter_output><empty_output><jupyter_text>Visualize results In this section, we compare the results of the 3 tested classifier analyzing their behavior in terms of accuracy<jupyter_code>history = BiggerCNN().load_history()
name='Bigger CNN with dropout'

acc = history['accuracy']
val_acc = history['val_accuracy']
loss = history['loss']
val_loss = history['val_loss']
best_loss_epoch = np.argmin(val_loss)
best_loss = np.min(val_loss)
best_acc = val_acc[best_loss_epoch]

epochs = range(len(acc))

plt.plot(epochs, acc, 'b.', label='Training accuracy')
plt.plot(epochs, val_acc, 'g.', label='Validation accuracy')
plt.title('Training and validation accuracy on '+name)
plt.legend(loc='upper right')

plt.figure()

plt.plot(epochs, loss, 'b', label='Training loss')
plt.plot(epochs, val_loss, 'g', label='Validation loss')
plt.plot(best_loss_epoch, best_loss, 'ro', label='Selected model')
plt.title('Training and validation loss on '+name)
plt.legend(loc='upper right')

plt.show()
print(f"Lowest validation loss has been reached at epoch {best_loss_epoch:d} with validation accuracy of {best_acc:.3f}")<jupyter_output><empty_output><jupyter_text>Evaluate against test set Once trained our classifier, we pick the most performing one in the validation set and we test it against the test set<jupyter_code>bigger_model.load_best_model()
bigger_model.evaluate(test_images)<jupyter_output>4/4 [==============================] - 3s 92ms/step - loss: 1.1742 - accuracy: 0.6443
Test accuracy: 0.644, test loss: 1.174<jupyter_text>Considerations Combining the benefit of dropout and of improved capacity, the network is capable to perform better both on test and on validation set   CNN with Batch Normalization Batch is a techniques that, through the normalization (subtract mean, divide std dev) of each input belonging to a batch, aims at improving the stability of the network itself. Batch normalization layer is tipically applied before each non-linearity.   Model definition<jupyter_code>class BatchNormalizationCNN(BiggerCNN):
  def __init__(self, train=None, test=None, val=None, classes=11, name='BatchNormalizationCNN'):
    super().__init__(train, test, val, classes, name)
    inputs = ks.Input(shape=(224, 224, 3))
    x = layers.Rescaling(1. / 255)(inputs)
    x = layers.Conv2D(filters=32, kernel_size=(3, 3), padding="same")(x)
    x = layers.BatchNormalization(epsilon = 1e-4)(x)
    x = layers.Activation(('relu'))(x)
    x = layers.MaxPooling2D(pool_size=(2, 2), strides=4)(x)
    x = layers.Conv2D(filters=64, kernel_size=(3, 3), padding="same")(x)
    x = layers.BatchNormalization(epsilon = 1e-4)(x)
    x = layers.Activation(('relu'))(x)
    x = layers.MaxPooling2D(pool_size=(2, 2), strides=4)(x)
    x = layers.Conv2D(filters=128, kernel_size=(3, 3), padding="same")(x)
    x = layers.BatchNormalization(epsilon = 1e-4)(x)
    x = layers.Activation(('relu'))(x)
    x = layers.MaxPooling2D(pool_size=(2, 2))(x)
    x = layers.Conv2D(filters=256, kernel_size=(3, 3), padding="same")(x)
    x = layers.BatchNormalization(epsilon = 1e-4)(x)
    x = layers.Activation(('relu'))(x)
    x = layers.Flatten()(x)
    x = layers.Dense(64, activation='relu')(x)
    x = layers.Dropout(0.2)(x)
    outputs = layers.Dense(classes, activation='softmax', name='predictions')(x)
    self.model = ks.Model(inputs=inputs, outputs=outputs)

batch_normalization_model = BatchNormalizationCNN(train=training_images, val=val_images, test=test_images, classes=NUM_CLASSES)
batch_normalization_model.plot_model(model_name="BatchNormalizationCNN")
batch_normalization_model.summary()<jupyter_output>Model: "model_13"
_________________________________________________________________
 Layer (type)                Output Shape              Param #   
=================================================================
 input_14 (InputLayer)       [(None, 224, 224, 3)]     0         
                                                                 
 rescaling_13 (Rescaling)    (None, 224, 224, 3)       0         
                                                                 
 conv2d_46 (Conv2D)          (None, 224, 224, 32)      896       
                                                                 
 batch_normalization_4 (Batc  (None, 224, 224, 32)     128       
 hNormalization)                                                 
                                                                 
 activation_4 (Activation)   (None, 224, 224, 32)      0         
                                                                 
 max_pooling2d_34 (MaxPoolin  (None, 56, 56, 32)       0  [...]<jupyter_text>Train network on the whole dataset<jupyter_code>batch_normalization_model.compile_and_fit(epochs = 100)
batch_normalization_model.save_history()<jupyter_output><empty_output><jupyter_text>Visualize results In this section, we compare the results of the 3 tested classifier analyzing their behavior in terms of accuracy<jupyter_code>history = BatchNormalizationCNN().load_history()
name='CNN with Batch Normalization and dropout'

acc = history['accuracy']
val_acc = history['val_accuracy']
loss = history['loss']
val_loss = history['val_loss']
best_loss_epoch = np.argmin(val_loss)
best_loss = np.min(val_loss)
best_acc = val_acc[best_loss_epoch]

epochs = range(len(acc))

plt.plot(epochs, acc, 'b.', label='Training accuracy')
plt.plot(epochs, val_acc, 'g.', label='Validation accuracy')
plt.title('Training and validation accuracy on '+name)
plt.legend(loc='upper right')

plt.figure()

plt.plot(epochs, loss, 'b', label='Training loss')
plt.plot(epochs, val_loss, 'g', label='Validation loss')
plt.plot(best_loss_epoch, best_loss, 'ro', label='Selected model')
plt.title('Training and validation loss on '+name)
plt.legend(loc='upper right')

plt.show()
print(f"Lowest validation loss has been reached at epoch {best_loss_epoch:d} with validation accuracy of {best_acc:.3f}")<jupyter_output><empty_output><jupyter_text>Evaluate against test set Once trained our classifier, we pick the most performing one in the validation set and we test it against the test set<jupyter_code>batch_normalization_model.load_best_model()
batch_normalization_model.evaluate(test_images)<jupyter_output>4/4 [==============================] - 3s 143ms/step - loss: 1.2942 - accuracy: 0.6144
Test accuracy: 0.614, test loss: 1.294<jupyter_text>Considerations As results show, batch normalization is not very suitable for our task and especially for the reduced size of our network and dataset: the test and validation accuracy did not improve, but on the contrary the training process is noisier.  CNN with Data Augmentation The aim of data augmentation is to avoid overfitting by generating training images to which we apply small modifications, like random rotations, flipping or scale. In the following, we try to add a Data Augmentation Layer to see if it improves the prediction accuracy of the network  Model definition<jupyter_code>data_augmentation = ks.Sequential(
  [
    layers.RandomFlip("horizontal"), 
    layers.RandomRotation(0.1), 
    layers.RandomZoom(0.2) #, 
    #layers.RandomTranslation(height_factor=(-0.1, 0,1), width_factor=(-0.1, 0.1))
  ]
)
class DataAugmentedCNN(BiggerCNN):
  def __init__(self, train=None, test=None, val=None, classes=11, name='DataAugmentedCNN'):
    super().__init__(train, test, val, classes, name)
    inputs = ks.Input(shape=(224, 224, 3))
    x = data_augmentation(inputs)
    x = layers.Rescaling(1. / 255)(x)
    x = layers.Conv2D(filters=64, kernel_size=(3, 3), padding="same", activation="relu")(x)
    x = layers.MaxPooling2D(pool_size=(2, 2), strides=4)(x)
    x = layers.Conv2D(filters=128, kernel_size=(3, 3), padding="same", activation="relu")(x)
    x = layers.MaxPooling2D(pool_size=(2, 2), strides=4)(x)
    x = layers.Conv2D(filters=256, kernel_size=(3, 3), padding="same", activation="relu")(x)
    x = layers.MaxPooling2D(pool_size=(2, 2))(x)
    x = layers.Conv2D(filters=512, kernel_size=(3, 3), padding="same", activation="relu")(x)
    x = layers.Flatten()(x)
    x = layers.Dense(512, activation='relu')(x)
    x = layers.Dropout(0.2)(x)
    x = layers.Dense(256, activation='relu')(x)
    x = layers.Dropout(0.2)(x)
    x = layers.Dense(128, activation='relu')(x)
    x = layers.Dropout(0.2)(x)
    outputs = layers.Dense(classes, activation='softmax', name='predictions')(x)
    self.model = ks.Model(inputs=inputs, outputs=outputs)

data_augmented_model = DataAugmentedCNN(train=training_images, val=val_images, test=test_images, classes=NUM_CLASSES)
data_augmented_model.plot_model(model_name="DataAugmentedCNN")
data_augmented_model.summary()<jupyter_output>Model: "model_10"
_________________________________________________________________
 Layer (type)                Output Shape              Param #   
=================================================================
 input_11 (InputLayer)       [(None, 224, 224, 3)]     0         
                                                                 
 sequential (Sequential)     (None, 224, 224, 3)       0         
                                                                 
 rescaling_10 (Rescaling)    (None, 224, 224, 3)       0         
                                                                 
 conv2d_35 (Conv2D)          (None, 224, 224, 64)      1792      
                                                                 
 max_pooling2d_26 (MaxPoolin  (None, 56, 56, 64)       0         
 g2D)                                                            
                                                                 
 conv2d_36 (Conv2D)          (None, 56, 56, 128)       738[...]<jupyter_text>Train network on the whole dataset<jupyter_code>data_augmented_model.compile_and_fit(epochs = 100)
data_augmented_model.train_more(epochs=50)
data_augmented_model.save_history()<jupyter_output><empty_output><jupyter_text>Visualize results In this section, we see the results of the tested classifier analyzing its behavior in terms of accuracy<jupyter_code>history = DataAugmentedCNN().load_history()
name='CNN with Data Augmentation'

acc = history['accuracy']
val_acc = history['val_accuracy']
loss = history['loss']
val_loss = history['val_loss']
best_loss_epoch = np.argmin(val_loss)
best_loss = np.min(val_loss)
best_acc = val_acc[best_loss_epoch]

epochs = range(len(acc))

plt.plot(epochs, acc, 'b.', label='Training accuracy')
plt.plot(epochs, val_acc, 'g.', label='Validation accuracy')
plt.title('Training and validation accuracy on '+name)
plt.legend(loc='upper right')

plt.figure()

plt.plot(epochs, loss, 'b', label='Training loss')
plt.plot(epochs, val_loss, 'g', label='Validation loss')
plt.plot(best_loss_epoch, best_loss, 'ro', label='Selected model')
plt.title('Training and validation loss on '+name)
plt.legend(loc='upper right')

plt.show()
print(f"Lowest validation loss has been reached at epoch {best_loss_epoch:d} with validation accuracy of {best_acc:.3f}")<jupyter_output><empty_output><jupyter_text>Evaluate against test set Once trained our classifier, we pick the most performing one in the validation set and we test it against the test set<jupyter_code>data_augmented_model.load_best_model()
data_augmented_model.evaluate(test_images)<jupyter_output>4/4 [==============================] - 3s 248ms/step - loss: 1.7557 - accuracy: 0.4055
Test accuracy: 0.405, test loss: 1.756
{'fraction_non_alphanumeric': 0.1290052606408417, 'fraction_numerical': 0.02971066475370636, 'mean_word_length': 3.494492208490059, 'pattern_counts': {'":': 0, '<': 97, '<?xml version=': 0, '>': 99, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 66, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '27', 'dir': 'jupyter-structured-clean-dedup', 'id': '361026', 'license': 'MIT', 'n_tokens_mistral': 11242, 'n_tokens_neox': 10264, 'n_words': 2451, 'path': '1a_CNN from scratch.ipynb', 'repo_name': 'edofazza/ArtistIdentificationCNN', 'size': '33456'}