<jupyter_start><jupyter_text>Dataset downloadIn this section the dataset is downloaded from *Kaggle*, unzipped and well formatted.<jupyter_code>import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import json
import os
from tqdm import tqdm, tqdm_notebook
import random

import tensorflow as tf
from tensorflow.keras.models import Sequential, Model
from tensorflow.keras.layers import *
from tensorflow.keras.optimizers import *
from tensorflow.keras.applications import *
from tensorflow.keras.callbacks import *
from tensorflow.keras.initializers import *
from tensorflow.keras.preprocessing.image import ImageDataGenerator

from numpy.random import seed
seed(42)
tf.random.set_seed(42)
! pip install -q kaggle

from google.colab import files
_ = files.upload()

! mkdir -p ~/.kaggle
! cp kaggle.json ~/.kaggle/
! chmod 600 ~/.kaggle/kaggle.json
! kaggle datasets download -d ikarus777/best-artworks-of-all-time
! unzip best-artworks-of-all-time.zip<jupyter_output>[1;30;43mOutput streaming troncato alle ultime 5000 righe.[0m
  inflating: resized/resized/Henri_Rousseau_61.jpg  
  inflating: resized/resized/Henri_Rousseau_62.jpg  
  inflating: resized/resized/Henri_Rousseau_63.jpg  
  inflating: resized/resized/Henri_Rousseau_64.jpg  
  inflating: resized/resized/Henri_Rousseau_65.jpg  
  inflating: resized/resized/Henri_Rousseau_66.jpg  
  inflating: resized/resized/Henri_Rousseau_67.jpg  
  inflating: resized/resized/Henri_Rousseau_68.jpg  
  inflating: resized/resized/Henri_Rousseau_69.jpg  
  inflating: resized/resized/Henri_Rousseau_7.jpg  
  inflating: resized/resized/Henri_Rousseau_70.jpg  
  inflating: resized/resized/Henri_Rousseau_8.jpg  
  inflating: resized/resized/Henri_Rousseau_9.jpg  
  inflating: resized/resized/Henri_de_Toulouse-Lautrec_1.jpg  
  inflating: resized/resized/Henri_de_Toulouse-Lautrec_10.jpg  
  inflating: resized/resized/Henri_de_Toulouse-Lautrec_11.jpg  
  inflating: resized/resized/Henri_de_Toulouse-Lautrec_12.j[...]<jupyter_text>Data preprocessing<jupyter_code>artists = pd.read_csv('artists.csv')
# Sort artists by number of paintings
artists = artists.sort_values(by=['paintings'], ascending=False)

# Create a dataframe with artists having more than 200 paintings
artists_top = artists[artists['paintings'] >= 200].reset_index()
artists_top = artists_top[['name', 'paintings']]
artists_top['class_weight'] = artists_top.paintings.sum() / (artists_top.shape[0] * artists_top.paintings)
#artists_top = artists_top.loc[artists_top['paintings'] >200]
artists_top
# Set class weights - assign higher weights to underrepresented classes
class_weights = artists_top['class_weight'].to_dict()
class_weights<jupyter_output><empty_output><jupyter_text>Next step is to solve a problem: the downloaded dataset present two directory containing the same paintings but they have different name. What we do in the next cells is to create a single directory called 'Albrecht_Durer' containing one copy of each painting, and then we delete all unuseful directories.<jupyter_code>updated_name = "Albrecht_Dürer".replace("_", " ")
artists_top.iloc[4, 0] = updated_name
images_dir = 'images/images'
artists_dirs = os.listdir(images_dir)
artists_top_name = artists_top['name'].str.replace(' ', '_').values
! mv '/content/images/images/Albrecht_Du╠Иrer' /content/images/images/Albrecht_Dürer
! rm -R '/content/images/images/Albrecht_DuтХа├кrer' 
! rm -R '/content/resized'<jupyter_output><empty_output><jupyter_text>Data fetch<jupyter_code>import pathlib
import os

IMAGE_DIR = '/content/images/'
TRAIN_DIR = pathlib.Path(os.path.join(IMAGE_DIR, 'train'))
TEST_DIR = pathlib.Path(os.path.join(IMAGE_DIR, 'test'))

IMAGE_HEIGHT = 299
IMAGE_WIDTH = 299
BATCH_SIZE = 128
RANDOM_SEED = 42
VALIDATION_SPLIT = 0.10
import os
import numpy as np
import shutil

rootdir= '/content/images/images' #path of the original folder
classes = os.listdir(rootdir)

for i, c in enumerate(classes, start=1):
  if c not in artists_top_name.tolist():
    shutil.rmtree(rootdir + '/' + c)
    continue
  if not os.path.exists(rootdir + '/train/' + c):
    os.makedirs(rootdir + '/train/' + c)
  if not os.path.exists(rootdir + '/test/' + c):  
    os.makedirs(rootdir + '/test/' + c)

  source = os.path.join(rootdir, c)
  allFileNames = os.listdir(source)

  np.random.shuffle(allFileNames)

  test_ratio = 0.10
  train_FileNames, test_FileNames = np.split(np.array(allFileNames),
                                                        [int(len(allFileNames)* (1 - test_ratio))])

  train_FileNames = [source+'/'+ name for name in train_FileNames.tolist()]
  test_FileNames = [source+'/' + name for name in test_FileNames.tolist()]

  for name in train_FileNames:
    shutil.copy(name, rootdir +'/train/' + c)

  for name in test_FileNames:
    shutil.copy(name, rootdir +'/test/' + c)
! mv /content/images/images/train /content/images
! mv /content/images/images/test /content/images
! rm -r /content/images/images
import tensorflow as tf

training_images = tf.keras.preprocessing.image_dataset_from_directory(
    TRAIN_DIR, labels='inferred', label_mode='categorical',
    class_names=None, color_mode='rgb', batch_size=BATCH_SIZE, image_size=(IMAGE_HEIGHT,
    IMAGE_WIDTH), shuffle=True, seed=RANDOM_SEED, validation_split=VALIDATION_SPLIT, subset='training',
    interpolation='bilinear', follow_links=False
)

val_images = tf.keras.preprocessing.image_dataset_from_directory(
    TRAIN_DIR, labels='inferred', label_mode='categorical',
    class_names=None, color_mode='rgb', batch_size=BATCH_SIZE, image_size=(IMAGE_HEIGHT,
    IMAGE_WIDTH), shuffle=True, seed=RANDOM_SEED, validation_split=VALIDATION_SPLIT, subset='validation',
    interpolation='bilinear', follow_links=False
)

test_images = tf.keras.preprocessing.image_dataset_from_directory(
    TEST_DIR, labels='inferred', label_mode='categorical',
    class_names=None, color_mode='rgb', batch_size=BATCH_SIZE, image_size=(IMAGE_HEIGHT,
    IMAGE_WIDTH), shuffle=True, seed=RANDOM_SEED, interpolation='bilinear', follow_links=False
)

NUM_CLASSES = len(training_images.class_names)<jupyter_output><empty_output><jupyter_text>Build model Needed imports:<jupyter_code>import tensorflow as tf
from tensorflow import keras as ks
from tensorflow.keras import layers
from tensorflow.keras.applications import InceptionV3
from tensorflow.keras import regularizers
import pathlib
import matplotlib.pyplot as plt
import numpy as np<jupyter_output><empty_output><jupyter_text>Callback and data augmentation objects:<jupyter_code>callbacks_list = [
    ks.callbacks.EarlyStopping(
        monitor='val_accuracy',
        patience=5,
    ),
    ks.callbacks.ModelCheckpoint(
        filepath='checkpoint_inception.keras',
        monitor='val_loss',
        save_best_only=True,
    )
]
data_augmentation = ks.Sequential(
    [
        layers.RandomFlip('horizontal'),
        layers.RandomRotation(0.1),
        layers.RandomZoom(0.2),
        layers.RandomHeight(0.1),
        layers.RandomWidth(0.1)
    ]
)<jupyter_output><empty_output><jupyter_text>Inception class:<jupyter_code>class Inception:
    LOSS = 'categorical_crossentropy'

    def __init__(self, train=None, test=None, val=None, classes=11, epochs=10,
                 callbacks_list=None, data_augmentation=None, class_weights=None):
        # set datasets
        self.training_set = train
        self.test_set = test
        self.val_set = val

        # model
        self.model = None

        # utils
        self.history = None
        self.classes = classes
        self.epochs = epochs
        self.callbacks_list = callbacks_list
        self.data_augmentation = data_augmentation
        self.class_weights = class_weights

    """
        ARCHITECTURE TO TEST
    """
    def classic_inception(self):
        inputs = self.input()
        x = self.base_inception()(inputs)
        x = layers.GlobalAveragePooling2D(name='my_glo_avg_pool')(x)
        outputs = layers.Dense(self.classes, activation='softmax', name='predictions')(x)
        self.model = ks.Model(inputs=inputs, outputs=outputs)

    def inception_finetuned(self, num_of_blocks=1):
        inputs = self.input()
        x = self.fine_tuning(num_of_blocks=num_of_blocks)(inputs)
        x = layers.GlobalAveragePooling2D(name='my_glo_avg_pool')(x)
        outputs = layers.Dense(self.classes, activation='softmax', name='predictions')(x)
        self.model = ks.Model(inputs=inputs, outputs=outputs)

    def inception_finetuned_drop(self, num_of_blocks=1):
        inputs = self.input()
        x = self.fine_tuning(num_of_blocks=num_of_blocks)(inputs)
        x = layers.GlobalAveragePooling2D(name='my_glo_avg_pool')(x)
        x = layers.Dropout(0.5)(x)
        outputs = layers.Dense(self.classes, activation='softmax', name='predictions')(x)
        self.model = ks.Model(inputs=inputs, outputs=outputs)
    
    def inception_finetuned_dense(self, num_of_blocks=1):
        inputs = self.input()
        x = self.fine_tuning(num_of_blocks=num_of_blocks)(inputs)
        x = layers.GlobalAveragePooling2D(name='my_glo_avg_pool')(x)
        x = layers.Dropout(0.5)(x)
        x = layers.Dense(512,
                         activation='relu',
                         name='my_dense1')(x)
        x = layers.Dense(256,
                         activation='relu',
                         name='my_dense2')(x)
        outputs = layers.Dense(self.classes, activation='softmax', name='predictions')(x)
        self.model = ks.Model(inputs=inputs, outputs=outputs)


    """
        NETWORK PIECE BY PIECE STRUCTURE BLOCK
    """

    def input(self):
        inputs = ks.Input(shape=(299, 299, 3))
        x = ks.applications.inception_v3.preprocess_input(inputs)
        x = self.data_augmentation(x)
        x = layers.Rescaling(1. / 255)(x)
        return x

    def base_inception(self):
        res = InceptionV3(
            weights='imagenet',
            include_top=False,
            input_shape=((299, 299, 3))
        )
        res.trainable = False
        return res

    def fine_tuning(self, num_of_blocks=1):
        if num_of_blocks < 1:
            num_of_blocks = 1

        res = self.base_inception()
        res.trainable = True
        set_trainable = False

        if num_of_blocks == 1:
            block_name = 'mixed9'
        elif num_of_blocks == 2:
            block_name = 'mixed8'
        elif num_of_blocks == 3:
            block_name = 'mixed7'
        elif num_of_blocks == 4:
            block_name = 'mixed6'
        else:
            block_name = 'mixed5'

        for layer in res.layers:
            if layer.name == block_name:
                set_trainable = True
            if set_trainable:
                layer.trainable = True
            else:
                layer.trainable = False

        return res

    """
            UTILITIES
    """

    def plot_model(self, model_name):
        ks.utils.plot_model(self.model, model_name, show_shapes=True)

    def plot_accuracy(self):
        acc = self.history.history['accuracy']
        val_acc = self.history.history['val_accuracy']
        plt.plot(range(1, len(acc) + 1), acc, 'r', label='Training Accuracy')
        plt.plot(range(1, len(acc) + 1), val_acc, 'g', label='Validation Accuracy')
        plt.title('Training and Validation Accuracy')
        plt.xlabel('Epochs')
        plt.ylabel('Accuracy')
        plt.legend()
        plt.plot()

    def plot_loss(self):
        loss = self.history.history['loss']
        val_loss = self.history.history['val_loss']
        plt.plot(range(1, len(loss) + 1), loss, 'r', label='Training Loss')
        plt.plot(range(1, len(loss) + 1), val_loss, 'g', label='Validation Loss')
        plt.title('Training and Validation Loss')
        plt.xlabel('Epochs')
        plt.ylabel('Loss')
        plt.legend()
        plt.plot()

    def summary(self):
        self.model.summary()

    """
          COMPILE AND FIT
    """

    def compile_and_fit(self, optimizer):
        self.model.compile(
            optimizer=optimizer,
            loss=self.LOSS,
            metrics=['accuracy']
        )

        if self.callbacks_list is None:
            self.history = self.model.fit(
                self.training_set,
                epochs=self.epochs,
                validation_data=self.val_set
            )
        else:
            self.history = self.model.fit(
                self.training_set,
                epochs=self.epochs,
                validation_data=self.val_set,
                callbacks=self.callbacks_list,
                class_weight=self.class_weights
            )

    def evaluate(self):
        test_loss, test_acc = self.model.evaluate(self.test_set)
        print(f"Test accuracy: {test_acc:.3f}")<jupyter_output><empty_output><jupyter_text>Feature extraction:<jupyter_code>inception = Inception(train=training_images, val=val_images, test=test_images, 
              classes=NUM_CLASSES, callbacks_list=callbacks_list, 
              data_augmentation=data_augmentation, class_weights=class_weights,
              epochs=60)
inception.classic_inception()
inception.summary()
inception.compile_and_fit(optimizer=ks.optimizers.Adam(learning_rate=0.01))
inception.plot_accuracy()
inception.plot_loss()
inception.evaluate()<jupyter_output>4/4 [==============================] - 6s 810ms/step - loss: 10.9338 - accuracy: 0.4092
Test accuracy: 0.409<jupyter_text>Fine tuning one block<jupyter_code>inception = Inception(train=training_images, val=val_images, test=test_images, 
              classes=NUM_CLASSES, callbacks_list=callbacks_list, 
              data_augmentation=data_augmentation, class_weights=class_weights,
              epochs=60)
inception.resnet101_finetuned(num_of_blocks=1)
inception.summary()
inception.compile_and_fit(optimizer=ks.optimizers.Adam(learning_rate=0.001))
inception.plot_accuracy()
inception.plot_loss()
inception.evaluate()<jupyter_output>4/4 [==============================] - 6s 806ms/step - loss: 3.3541 - accuracy: 0.4437
Test accuracy: 0.444<jupyter_text>Finetuned 2 blocks<jupyter_code>inception = Inception(train=training_images, val=val_images, test=test_images, 
              classes=NUM_CLASSES, callbacks_list=callbacks_list, 
              data_augmentation=data_augmentation, class_weights=class_weights,
              epochs=60)
inception.resnet101_finetuned(num_of_blocks=2)
inception.summary()
inception.compile_and_fit(optimizer=ks.optimizers.Adam(learning_rate=0.001))
inception.plot_accuracy()
inception.plot_loss()
inception.evaluate()<jupyter_output>4/4 [==============================] - 11s 848ms/step - loss: 6.3845 - accuracy: 0.4184
Test accuracy: 0.418<jupyter_text>Finetuning 3<jupyter_code>inception = Inception(train=training_images, val=val_images, test=test_images, 
              classes=NUM_CLASSES, callbacks_list=callbacks_list, 
              data_augmentation=data_augmentation, class_weights=class_weights,
              epochs=60)
inception.inception_finetuned(num_of_blocks=3)
inception.summary()
inception.compile_and_fit(optimizer=ks.optimizers.Adam(learning_rate=0.0001))
inception.plot_accuracy()
inception.plot_loss()
inception.evaluate()<jupyter_output>4/4 [==============================] - 9s 948ms/step - loss: 1.4673 - accuracy: 0.5908
Test accuracy: 0.591<jupyter_text>Ft3 dropout<jupyter_code>inception = Inception(train=training_images, val=val_images, test=test_images, 
              classes=NUM_CLASSES, callbacks_list=callbacks_list, 
              data_augmentation=data_augmentation, class_weights=class_weights,
              epochs=60)
inception.inception_finetuned_drop(num_of_blocks=3)
inception.summary()
inception.compile_and_fit(optimizer=ks.optimizers.Adam(learning_rate=0.0001))
inception.plot_accuracy()
inception.plot_loss()
inception.evaluate()<jupyter_output>4/4 [==============================] - 8s 907ms/step - loss: 1.3284 - accuracy: 0.6069
Test accuracy: 0.607<jupyter_text>Finetuned and Exponential Decay<jupyter_code>inception = Inception(train=training_images, val=val_images, test=test_images, 
              classes=NUM_CLASSES, callbacks_list=callbacks_list, 
              data_augmentation=data_augmentation, class_weights=class_weights,
              epochs=60)
inception.inception_finetuned(num_of_blocks=3)
inception.summary()<jupyter_output>Model: "model_2"
_________________________________________________________________
 Layer (type)                Output Shape              Param #   
=================================================================
 input_9 (InputLayer)        [(None, None, None, 3)]   0         
                                                                 
 inception_v3 (Functional)   (None, 8, 8, 2048)        21802784  
                                                                 
 my_glo_avg_pool (GlobalAver  (None, 2048)             0         
 agePooling2D)                                                   
                                                                 
 predictions (Dense)         (None, 11)                22539     
                                                                 
=================================================================
Total params: 21,825,323
Trainable params: 12,834,443
Non-trainable params: 8,990,880
_______________________________________[...]<jupyter_text>Learning rate is now a ExponentialDecay<jupyter_code>inception.compile_and_fit(optimizer=ks.optimizers.Adam(learning_rate=ks.optimizers.schedules.ExponentialDecay(initial_learning_rate=1e-2,
    decay_steps=25,
    decay_rate=0.9)))
inception.plot_accuracy()
inception.plot_loss()
inception.evaluate()<jupyter_output>4/4 [==============================] - 6s 869ms/step - loss: 2.3217 - accuracy: 0.6253
Test accuracy: 0.625
{'fraction_non_alphanumeric': 0.12142575150865716, 'fraction_numerical': 0.021713383339913146, 'mean_word_length': 3.4021847070506457, 'pattern_counts': {'":': 0, '<': 50, '<?xml version=': 0, '>': 51, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 16, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '14', 'dir': 'jupyter-structured-clean-dedup', 'id': '517684', 'license': 'MIT', 'n_tokens_mistral': 6088, 'n_tokens_neox': 5673, 'n_words': 980, 'path': '2-InceptionV3.ipynb', 'repo_name': 'edofazza/ArtistIdentificationCNN', 'size': '17731'}