<jupyter_start><jupyter_text>Image Classification This code shows how images, specifically vehicle images like cars, trucks, vans, bikes, motorcycles and others, can be classified using Convolutional Neural Network (CNN). This code is built based on an image classifier developed for the "Applied Data Science: Machine Learning" Program from the EPFL Lausanne.The dataset consists of Swissroads data set which contains several hundreds images of vehicles found in the EPFL - Lausanne area including cars, trucks, vans, bikes, motorcycles and others. The dataset is quite small, hence the results, but could easily be extended to include other datasets and other vehicles classes.<jupyter_code># filter warnings
import warnings
warnings.simplefilter(action="ignore")
import tensorflow as tf
from tensorflow import keras
from tensorflow.keras import layers
# keras imports
from keras.models import Model
from keras.models import Sequential
from keras.layers import Dense, Conv2D, Flatten, Dropout, MaxPooling2D
from keras.preprocessing.image import ImageDataGenerator
from keras.optimizers import Adam
from keras.utils.vis_utils import plot_model
from keras import backend as K 


import os
import numpy as np
import matplotlib as mpl
import matplotlib.pyplot as plt 
import seaborn as sns
import math
import glob
import random
import re

import cv2
from PIL import Image
BATCH_SIZE = 16
EPOCHS = 500
IMG_HEIGHT = 256
IMG_WIDTH = 256
NUM_CLASSES = 6
IMG_SIZE = (IMG_HEIGHT, IMG_WIDTH)
LEARNING_RATE = 1e-4
base_dir = os.getcwd()  # Get the current working directory
data_dir = os.path.join(base_dir, 'swissroads')
train_dir = os.path.join(data_dir, 'train')
test_dir = os.path.join(data_dir, 'test')
valid_dir = os.path.join(data_dir, 'valid')<jupyter_output><empty_output><jupyter_text>*****Utility functions*****<jupyter_code>mpl.rcParams['figure.figsize'] = (12, 10)
colors = plt.rcParams['axes.prop_cycle'].by_key()['color']
def plot_loss(history, label, n):
  # Use a log scale to show the wide range of values.
    plt.semilogy(history.epoch,  history.history['loss'], color = colors[n], label = 'Train ' + label)
    plt.semilogy(history.epoch,  history.history['val_loss'], color = colors[n], label = 'Val ' + label, linestyle = "--")
    plt.xlabel('Epoch')
    plt.ylabel('Loss')
  
    plt.legend()
def plot_accuracy(history, label, n):
  # Use a log scale to show the wide range of values.
    plt.semilogy(history.epoch,  history.history['accuracy'], color = colors[n], label = 'Train ' + label)
    plt.semilogy(history.epoch,  history.history['val_accuracy'], color = colors[n], label = 'Val ' + label, linestyle = "--")
    plt.xlabel('Epoch')
    plt.ylabel('Accuracy')
  
    plt.legend()
def plot_metrics(history):
    metrics = ['loss', 'accuracy']
    for n, metric in enumerate(metrics):
        name = metric.replace("_"," ").capitalize()
        plt.subplot(2, 2, n + 1)
        plt.plot(history.epoch,  history.history[metric], color = colors[0], label = 'Train')
        plt.plot(history.epoch, history.history['val_' + metric], color = colors[1], linestyle = "--", label = 'Val')
        plt.xlabel('Epoch')
        plt.ylabel(name)
        if metric == 'loss':
            plt.ylim([0, plt.ylim()[1]])
        else:
            plt.ylim([0,1])
        plt.legend()
colorspaces = ['bgr', 'hsv', 'yuv', 'crcb', 'hls', 'lab', 'luv', 'xyz']<jupyter_output><empty_output><jupyter_text>*****Standardizing the data*****Our image are already in a standard size (256x256), as they are being yielded as contiguous float32 batches by our dataset. However, their RGB channel values are in the [0, 255] range. This is not ideal for a neural network; in general we should seek to make your input values small. Here, we will standardize values to be in the [0, 1] by using a Rescaling layer at the start of our model.<jupyter_code># apply data augmentation to improve overfitting
train_image_generator = ImageDataGenerator(rescale = 1./255) # Generator for our training data
# only rescale, as we want our model to predict on real world images
validation_image_generator = ImageDataGenerator(rescale = 1./255) # Generator for our validation data
test_image_generator = ImageDataGenerator(rescale = 1./255) # Generator for our test data<jupyter_output><empty_output><jupyter_text>*****Data pre-processing*****In Keras this can be done via the keras.preprocessing.image.ImageDataGenerator class. This class allows to:- configure random transformations and normalization operations to be done on your image data during training- instantiate generators of augmented image batches (and their labels) via .flow(data, labels) or .flow_from_directory(directory). These generators can then be used with the Keras model methods that accept data generators as inputs, fit_generator, evaluate_generator and predict_generator.<jupyter_code>train_data_gen = train_image_generator.flow_from_directory(batch_size = BATCH_SIZE,
                                                           directory = train_dir,
                                                           shuffle = True,
                                                           target_size = IMG_SIZE,
                                                           class_mode = 'categorical')
valid_data_gen = validation_image_generator.flow_from_directory(batch_size = BATCH_SIZE,
                                                           directory = valid_dir,
                                                           shuffle = True,
                                                           target_size = IMG_SIZE,
                                                           class_mode = 'categorical')
test_data_gen = test_image_generator.flow_from_directory(batch_size = BATCH_SIZE,
                                                           directory = test_dir,
                                                           shuffle = True,
                                                           target_size = IMG_SIZE,
                                                           class_mode = 'categorical')<jupyter_output>Found 50 images belonging to 6 classes.<jupyter_text>This part is to check the data format i.e if the RGB channel is coming first or last so that whatever it may be, the model will first check and then get the appropriate input shape be fed accordingly.<jupyter_code>if K.image_data_format() == 'channels_first': 
    input_shape = (3, IMG_WIDTH, IMG_HEIGHT) 
else: 
    input_shape = (IMG_WIDTH, IMG_HEIGHT, 3)<jupyter_output><empty_output><jupyter_text>*****Build the model***** The right tool for an image classification job is a convnet, so let's try to train one on our data, as an initial baseline. Since we only have few examples, our number one concern should be overfitting. Overfitting happens when a model exposed to too few examples learns patterns that do not generalize to new data, i.e. when the model starts using irrelevant features for making predictions. For instance, if you, as a human, only see three images of people who are lumberjacks, and three, images of people who are sailors, and among them only one lumberjack wears a cap, you might start thinking that wearing a cap is a sign of being a lumberjack as opposed to a sailor. You would then make a pretty lousy lumberjack/sailor classifier.Data augmentation is one way to fight overfitting, but it isn't enough since our augmented samples are still highly correlated. Your main focus for fighting overfitting should be the entropic capacity of your model --how much information your model is allowed to store. A model that can store a lot of information has the potential to be more accurate by leveraging more features, but it is also more at risk to start storing irrelevant features. Meanwhile, a model that can only store a few features will have to focus on the most significant features found in the data, and these are more likely to be truly relevant and to generalize better.There are different ways to modulate entropic capacity. The main one is the choice of the number of parameters in the model, i.e. the number of layers and the size of each layer. Another way is the use of weight regularization, such as L1 or L2 regularization, which consists in forcing model weights to taker smaller values.The architectural decisions we will be making are the number of layers, the number of filters, and the size of the filters. In our case we will use a rather small convnet with few layers and few filters per layer, alongside data augmentation and dropout. Dropout also helps reduce overfitting, by preventing a layer from seeing twice the exact same pattern, thus acting in a way analoguous to data augmentation (you could say that both dropout and data augmentation tend to disrupt random correlations occuring in the data). This is very similar to the architectures that <NAME> advocated in the 1990s for image classification (with the exception of ReLU).- **Conv2D** is the layer to convolve the image into multiple images- **Activation** is the activation function.- **MaxPooling2D** is used to max pool the value from the given size matrix and same is used for the next 2 layers. then, Flatten is used to flatten the dimensions of the image obtained after convolving it.- **Dense** is used to make this a fully connected model and is the hidden layer.- **Dropout** is used to avoid overfitting on the dataset.- **Dense** is the output layer contains only one neuron which decide to which category image belongs.Typically the number of layers starts small and grows as the complexity realized by the convoluted layers grows. The number of filters in a layer should be set at ratios of 32, 64, 128, 256, 512 and so on according to one source. In this case I have elected to make four convolutional layers with succeeding filter sizes of 32, 64, 128, and 256.Filters have odd values since they need to be centered on the pixel being convolved. A 3x3 filter is usual although larger ones of 5x5 up to 7x7 may work better on larger images. Max pool layers typically have a pool size of (2, 2) and are applied after each convolutional layer.After flattening I chose to set dropout to 0.3. The model was given a single dense layer with 512 neurons with activation set to 'relu' as usual. The number of output elements is always set to the number of classes and activation to 'softmax'. *****Model Architecture*****<jupyter_code>def make_model():
        
    model = Sequential()

    model.add(Conv2D(32, kernel_size = (3, 3), activation = 'relu', padding = 'same', input_shape = input_shape))
    model.add(MaxPooling2D(pool_size = (2, 2)))

    model.add(Conv2D(64, (3, 3), activation = 'relu', padding = 'same'))
    model.add(MaxPooling2D(pool_size = (2, 2)))

    model.add(Conv2D(128, (3, 3), activation = 'relu', padding = 'same'))
    model.add(MaxPooling2D(pool_size = (2, 2)))

    model.add(Conv2D(256, (3, 3), activation = 'relu', padding = 'same'))
    model.add(MaxPooling2D(pool_size = (2, 2)))

    model.add(Flatten())
    model.add(Dense(512, activation = 'relu'))
    model.add(Dropout(0.3))
    model.add(Dense(NUM_CLASSES, activation = 'softmax'))
    model.compile(loss = 'categorical_crossentropy', optimizer = Adam(lr = LEARNING_RATE), metrics = ['accuracy'])

    return model
# import PIL.Image as Image
# plot_model(model, to_file='model_plot.png', show_shapes=True, show_layer_names=True)
# im = Image.open('model_plot.png')
# im<jupyter_output><empty_output><jupyter_text>**Early stopping**When we’re training a learning algorithm iteratively, we can measure how well each iteration of the model performs. Up until a certain number of iterations, new iterations improve the model. After that point, however, the model’s ability to generalize can weaken as it begins to overfit the training data. Early stopping refers stopping the training process before the learner passes that point.<jupyter_code>early_stopping = tf.keras.callbacks.EarlyStopping(
    monitor = 'val_accuracy', 
    verbose = 2,
    patience = 20,
    min_delta = 0.001, 
    mode = 'max',
    restore_best_weights = True)
callbacks = [early_stopping]<jupyter_output><empty_output><jupyter_text>*****Train the model*****<jupyter_code>TRAIN_STEPS = int(np.ceil(train_data_gen.n / float(BATCH_SIZE)))
VALIDATION_STEPS = int(np.ceil(valid_data_gen.n / float(BATCH_SIZE)))
TEST_STEPS = int(np.ceil(test_data_gen.n / float(BATCH_SIZE)))
model = make_model()
print(model.summary())
history = model.fit( 
    train_data_gen, 
    steps_per_epoch = TRAIN_STEPS, 
    epochs = EPOCHS,
    callbacks = callbacks, 
    validation_data = valid_data_gen, 
    validation_steps = VALIDATION_STEPS
)
plot_metrics(history)<jupyter_output><empty_output><jupyter_text>The model is clearly overfitting so let's see if augmenting the data by using different color space transformations will improve the model.<jupyter_code># evaluate model on validation data
score = model.evaluate(valid_data_gen, steps = VALIDATION_STEPS, verbose = 2)
print(f'Validation loss: {score[0]} / Validation accuracy: {score[1]}')
# evaluate model on test data
score = model.evaluate(test_data_gen, steps = TEST_STEPS, verbose = 2)
print(f'Test loss: {score[0]} / Test accuracy: {score[1]}')
from sklearn.metrics import classification_report
from sklearn.metrics import confusion_matrix
predictions = model.predict_generator(test_data_gen, steps = TEST_STEPS)
# Get most likely class
predicted_classes = np.argmax(predictions, axis=1)
# Get ground-truth classes and class-labels
true_classes = test_data_gen.classes
class_labels = list(test_data_gen.class_indices.keys())   
report = classification_report(true_classes, predicted_classes, target_names = class_labels)
print(report)
matrix = confusion_matrix(test_data_gen.classes, predicted_classes)
print(matrix)
model.save("cnn_no_opt.h5")
print("Saved model to disk")
from keras.models import load_model
# load, no need to initialize the loaded model
loaded_model = load_model('cnn_no_opt.h5')
total_train = glob.glob(train_dir + '/**/*.png', recursive=True)
total_valid = glob.glob(valid_dir + '/**/*.png', recursive=True)
history = []

for c in colorspaces:
    print('Run transformation: ' + c)

    model = make_model()
    
    def cspace_transf(image):
        image = np.array(image)
        if c == 'bgr':
            return cv2.cvtColor(image,cv2.COLOR_RGB2BGR)
        elif c == 'hsv':
            return cv2.cvtColor(image,cv2.COLOR_RGB2HSV)
        elif c == 'yuv':
            return cv2.cvtColor(image,cv2.COLOR_RGB2YUV)
        elif c =='crcb':
            return cv2.cvtColor(image,cv2.COLOR_RGB2CrCB)
        elif c == 'hls':
            return cv2.cvtColor(image,cv2.COLOR_RGB2HLS)
        elif c == 'lab':
            return cv2.cvtColor(image,cv2.COLOR_RGB2Lab)
        elif c == 'luv':
            return cv2.cvtColor(image,cv2.COLOR_RGB2Luv)
        elif c == 'xyz':
            return cv2.cvtColor(image,cv2.COLOR_RGB2XYZ)
        else:
            return image
    
    train_data_generator = ImageDataGenerator(rescale = 1./255, preprocessing_function = cspace_transf)
    train_data_gen = train_image_generator.flow_from_directory(batch_size = BATCH_SIZE,
                                                           directory = train_dir,
                                                           shuffle = True,
                                                           target_size = IMG_SIZE,
                                                           class_mode = 'categorical') 
    history.append(model.fit(
                  train_data_gen,
                  steps_per_epoch = TRAIN_STEPS,
                  epochs = 30,
#                   callbacks = callbacks, 
                  validation_data = valid_data_gen,
                  validation_steps = VALIDATION_STEPS
                  ))
epochs_range = range(30)

plt.figure(figsize = (20, 10))
plt.subplot(2, 2, 1)
for i, c in enumerate(colorspaces):
    plt.plot(epochs_range, history[i].history['accuracy'], label = c)
plt.legend(loc = 'lower right')
plt.title('Training Accuracy')

plt.subplot(2, 2, 2)
for i, c in enumerate(colorspaces):
    plt.plot(epochs_range, history[i].history['val_accuracy'], label = c)
plt.legend(loc = 'lower right')
plt.title('Validation Accuracy')

plt.subplot(2, 2, 3)
for i, c in enumerate(colorspaces):
    plt.plot(epochs_range, history[i].history['loss'], label = c)
plt.legend(loc = 'upper right')
plt.title('Training Loss')

plt.subplot(2, 2, 4)
for i, c in enumerate(colorspaces):
    plt.plot(epochs_range, history[i].history['val_loss'], label = c)
plt.legend(loc = 'upper right')
plt.title('Validation Loss')
plt.show()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.08737922342915441, 'fraction_numerical': 0.012542759407069556, 'mean_word_length': 3.3968337730870712, 'pattern_counts': {'":': 0, '<': 37, '<?xml version=': 0, '>': 37, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 32, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '9', 'dir': 'jupyter-structured-clean-dedup', 'id': '180042', 'license': 'MIT', 'n_tokens_mistral': 4880, 'n_tokens_neox': 4583, 'n_words': 1718, 'path': 'CNN-colorspaces.ipynb', 'repo_name': 'lauravoicu/Image-Classification', 'size': '16667'}