<jupyter_start><jupyter_text>Facial Emotion Recognition (Model Training)This dataset consists of 48*48 pixel grayscale face images.The images are centered and occupy an equal amount of space.The dataset consists of 7 categories:0. angry1. disgust (removed)2. fear3. happy4. sad5. surprise6. neutralTraining set: 28,709 samplesPublic test set: 3,589 examples<jupyter_code># Import Libraries
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import seaborn as sns
import os
import cv2

from PIL import Image
import tensorflow as tf
from IPython.display import Image 

from sklearn.model_selection import train_test_split
from skimage.transform import resize
from sklearn.metrics import accuracy_score


from keras.applications.resnet import ResNet50
from keras.applications.nasnet import NASNetLarge
from keras.models import Model, Sequential
from keras.layers import Dense, Dropout, Flatten, Conv2D, MaxPooling2D, BatchNormalization, Activation
from keras.optimizers import Adam
from keras.preprocessing.image import ImageDataGenerator
from keras.callbacks import ReduceLROnPlateau, EarlyStopping,ModelCheckpoint
from tensorflow.keras.utils import plot_model
from keras import regularizers
import keras.backend as K

import warnings
warnings.filterwarnings('ignore')<jupyter_output><empty_output><jupyter_text>Explore dataset<jupyter_code># We need to import the file on google drive (authorization)
from google.colab import drive
drive.mount('/content/drive') 
# Count the images in the train file
train = {}
count = 0
for folder in os.listdir('/content/drive/MyDrive/Colab Notebooks/FER_2013_images/train'):
    temp = []
    for files in os.listdir(f"/content/drive/MyDrive/Colab Notebooks/FER_2013_images/train/{folder}"):
        temp.append(files)
    count += len(temp)
    train[folder] = temp
    print(f"{folder} has {len(temp)} images")
print(f"Total images in all folders are {count}")
# Count the images in the test file
test = {}
count = 0
for folder in os.listdir('/content/drive/MyDrive/Colab Notebooks/FER_2013_images/test'):
    temp = []
    for files in os.listdir(f"/content/drive/MyDrive/Colab Notebooks/FER_2013_images/test/{folder}"):
        temp.append(files)
    count += len(temp)
    test[folder] = temp
    print(f"{folder} has {len(temp)} images")
print(f"Total images in all folders are {count}")
# training dataframe with folder name as index
train_df = pd.DataFrame.from_dict(train.values())
train_df.index = train.keys()
train_df
# list of all the emotions to classify in the output
emotions = [k for k in train.keys()]
emotions
base_dir = '/content/drive/MyDrive/Colab Notebooks/FER_2013_images/'
# lets see one image from each category from training data
plt.figure(figsize = (20, 8))
for i in range(6):
    ax = plt.subplot(2,4, i+1)
    img = cv2.imread(f"{base_dir}train/{emotions[i]}/{train_df.loc[emotions[i], i+7]}")
    ax.imshow(img, cmap = 'gray')
    ax.set_xticks([])
    ax.set_yticks([])
    ax.set_title(emotions[i])
# To know the shape of image
one_img = cv2.imread('/content/drive/MyDrive/Colab Notebooks/FER_2013_images/train/surprise/Training_10013223.jpg')
one_img.shape<jupyter_output><empty_output><jupyter_text>Data PreprocessingUsing Keras "ImageDataGenerator" to augment images in real-time while our model is still training.The ImageDataGenerator class has 3 methods `flow()`, `flow_from_directory()`, and `flow_from_dataframe()` to read the images from a big numpy array and folders containing images.Here, we use flow_from_directory() because it expects at least one directory under the given directory path.<jupyter_code># Data preprocessing(augmentation)
train_dir = f"{base_dir}/train"
test_dir = f"{base_dir}/test"

train_datagen = ImageDataGenerator(horizontal_flip = True,
                  rotation_range= 10,
                  zoom_range = 0.1, 
                  width_shift_range = 0.1,
                  height_shift_range = 0.1,
                  rescale = 1./255,
                  fill_mode = 'nearest',
                  validation_split = 0.2)

valid_datagen = ImageDataGenerator(rescale=1./255,
                  validation_split=0.2)

test_datagen = ImageDataGenerator(rescale= 1./255)

train_generator = train_datagen.flow_from_directory(
                        train_dir,
                        target_size=(48,48),
                        batch_size=64,
                        class_mode='categorical',
                        color_mode='grayscale',
                        subset = "training")

valid_generator = valid_datagen.flow_from_directory(train_dir,
                           target_size=(48,48),
                           class_mode='categorical',
                           subset='validation',
                           color_mode='grayscale',
                           batch_size=64)

test_generator = test_datagen.flow_from_directory(
                        test_dir,
                        target_size=(48,48),
                        batch_size=64,
                        color_mode='grayscale',
                        class_mode='categorical')<jupyter_output>Found 22988 images belonging to 6 classes.
Found 5744 images belonging to 6 classes.
Found 7075 images belonging to 6 classes.<jupyter_text>CNN Model Building<jupyter_code>model = None

# Build model on the top of base model (transfer learning)
model = Sequential()

model.add(Conv2D(32, kernel_size=(3, 3), padding='same', activation='relu', input_shape=(48, 48, 1)))
model.add(Conv2D(64, kernel_size=(3, 3), activation='relu', padding='same'))
model.add(BatchNormalization())
model.add(MaxPooling2D(2, 2))
model.add(Dropout(0.25))

model.add(Conv2D(128, kernel_size=(3, 3), activation='relu', padding='same', kernel_regularizer=regularizers.l2(0.01)))
model.add(Conv2D(256, kernel_size=(3, 3), activation='relu', kernel_regularizer=regularizers.l2(0.01)))
model.add(BatchNormalization())
model.add(MaxPooling2D(pool_size=(2, 2)))
model.add(Dropout(0.25))

model.add(Flatten())
model.add(Dense(1024, activation='relu'))
model.add(Dropout(0.5))
    
model.add(Dense(6, activation='softmax'))


# model Summary
print(f'Number of layers:', len(model.layers))
model.summary()
# function to calculate f1_score
def f1_score(y_true, y_pred):
    true_positives = K.sum(K.round(K.clip(y_true * y_pred, 0, 1)))
    possible_positives = K.sum(K.round(K.clip(y_true, 0, 1)))
    predicted_positives = K.sum(K.round(K.clip(y_pred, 0, 1)))
    precision = true_positives / (predicted_positives + K.epsilon())
    recall = true_positives / (possible_positives + K.epsilon())
    f1_val = 2*(precision*recall)/(precision+recall+K.epsilon())
    return f1_val
# evaluation metrics
from keras.metrics import AUC, BinaryAccuracy, Precision, Recall
metric = [BinaryAccuracy(name = 'accuracy'), 
      Precision(name = 'precision'), 
      Recall(name = 'recall'), 
      AUC(name = 'AUC'),]
# callbacks
checkpoint = ModelCheckpoint('model.h5')

earlystop = EarlyStopping(patience=20, verbose=1)
# restore_best_weights=True)
                      
reduce_lr = ReduceLROnPlateau(monitor='val_loss', 
                factor=0.5, 
                patience=20, 
                verbose=1,
                min_lr=1e-10)
# min_delta=0.0001)
callbacks = [checkpoint,earlystop,reduce_lr]
# compile model (optimization)
model.compile(optimizer= 'Adam', loss='categorical_crossentropy', metrics=metric)<jupyter_output><empty_output><jupyter_text>The difference between `Keras.fit` and `Keras.fit_generator` functions used to train a deep learning neural network:`model.fit` is used when the entire training dataset can fit into the memory *   `model.fit` is used when the entire training dataset can fit into the memory and no data augmentation is applied.*   `model.fit_generator` is used when either we have a huge dataset to fit into our memory or when data augmentation needs to be applied.<jupyter_code># model fitting
history = model.fit_generator(train_generator, validation_data=valid_generator,epochs=100,verbose = 1,callbacks=callbacks)<jupyter_output><empty_output><jupyter_text>Save Model<jupyter_code>model.save('model_optimal_singlechannel_addingdata_0626.h5')
model.save_weights('model_weights_singlechannel_addingdata_0626.h5')<jupyter_output><empty_output><jupyter_text>Plotting<jupyter_code>plt.figure(0)
plt.plot(history.history['accuracy'], label= 'training accuracy')
plt.plot(history.history['val_accuracy'], label= 'validation accuracy')
plt.title('Accuracy')
plt.xlabel('epochs')
plt.ylabel('Accuracy')
plt.legend()

plt.figure(0)
plt.plot(history.history['loss'], label= 'training loss')
plt.plot(history.history['val_loss'], label= 'validation loss')
plt.title('Loss')
plt.xlabel('epochs')
plt.ylabel('Loss')
plt.legend()
plt.figure(0)
plt.plot(history.history['AUC'], label= 'training auc')
plt.plot(history.history['val_AUC'], label= 'validation auc')
plt.title('auc')
plt.xlabel('epochs')
plt.ylabel('auc')
plt.legend()
plt.figure(0)
plt.plot(history.history['precision'], label= 'train precision')
plt.plot(history.history['val_precision'], label= 'validation precision')
plt.title('precision')
plt.xlabel('epochs')
plt.ylabel('precision')
plt.legend()
plt.figure(0)
plt.plot(history.history['recall'], label= 'train recall')
plt.plot(history.history['val_recall'], label= 'validation recall')
plt.title('recall')
plt.xlabel('epochs')
plt.ylabel('recall')
plt.legend()<jupyter_output><empty_output><jupyter_text>Testing Dataset Accuracy<jupyter_code>model.evaluate(test_generator)<jupyter_output><empty_output><jupyter_text>Confusion Matrix<jupyter_code>y_pred = model.predict(train_generator)
y_pred = np.argmax(y_pred, axis=1)
class_labels = test_generator.class_indices
class_labels = {v:k for k,v in class_labels.items()}

from sklearn.metrics import classification_report, confusion_matrix
cm_train = confusion_matrix(train_generator.classes, y_pred)
print('Confusion Matrix')
print(cm_train)
print('Classification Report')
target_names = list(class_labels.values())
print(classification_report(train_generator.classes, y_pred, target_names=target_names))

plt.figure(figsize=(8,8))
plt.imshow(cm_train, interpolation='nearest')
plt.colorbar()
tick_mark = np.arange(len(target_names))
_ = plt.xticks(tick_mark, target_names, rotation=90)
_ = plt.yticks(tick_mark, target_names)











# Import the model (if re-connect)
from tensorflow import keras
model = keras.models.load_model('/content/drive/MyDrive/Colab Notebooks/Kevin/model_optimal.h5')<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.11662531017369727, 'fraction_numerical': 0.02252338232487116, 'mean_word_length': 4.526898734177215, 'pattern_counts': {'":': 0, '<': 36, '<?xml version=': 0, '>': 36, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '9', 'dir': 'jupyter-structured-clean-dedup', 'id': '11574', 'license': 'MIT', 'n_tokens_mistral': 3559, 'n_tokens_neox': 3287, 'n_words': 753, 'path': 'Facial_Emotion_Recognition_CNN_model.ipynb', 'repo_name': 'itoyuhao/Real_Time_Facial_Expression_Recognition', 'size': '10478'}