<jupyter_start><jupyter_text>Facial Expression Recognition  Loading the datasets<jupyter_code>import os
os.chdir('./src')

from trainer import Trainer
from cnn_model import CNNModel
from fer_reader import FERReader
os.chdir('..')

import tensorflow as tf
from IPython.display import display
import numpy as np

np.random.seed(42)

config = tf.ConfigProto()
config.gpu_options.allow_growth = True
sess = tf.Session(config = config)

reader = FERReader("datasets/fer2013.csv", "datasets/fer2013new.csv")
X_train, X_validation, X_test = reader.read()<jupyter_output><empty_output><jupyter_text>Utility functions<jupyter_code>from sklearn.metrics import confusion_matrix
from sklearn.metrics import f1_score
import matplotlib.pyplot as plt
    
def generate_confusion_matrix(y_true, y_pred, labels):
    return confusion_matrix(y_true, y_pred, labels=labels)

def plot_confusion_matrix(y_true, y_pred, target_names, title='Confusion matrix', cmap=None, normalize=True):
    import itertools
    
    y_true = [target_names[np.argmax(y_true[i])] for i in range(len(y_true))]
    y_pred = [target_names[np.argmax(y_pred[i])] for i in range(len(y_true))]
    
    cm = confusion_matrix(y_true, y_pred, labels=target_names)
    score = f1_score(y_true, y_pred, average="macro")
    accuracy = np.trace(cm) / float(np.sum(cm))
    misclass = 1 - accuracy

    if normalize:
        cm = cm.astype('float') / cm.sum(axis=1)[:, np.newaxis]    

    if cmap is None:
        cmap = plt.get_cmap('Blues')

    plt.figure(figsize=(8, 6))
    plt.imshow(cm, interpolation='nearest', cmap=cmap)
    plt.title(title)
    plt.colorbar()

    if target_names is not None:
        tick_marks = np.arange(len(target_names))
        plt.xticks(tick_marks, target_names, rotation=45)
        plt.yticks(tick_marks, target_names)

    thresh = cm.max() / 1.5 if normalize else cm.max() / 2
    for i, j in itertools.product(range(cm.shape[0]), range(cm.shape[1])):
        if normalize:
            plt.text(j, i, "{:0.2f}".format(cm[i, j]),
                     horizontalalignment="center",
                     color="white" if cm[i, j] > thresh else "black")
        else:
            plt.text(j, i, "{:,}".format(cm[i, j]),
                     horizontalalignment="center",
                     color="white" if cm[i, j] > thresh else "black")

    plt.tight_layout()
    plt.ylabel('True label')
    plt.xlabel('Predicted label\nf1_score={:0.4f}; accuracy={:0.4f}; misclass={:0.4f}'.format(score, accuracy, misclass))
    plt.show()
    
def plot_history(history):
    plt.plot(history.history['acc'])
    plt.plot(history.history['val_acc'])
    plt.title('model accuracy')
    plt.ylabel('accuracy')
    plt.xlabel('epoch')
    plt.legend(['train', 'validation'], loc='upper left')
    plt.show()
    plt.plot(history.history['loss'])
    plt.plot(history.history['val_loss'])
    plt.title('model loss')
    plt.ylabel('loss')
    plt.xlabel('epoch')
    plt.legend(['train', 'validation'], loc='upper left')
    plt.show()<jupyter_output><empty_output><jupyter_text>Naive predictions<jupyter_code>from keras.utils import to_categorical

def naive_predict_array(images):
    return np.tile(to_categorical(reader.emotion_columns.index("neutral"), reader.emotion_count), (len(images), 1)) 

y_test = reader.generate_emotions(reader.test_set)
y_pred = naive_predict_array(X_test) 
plot_confusion_matrix(y_test, y_pred, reader.emotion_columns, normalize=True, title="Confusion Matrix")<jupyter_output><empty_output><jupyter_text>Building the model<jupyter_code>import keras

model = CNNModel((48, 48, 1), reader.emotion_count).build_model()
model.compile(loss=keras.losses.categorical_crossentropy, optimizer="adam", metrics=['acc'])<jupyter_output><empty_output><jupyter_text>Training the model<jupyter_code>trainer = Trainer("model/cnnmodel-{epoch:03d}-{val_loss:.4f}-{val_acc:.4f}.hdf5", verbose=1, batch_size=512)
history = trainer.train(model, reader, X_train, X_validation)<jupyter_output>Epoch 1/5000
56/55 [==============================] - 35s 628ms/step - loss: 1.8301 - acc: 0.3656 - val_loss: 1.6201 - val_acc: 0.4306

Epoch 00001: val_loss improved from inf to 1.62009, saving model to model/cnnmodel-001-1.6201-0.4306.hdf5
Epoch 2/5000
56/55 [==============================] - 29s 516ms/step - loss: 1.5159 - acc: 0.4639 - val_loss: 2.0058 - val_acc: 0.3403

Epoch 00002: val_loss did not improve from 1.62009
Epoch 3/5000
56/55 [==============================] - 32s 568ms/step - loss: 1.3943 - acc: 0.5082 - val_loss: 2.1490 - val_acc: 0.3174

Epoch 00003: val_loss did not improve from 1.62009
Epoch 4/5000
56/55 [==============================] - 32s 567ms/step - loss: 1.3065 - acc: 0.5428 - val_loss: 1.5864 - val_acc: 0.4543

Epoch 00004: val_loss improved from 1.62009 to 1.58638, saving model to model/cnnmodel-004-1.5864-0.4543.hdf5
Epoch 5/5000
56/55 [==============================] - 32s 564ms/step - loss: 1.2481 - acc: 0.5627 - val_loss: 1.3318 - val_acc: 0.5342

Ep[...]<jupyter_text>Evaluation<jupyter_code>score = model.evaluate(X_test, reader.generate_emotions(reader.test_set), verbose=0)
print("Test accuracy: ", score)

plot_history(history)
y_pred = model.predict(X_test)
plot_confusion_matrix(y_test, y_pred, reader.emotion_columns, normalize=True, title="Confusion Matrix - Test set")<jupyter_output>Test accuracy:  [0.8586603989629161, 0.7024909041190127]<jupyter_text>Prediction<jupyter_code>from src.predict import process_image
from matplotlib import pyplot as plt
import cv2 as cv
import os

face_cascade = cv.CascadeClassifier("haarcascade_frontalface_default.xml")

def predict_and_print(image_path):
    print_image(image_path)
    print(predict(image_path))
    
def predict(image_path):
    faces = process_image(image_path)
    predictions = model.predict(faces)
    prediction_labels = [reader.emotion_columns[np.argmax(predictions[i])] for i in range(len(predictions))]
    return prediction_labels    

def print_image(image_path):
    image = cv.imread(image_path)
    gray = cv.cvtColor(image, cv.COLOR_BGR2GRAY)
    faces = face_cascade.detectMultiScale(gray, 1.1, 6)
    if len(faces) > 3:
        fig=plt.figure(figsize=(10, 10))
    else:
        fig=plt.figure(figsize=(4, 4))
    
    sub = fig.add_subplot(1, len(faces) + 1, 1)
    sub.axis('off')
    sub.imshow(gray, cmap = 'gist_gray')

    croppeds = []
    for i, (x, y, w, h) in enumerate(faces):
        cv.rectangle(gray, (x, y), (x+w, y+h), (0, 255, 0), 2)
        croppeds.append(gray[y : y+h, x : x+w])
        
        sub = fig.add_subplot(1, len(faces) + 1, i + 2)
        sub.axis('off')
        sub.imshow(croppeds[i], cmap = 'gist_gray')

    plt.show()

for img in os.listdir("sample_images/"):
    predict_and_print("sample_images/" + img)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.15806546741374225, 'fraction_numerical': 0.05219699203774698, 'mean_word_length': 4.258139534883721, 'pattern_counts': {'":': 0, '<': 27, '<?xml version=': 0, '>': 30, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 1}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '7', 'dir': 'jupyter-structured-clean-dedup', 'id': '301304', 'license': 'Apache-2.0', 'n_tokens_mistral': 2712, 'n_tokens_neox': 2443, 'n_words': 479, 'path': 'capstone/facial_expression_recognition.ipynb', 'repo_name': 'lpicanco/nanodegree-machine-learning', 'size': '6782'}