<jupyter_start><jupyter_text>Visualization and comparison of the trained models Model visualization and comparison is important to give comprehensive interpretation of the results.  Preprocess and load data<jupyter_code>from typing import List
import datetime
import io
import os
import pickle

import torch
import torch.nn as nn
from torch.utils.data import DataLoader, Dataset
import torch.nn.functional as F
from xgboost import XGBClassifier

from torchvision import transforms
from PIL import Image
from sklearn.model_selection import train_test_split
from tqdm import tqdm

import torchvision.models as models
import matplotlib.pyplot as plt
import pandas as pd
import numpy as np


from src.data.preprocess_data import DatasetManager
from src.features.dimension_reduction import fit_pca, load_cnn_embedding


# S3 bucket
import boto3
from dotenv import dotenv_values<jupyter_output><empty_output><jupyter_text>Define constants to be used later. *DRIVE_PATH* and *MODEL_PATH* should be the root path of project and path for models, respectively<jupyter_code>EMOTION_LIST = ['Angry', 'Disgust', 'Fear', 'Happy', 'Sad', 'Surprise', 'Neutral']
MODEL_PATH = "../models/"
FIGURE_PATH = '../reports/figures/'

# Set values
BATCH_SIZE = 8
VAL_SIZE = 0.2
TEST_SIZE = 0.2
N_EPOCHS = 100
INPUT_SIZE = 224
N_FEATURES = len(EMOTION_LIST)
dataset_manager = DatasetManager(batch_size=BATCH_SIZE, test_size=TEST_SIZE, 
                        validation_size=VAL_SIZE, transform=None, 
                        test_transform=None)
data = dataset_manager.load_dataloaders(return_raw_data=True)<jupyter_output><empty_output><jupyter_text>Unpack test data. Also load all data to be used later.<jupyter_code>X_train, X_test, X_val = [x[1] for x in data['X'].items()]
y_train, y_test, y_val = [x[1] for x in data['y'].items()]<jupyter_output><empty_output><jupyter_text>Define own dataset manager for deep learning model, because it needs input preprocessing<jupyter_code>test_preprocess = transforms.Compose([
    transforms.ToTensor(),
    transforms.Resize(INPUT_SIZE),
    transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]),
])

torch_manager = DatasetManager(batch_size=BATCH_SIZE, test_size=TEST_SIZE, 
                        validation_size=VAL_SIZE, transform=None, 
                        test_transform=test_preprocess)<jupyter_output><empty_output><jupyter_text>Utility functions<jupyter_code>from sklearn.metrics import precision_score, recall_score

def calculate_precision_recall(y_true, y_pred):
    precisions = precision_score(y_true, y_pred, average=None)
    recalls = recall_score(y_true, y_pred, average=None)
    return precisions, recalls

def plot_result_barplot(results: List[np.ndarray], result_type: str, model_names: List[str], figsize, save_as: str):
    fig = plt.figure(figsize=figsize)
    width = 0.3
    for i, (result, name) in enumerate(zip(results, model_names)):
        plt.bar(np.arange(N_FEATURES) + i*width, result, width, label=f"{name}, mean: {round(result.mean(), 2)}")

    plt.title(result_type, fontsize=14)
    plt.ylabel('Probability')
    plt.xticks(np.arange(N_FEATURES) + width / 2, EMOTION_LIST)

    plt.legend(loc='best')
    
    plt.savefig('../reports/figures/' + save_as)<jupyter_output><empty_output><jupyter_text>Load models Load all models and convert data for the right format for each individual model  Resnet-50<jupyter_code>RESNET_NAME = 'resnet-50.pt'
device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')

resnet = models.resnet50(pretrained=True)
num_ftrs = resnet.fc.in_features
resnet.fc = nn.Sequential(
                nn.Linear(num_ftrs, 2048),
                nn.Dropout(0.5),
                nn.ReLU(),
                nn.Linear(2048, 1024),
                nn.Dropout(0.5),
                nn.ReLU(),
                nn.Linear(1024,  N_FEATURES)
)

checkpoint = torch.load(MODEL_PATH + RESNET_NAME)
# checkpoint = load_torch_model(RESNET_NAME)
resnet.load_state_dict(checkpoint['model_state_dict'])
resnet.to(device)
resnet.eval()<jupyter_output><empty_output><jupyter_text>Traditional models<jupyter_code>from joblib import load
cnn_root_folder = MODEL_PATH + 'cnn_features/'
hog_root_folder = MODEL_PATH + 'hog_features/'
pca_root_folder = MODEL_PATH + 'pca_features/'

cnn_models = [load(cnn_root_folder + name) for name in os.listdir(cnn_root_folder)]
hog_models = [load(hog_root_folder + name) for name in os.listdir(hog_root_folder)]
pca_models = [load(pca_root_folder + name) for name in os.listdir(pca_root_folder)]<jupyter_output><empty_output><jupyter_text>View the performance  Resnet-50 For evaluation we need test dataloader<jupyter_code>_, test_loader, _ = torch_manager.load_dataloaders()<jupyter_output><empty_output><jupyter_text>Predict for the whole test set.<jupyter_code>predictions = []
with torch.no_grad():
    for batch in tqdm(test_loader):
        X, y = batch[0].to(device), batch[1].to(device)

        outputs = resnet(X)
        preds = outputs.topk(1)[-1]
        y = y.long()
        
        scores = {'predictions': preds, 'true': y}
        predictions.append(scores)
y_true = [element.detach().cpu().numpy() for array in predictions for element in array['true']]
y_pred = [element.detach().cpu().numpy() for array in predictions for element in array['predictions']]
y_true = np.array(y_true)
y_pred = np.array(y_pred)
from sklearn.metrics import accuracy_score
accuracy_score(y_true, y_pred)<jupyter_output><empty_output><jupyter_text>Calculate precision and recall.<jupyter_code>y_true = [element.detach().cpu().numpy() for array in predictions for element in array['true']]
y_pred = [element.detach().cpu().numpy() for array in predictions for element in array['predictions']]
y_true = np.array(y_true)
y_pred = np.array(y_pred)
resnet_preds = y_pred.squeeze()
class_precisions, class_recalls = calculate_precision_recall(y_true, y_pred)

plot_result_barplot([class_precisions, class_recalls], 'ResNet precision-recall', ['precision', 'recall'], save_as='resnet-50-of-prec-recall', figsize=(16,10))<jupyter_output><empty_output><jupyter_text>From the plots we see that precision and recall are often closely related. However on disgust it seems like our classifier is exeptionally good with recall. In training I used perfectly balanced weights on inverse to class proportion, so this might have emphasized the prediction of disgust more than necessary.  **Misclassifications** Let's inspect what misclassifications are the most common. Does the model mix angry and fearful expression? Let's take a look.<jupyter_code>fig, ax = plt.subplots(3, 3, figsize=(20, 16))
i = 0
j = 0
for category in range(N_FEATURES):
    class_indices = np.where(y_true == category)[0]
    preds = y_pred[class_indices]
    
    preds_hist = np.histogram(preds, bins=N_FEATURES)[0]
    ax[i, j].set_title(f"Predictions for emotion: {EMOTION_LIST[category]}")
    ax[i, j].bar(EMOTION_LIST, preds_hist)
    ax[i, j].set_ylabel("Count of occurences")
    
    if j < 2:
        j += 1
    else:
        i += 1
        j = 0

ax[2,1].set_axis_off()
ax[2,2].set_axis_off()
plt.savefig(FIGURE_PATH + 'emotion-prediction-of-distributions.png')<jupyter_output><empty_output><jupyter_text>We see that hard emotions for humans are also hard for the network - Anger and fearness tend to mix between each other, and also sad and neutral.  Let's investige whether we find the reason why these feelings tend to mix Lime is great for easy and fast prediction summarization, but it has its own drawbacks. Lime uses linear predictors to define which local areas are significant for predicting the value. This assumes that the local areas have linear predictability, which might not be true in the real world.<jupyter_code>from lime import lime_image
explainer = lime_image.LimeImageExplainer()<jupyter_output><empty_output><jupyter_text>Lime doesn't work with pytorch out of the box, because it expects numpy array. We need to create our own batch generator function which processes the image.<jupyter_code>def torch_image_to_tensor(im):
    image = im.permute(1,2,0).numpy()
    image = image.astype(np.double)
    return image
    
def single_image_predict(imgs):
    torch_image = torch.tensor(imgs)
    torch_image = torch_image.type(torch.FloatTensor)
    torch_image = torch_image.permute(0,3,1,2)
    imgs = torch_image.to(device)

    logits = resnet(imgs)
    probs = F.softmax(logits, dim=1)
    return probs.detach().cpu().numpy()<jupyter_output><empty_output><jupyter_text>Load all of the images to pick certain label to describe<jupyter_code>torch_images = next(iter(test_loader))[0]
explanations = []
for img in torch_images:
    single_image = torch_image_to_tensor(img)
    explanation = explainer.explain_instance(single_image, single_image_predict, num_samples=3000)
    explanations.append(explanation)<jupyter_output><empty_output><jupyter_text>Visualize prediction. Only show what areas affect prediction positively with green, and negatively with red.Images are normalized for CNN, so we need to do inverse_normalization to get clearer images<jupyter_code>inv_normalize = transforms.Normalize(
    mean=[-0.485/0.229, -0.456/0.224, -0.406/0.255],
    std=[1/0.229, 1/0.224, 1/0.255]
)
from skimage.segmentation import mark_boundaries

for i, explanation in enumerate(explanations):
    temp, mask = explanation.get_image_and_mask(explanation.top_labels[0], positive_only=False, 
                                                num_features=7, hide_rest=False)
    temp_tensor = inv_normalize(torch.tensor(temp).permute(2,0,1)) # Convert to shape (3, 224, 224) for torch
    temp = temp_tensor.permute(1,2,0).numpy() # Flip image back to shape (224, 224, 3) and set as np.ndarray
    img_boundry1 = mark_boundaries(temp, mask, mode='inner')
    plt.figure()
    plt.imshow(img_boundry1)
    plt.title(f"Prediction 1: {EMOTION_LIST[explanation.top_labels[0]]}")
    plt.savefig(f'../reports/figures/pred_{EMOTION_LIST[explanation.top_labels[0]]}_true_angry_{i}')
    plt.show()<jupyter_output>Clipping input data to the valid range for imshow with RGB data ([0..1] for floats or [0..255] for integers).<jupyter_text>Traditional models The performance and explainability is a whole different topic with the other models that I trained. Now as the input images are turned in to embeddings, we can't define the  Define the order in which the models are stored. This will be hard coded, and depends on the order that the models are stored in folder. This is risky if it would be put to production, but in this case it's just for convenience.<jupyter_code>model_names = ['knn', 'LR', 'rbf-SVC', 'SGD', 'linear-SVC', 'XGBoost']<jupyter_output><empty_output><jupyter_text>**EfficientNet embeddings** *NOTE:* loading the embeddings might take a minute.<jupyter_code>X_train_embed, y_train_cnn = load_cnn_embedding('train')
X_test_embed, y_test_cnn = load_cnn_embedding('test')
X_val_embed, y_val_cnn = load_cnn_embedding('val')

pca, X_train_cnn, X_test_cnn, X_val_cnn = fit_pca(X_train_embed, X_test_embed, X_val_embed, n_components=200)<jupyter_output><empty_output><jupyter_text>**PCA embeddings**<jupyter_code>pca, X_train_pca, X_test_pca, X_val_pca = fit_pca(X_train, X_test, X_val, n_components = 50)<jupyter_output><empty_output><jupyter_text>**HoG embeddings**<jupyter_code>from skimage.feature import hog
from skimage import data, exposure

ppc = 10
train_imgs = [hog(x, orientations=8, pixels_per_cell=(ppc, ppc)) for x in X_train]
test_imgs = [hog(x, orientations=8, pixels_per_cell=(ppc, ppc)) for x in X_test]
val_imgs = [hog(x, orientations=8, pixels_per_cell=(ppc, ppc)) for x in X_val]

X_train_hog = np.vstack(train_imgs)
X_test_hog = np.vstack(test_imgs)
X_val_hog = np.vstack(val_imgs)<jupyter_output><empty_output><jupyter_text>**Do predictions**<jupyter_code>cnn_predictions = [{model_names[i]: model.predict(X_test_cnn)} for i, model in enumerate(cnn_models)]
pca_predictions = [{model_names[i]: model.predict(X_test_pca)} for i, model in enumerate(pca_models)]
hog_predictions = [{model_names[i]: model.predict(X_test_hog)} for i, model in enumerate(hog_models)]<jupyter_output><empty_output><jupyter_text>**Calculate precision-recall** Utility function<jupyter_code>def plot_precision_recall(predictions, name):
    values = [list(*clf.values()) for clf in predictions]
    names = [list(clf.keys())[0] for clf in predictions]

    class_precisions = [calculate_precision_recall(y_test_cnn, y_p)[0] for y_p in values]
    class_recalls = [calculate_precision_recall(y_test_cnn, y_p)[1] for y_p in values]
    
    print(class_precisions)
    plot_result_barplot(class_precisions, f'Precision with {name} features', model_names=names, 
                        figsize=(16,6), save_as=f'traditional_precisions_{name}.png')
    plot_result_barplot(class_recalls, f'Recall with {name} features', model_names=names, 
                        figsize=(16,8), save_as=f'traditional_recalls_{name}.png')<jupyter_output><empty_output><jupyter_text>CNN features<jupyter_code>plot_precision_recall(cnn_predictions, 'CNN')<jupyter_output>[array([0.31440162, 0.21978022, 0.4       , 0.36172345, 0.34308511,
       0.56794425, 0.41127349]), array([0.37161085, 0.10067114, 0.33825702, 0.66503667, 0.38257576,
       0.54573171, 0.44429882]), array([0.40491803, 0.66666667, 0.38983051, 0.63795424, 0.39215686,
       0.6742268 , 0.44851259]), array([0.3151751 , 0.125     , 0.31866197, 0.59810726, 0.34149856,
       0.45571096, 0.41761827]), array([0.36538462, 0.06982544, 0.39702233, 0.61333333, 0.40294118,
       0.4857513 , 0.43455497]), array([0.49501661, 1.        , 0.49560117, 0.4894636 , 0.36610879,
       0.64570231, 0.42677346])]<jupyter_text>PCA features<jupyter_code>plot_precision_recall(pca_predictions, 'PCA')<jupyter_output>[array([0.24744027, 0.1496063 , 0.24169184, 0.41166078, 0.28830313,
       0.52531646, 0.28256705]), array([0.2224622 , 0.03891051, 0.20668693, 0.50047125, 0.29350649,
       0.34623894, 0.31940701]), array([0.31473534, 0.44827586, 0.296875  , 0.56630037, 0.3183432 ,
       0.59357278, 0.3702381 ]), array([0.14149822, 0.        , 0.25      , 0.39724981, 0.18933624,
       0.29230769, 0.21956769]), array([0.24545455, 0.04213483, 0.19496855, 0.45276074, 0.29699666,
       0.30642202, 0.31195335]), array([0.41082803, 1.        , 0.38235294, 0.43188137, 0.3375576 ,
       0.63265306, 0.38328912])]<jupyter_text>HOG features<jupyter_code>plot_precision_recall(hog_predictions, 'HOG')<jupyter_output>[array([0.31019202, 0.26785714, 0.29684601, 0.43842837, 0.38215103,
       0.45663717, 0.36966127]), array([0.28148148, 0.07252747, 0.24632353, 0.57973568, 0.28947368,
       0.4047619 , 0.3522316 ]), array([0.34477825, 0.74358974, 0.38125   , 0.5988935 , 0.36197022,
       0.63461538, 0.42770352]), array([0.24311927, 0.09251101, 0.19158361, 0.58202443, 0.23894863,
       0.48807339, 0.25961538]), array([0.29558541, 0.06603774, 0.26268657, 0.51387054, 0.28326996,
       0.36403996, 0.36044881]), array([0.43342037, 1.        , 0.42469136, 0.49271723, 0.34547461,
       0.62553191, 0.39678899])]<jupyter_text>Calculate F1 score for each model F1-score is a great balance between precision and recall. We can evaluate it on every classifier to see how well they compare against each other<jupyter_code>from sklearn.metrics import f1_score
cnn_scores = [f1_score(list(*y_p.values()), y_test_cnn, average='weighted') for y_p in cnn_predictions]
pca_scores = [f1_score(list(*y_p.values()), y_test, average='weighted') for y_p in pca_predictions]
hog_scores = [f1_score(list(*y_p.values()), y_test, average='weighted') for y_p in hog_predictions]
resnet_scores = f1_score(resnet_preds, y_test, average='weighted')
fig = plt.figure(figsize=(14,7))
width = 0.15
plt.title("F1 scores", fontsize=18)
plt.bar(np.arange(6), cnn_scores, width, label="CNN")
plt.bar(np.arange(6) + width, pca_scores, width, label="PCA")
plt.bar(np.arange(6) + 2*width, hog_scores, width, label="HOG")
plt.bar(np.arange(6) + 3*width + 0.3*width, resnet_scores, width, label="RESNET")
plt.xticks(np.arange(N_FEATURES) + width / 2, model_names)
plt.ylabel("Probability")
plt.xlabel("Classifier")
plt.legend(loc='best')
plt.savefig('../reports/figures/traditional_f1.png')<jupyter_output><empty_output><jupyter_text>It seems like linear regression with CNN embeddings tend to work well. Linear regression is also the fastest algorithm among these, so it provides nice results in conclusion.  Ensembles Let's see whether we could improve performance by aggregating all of the classifiers, and getting ensemble predictions. Theory behind ensemble models is that each model would train independently, thus each model could possibly complement the other models predictions. Define function that takes the majority vote on columns. This is the same as taking majority vote on classifier outputs for each sample.<jupyter_code>def most_frequent(x):
    return np.bincount(x).argmax()

def calculate_majority_voting(predictions, np_array=False):
    if not np_array:
        predictions = np.array([list(*y_p.values()) for y_p in predictions]).astype(int)
    embed_preds = np.apply_along_axis(most_frequent, 0, predictions)
    return embed_preds

cnn_embed_predictions = calculate_majority_voting(cnn_predictions)
pca_embed_predictions = calculate_majority_voting(pca_predictions)
hog_embed_predictions = calculate_majority_voting(hog_predictions)

cnn_embed_f1 = f1_score(cnn_embed_predictions, y_test_cnn, average='weighted')
pca_embed_f1 = f1_score(pca_embed_predictions, y_test, average='weighted')
hog_embed_f1 = f1_score(hog_embed_predictions, y_test, average='weighted')
print(f"""
F1-score for majority votes, 
CNN: {round(cnn_embed_f1, 2)}, 
PCA: {round(pca_embed_f1, 2)},
HOG: {round(hog_embed_f1, 2)}
       """)

fig = plt.figure(figsize=(12,7))
width = 0.15
plt.title("F1 scores", fontsize=18)
plt.bar(np.arange(6), cnn_scores, width, label="CNN")
plt.bar(np.arange(6) + width, pca_scores, width, label="PCA")
plt.bar(np.arange(6) + 2*width, hog_scores, width, label="HOG")
plt.bar(np.arange(6) + 3*width + 0.3*width, resnet_scores, width, label="RESNET")
plt.xticks(np.arange(N_FEATURES) + width / 2, model_names)
plt.ylabel("Probability")
plt.xlabel("Classifier")
plt.legend(loc='best')<jupyter_output>F1-score for majority votes, 
CNN: 0.51, 
PCA: 0.42,
HOG: 0.44<jupyter_text>It seems like each prediction got slightly higher scores than individually. We can then try taking ensemble of all models.<jupyter_code>all_preds = np.vstack([cnn_embed_predictions, pca_embed_predictions, hog_embed_predictions, resnet_preds])
all_preds_traditional = np.vstack([cnn_embed_predictions, pca_embed_predictions, hog_embed_predictions])

all_embed_preds = calculate_majority_voting(all_preds, np_array=True)
all_embed_preds_traditional = calculate_majority_voting(all_preds_traditional, np_array=True)

all_embed_f1 = f1_score(all_embed_preds, y_test, average='weighted')
all_embed_f1_traditional = f1_score(all_embed_preds_traditional, y_test, average='weighted')
print(f"F1-score for majority voting of all classifiers: {round(all_embed_f1, 2)}")
print(f"F1-score for majority voting of traditional classifiers: {round(all_embed_f1_traditional, 2)}")
print(f"F1-score for majority voting of traditional classifiers: {round(f1_score(resnet_preds, y_test, average='weighted'), 2)}")<jupyter_output>F1-score for majority voting of traditional classifiers: 0.57<jupyter_text>Project evaluation metrics **Project will be evaluated based on accuracy, so now we must calculate it**<jupyter_code>from sklearn.metrics import accuracy_score
ensemble_accuracy = accuracy_score(y_test, all_embed_preds)
ensemble_accuracy_traditional = accuracy_score(y_test, all_embed_preds_traditional)
resnet_accuracy = accuracy_score(y_test, resnet_preds)
rbf_svm_accuracy = accuracy_score(y_test, list(*cnn_predictions[2].values()))
print(f"""
Ensemble w/resnet accuracy: {round(ensemble_accuracy, 3)}, 
Ensemble no resnet accuracy: {round(ensemble_accuracy_traditional, 3)}, 
Resnet accuracy: {round(resnet_accuracy, 3)}, 
RBF SVM accuracy: {round(rbf_svm_accuracy, 3)}
""")<jupyter_output>Ensemble w/resnet accuracy: 0.541, 
Ensemble no resnet accuracy: 0.484, 
Resnet accuracy: 0.567, 
RBF SVM accuracy: 0.504
{'fraction_non_alphanumeric': 0.11449475973897567, 'fraction_numerical': 0.06812339331619537, 'mean_word_length': 5.444409047467346, 'pattern_counts': {'":': 0, '<': 107, '<?xml version=': 0, '>': 106, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 1, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '28', 'dir': 'jupyter-structured-clean-dedup', 'id': '438165', 'license': 'MIT', 'n_tokens_mistral': 8128, 'n_tokens_neox': 7121, 'n_words': 1625, 'path': 'notebooks/0.3-visualization.ipynb', 'repo_name': 'TeemuSo/Loop-Q-prize', 'size': '20228'}