<jupyter_start><jupyter_text><jupyter_code>import pandas as pd
import matplotlib.pyplot as plt
import numpy as np
import plotly.express as px
from google.colab import drive
drive.mount('/content/drive')<jupyter_output>Mounted at /content/drive<jupyter_text>Data is from https://www.kaggle.com/c/challenges-in-representation-learning-facial-expression-recognition-challenge/dataThe ```icml_face_data.csv``` dataset which contains a total of 34034 samples. It contains 3 columns. The target label is an integer (0, 6), the seven basic emtions:```['Angry', 'Disgust', 'Fear', 'Happy', 'Sad', 'Surprise', 'Neutral']```The images are stored in the 'pixels' column, as a string of 2304 integer values (0, 255) separated by spaces. These will need be coverted into an array of integer values. Subsequently, they will be needs reshaped into 3-d numpy arrays to be input to the CNN.<jupyter_code>def loadCleanData(fname):
    df = pd.read_csv(fname)
    
    # regularize capitalization and remove leading space in column headings
    fix_cols = {s : s.lower().strip() for s in df.columns.to_list()}
    df.rename(columns = fix_cols, inplace=True)
    
    # convert pixels from strings to lists of integers
    df['pixels'] = df['pixels'].apply(lambda x: [int(i) for i in x.split()])
    
    # Not using the 'usage' column, so can be dropped
    df.drop('usage', axis=1, inplace=True)
    
    return df
df = loadCleanData('/content/drive/MyDrive/face express data/icml_face_data.csv')
df.head()
df_labels = pd.read_csv('https://raw.githubusercontent.com/microsoft/FERPlus/master/fer2013new.csv')
df_labels.head()
df_labels = df_labels[['neutral', 'happiness', 'surprise', 'sadness', 'anger', 'disgust', 'fear', 'contempt', 'unknown', 'NF']]
fer2013_new = pd.concat([df, df_labels], axis=1)
fer2013_new = fer2013_new[fer2013_new['NF'] < 10]

fer2013_new.head()
# Select all but NF for use
y = fer2013_new[['neutral', 'happiness', 'surprise', 'sadness', 'anger', 'disgust', 'fear', 'contempt']].values

# Majority rule
def makeClasses(votes):
  probs = []
  for row in range(votes.shape[0]):
    # put any contempt votes to disgust
    votes[row][5] += votes[row][7]
    y = np.argmax(votes[row][:7])
    probs.append(y)

  return np.array(probs)
classes = ['Neutral', 'Happy', 'Surprise', 'Sad', 'Angry', 'Disgust', 'Fear']
fer2013_new['emotion'] = makeClasses(y)
fer2013_new.head(10)
import dlib
from imutils import face_utils

!wget https://github.com/italojs/facial-landmarks-recognition/blob/master/shape_predictor_68_face_landmarks.dat?raw=true
predictor = dlib.shape_predictor("shape_predictor_68_face_landmarks.dat?raw=true")

def align(img):
  image = np.array(img, dtype=np.uint8).reshape(48, 48)
  fa = face_utils.FaceAligner(predictor, desiredLeftEye=(.23, .23), desiredFaceWidth=48)
  rect = dlib.rectangle(0, 0, 48, 48)
  aligned_img = fa.align(image, image, rect)
  return aligned_img

img = align(df.iloc[8]['pixels'])
plt.imshow(img, cmap='gray')
predictor = dlib.shape_predictor("shape_predictor_68_face_landmarks.dat?raw=true")

def vectorizeFace(image):
  img = align(image)
  rect = dlib.rectangle(0, 0, 48, 48)
  shapes = predictor(img, rect)
  vect = list(face_utils.shape_to_np(shapes).reshape(136))
  return vect
fer2013_new['vector'] = fer2013_new['pixels'].apply(vectorizeFace)
fer2013_new.head()
def visualizeFaceVector(row):
  vector = fer2013_new.iloc[row]['vector']
  vec = np.array(vector).reshape(len(vector) // 2, 2)
  x = [vec[i][0] for i in range(vec.shape[0])]
  y = [vec[i][1] for i in range(vec.shape[0])]
  img = align(fer2013_new.iloc[row]['pixels'])
  plt.figure(figsize=(5,5))
  plt.imshow(img, cmap='gray')
  plt.scatter(x, y, color='red')
  plt.title(classes[fer2013_new.iloc[row]['emotion']])
  plt.show()
visualizeFaceVector(8)
from sklearn.model_selection import train_test_split

from tensorflow.keras.layers import Dense, Input
from tensorflow.keras.models import Sequential
from tensorflow.keras.callbacks import EarlyStopping, ModelCheckpoint, ReduceLROnPlateau
from tensorflow.keras.layers.experimental.preprocessing import Normalization
from tensorflow.keras.optimizers import Adam, RMSprop, SGD, Adagrad
x_train, x_test, y_train, y_test = train_test_split(fer2013_new['vector'],
                                                    fer2013_new['emotion'],
                                                    test_size=0.2,
                                                    stratify=fer2013_new['emotion'],
                                                    random_state=42)

x_train.shape, x_test.shape, y_train.shape, y_test.shape<jupyter_output><empty_output><jupyter_text><jupyter_code># transform lists of vector values into numpy arrays
x_train = np.vstack(x_train)
x_test = np.vstack(x_test)

# Convert the pandas series of target classes to numpy array
y_train = np.asarray(y_train)
y_test = np.asarray(y_test)

x_train.shape, y_train.shape
model = Sequential([Input(shape=x_train.shape[1:]),
                    Normalization(),
                    Dense(512, activation='relu'),
                    Dense(256, activation='relu'),
                    Dense(128, activation='relu'),
                    Dense(64, activation='relu'),
                    Dense(7, activation='softmax')])

model.compile(optimizer=Adam(lr=.001), loss='sparse_categorical_crossentropy', metrics=['acc'])
model.summary()
stop = EarlyStopping(monitor='val_acc', patience=5, restore_best_weights=True)
reduce_lr = ReduceLROnPlateau(monitor='val_loss', factor=0.9, patience=2, min_delta=.001)

history = model.fit(x_train, 
          y_train,
          validation_data=(x_test, y_test),
          epochs=100, 
          batch_size=32, 
          callbacks=[stop, reduce_lr])
plt.plot(history.history['loss'], label='training')
plt.plot(history.history['val_loss'], label='validation')
plt.legend()
plt.show()

plt.plot(history.history['acc'], label='training')
plt.plot(history.history['val_acc'], label='validation')
plt.legend()
plt.show()
score = model.evaluate(x_test, y_test)[1]
print(f'Accuracy on test {round(score * 100, 2)} %')
from sklearn.metrics import confusion_matrix
import seaborn as sns

preds = model.predict(x_test)
y_pred = [np.argmax(pred) for pred in preds]
cm = confusion_matrix(y_test, y_pred)

ax = sns.heatmap(cm, annot=True, fmt='d', linewidths=.2, square=True)
bottom, top = ax.get_ylim()
ax.set_ylim(bottom + 0.5, top - 0.5);
classes
def showSample():
  index = np.random.randint(0, fer2013_new.shape[0])
  vector = np.array(fer2013_new.iloc[index]['vector'])

  y_preds = model([vector])
  predict = np.argmax(preds[0])
  prob = float(y_preds[0][predict])

  print(predict == fer2013_new.iloc[index]['emotion'])
  print(f'Predicted {classes[predict]}')
  print(f'Confidence {round(prob * 100, 2)} %')
  print(f'Actual {classes[fer2013_new.iloc[index]["emotion"]]}')

  vec = np.array(vector).reshape(len(vector) // 2, 2)
  x = [vec[i][0] for i in range(vec.shape[0])]
  y = [vec[i][1] for i in range(vec.shape[0])]
  img = align(fer2013_new.iloc[index]['pixels'])
  plt.figure(figsize=(5,5))
  plt.imshow(img, cmap='gray')
  plt.scatter(x, y, color='red')
  plt.show()
showSample()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1464026770775237, 'fraction_numerical': 0.03053541550474066, 'mean_word_length': 4.373033707865169, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 12, 'https://': 3, 'lorem ipsum': 0, 'www.': 1, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 8, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '402761', 'license': 'MIT', 'n_tokens_mistral': 2673, 'n_tokens_neox': 2475, 'n_words': 527, 'path': 'notebooks/vector_model1.ipynb', 'repo_name': 'karencfisher/face_express', 'size': '7172'}