<jupyter_start><jupyter_text>Speech Emotion Recognition using CNN  How do I apply CNN to audio? * Audio can be thought of an image > Spectogram/MFCC = image* time frequency = x,y* amplitude = pixel value* 13 MFCCs, hop_length=512* Audio data can be compared to grey scale image, depth=1<jupyter_code>import os
import librosa
import math
import json
import numpy as np
from sklearn.model_selection import train_test_split
import tensorflow as tf
import matplotlib.pyplot as plt
from sklearn.metrics import confusion_matrix
# these lines were required as keras could not identify my system GPU
import os
os.environ["CUDA_VISIBLE_DEVICES"]="-1"
# loading mfcc data > conversion of audio files to mfcc is done in the preprocessing of audio notebook
def load_data(dataset_path):
    with open(dataset_path,'r') as fp:
        data = json.load(fp)
        
    #convert lists to numpy arrays
    inputs = np.array(data["mfcc"])
    targets = np.array(data["labels"])
    
    return inputs,targets
JSON_PATH = "./outputs/ravdess_complete.json"
# load the data
X,y = load_data(JSON_PATH)
X.shape,y.shape
#Emotion (0 = neutral, 1 = calm, 2 = happy, 3 = sad, 4 = angry, 5 = fearful, 6 = disgust, 7 = surprised)
# selecting only calm,happy,angry 

X = X[(y!=0)]
y = y[(y!=0)]
X = X[(y!=3)]
y = y[(y!=3)]
X = X[(y<=4)]
y = y[(y<=4)]
# So I have now 2,3 and 4 in the training > it needs to be 0,1,2
y = np.where(y==1, 0, y)
y = np.where(y==2, 1, y)
y = np.where(y==4, 2, y)
y.shape, X.shape
# shape of each input entry : (number of samples per second/hop_length)*duration
def prepare_data(X,y,test_size,validation_size):
    
    # splitting data into train and test set
    X_train,X_test,y_train,y_test = train_test_split(X,y,test_size=test_size)
    
    # splitting data into train and validation set
    X_train,X_validation,y_train,y_validation = train_test_split(X,y,test_size=validation_size)
    
    # add extra dimension , as sound files are treated like greyscale images
    X_train = X_train[...,np.newaxis] #4d array num_samples x 130 x 13 x 1
    X_test = X_test[...,np.newaxis]
    X_validation = X_validation[...,np.newaxis]
    return X_train,X_validation,X_test,y_train,y_validation,y_test
# splitting data into train,validation and test set
X_train,X_validation,X_test,y_train,y_validation,y_test = prepare_data(X,y,test_size=0.25,validation_size=0.2)
X_train.shape
def build_model(input_shape):
    #create model
    model = tf.keras.Sequential()
    #model.add(tf.keras.layers.BatchNormalization())
    
    #1st conv layer
    model.add(tf.keras.layers.Conv2D(8,(3,3),activation="relu",input_shape=input_shape))
    model.add(tf.keras.layers.MaxPool2D((3,3),strides=(2,2),padding='same'))
    model.add(tf.keras.layers.BatchNormalization())
    
    #2nd conv layer
    #model.add(tf.keras.layers.Conv2D(32,(3,3),activation="relu",input_shape=input_shape))
    #model.add(tf.keras.layers.MaxPool2D((3,3),strides=(2,2),padding='same'))
    #model.add(tf.keras.layers.BatchNormalization())

    #3rd conv layer
    #model.add(tf.keras.layers.Conv2D(32,(2,2),activation="relu",input_shape=input_shape))
    #model.add(tf.keras.layers.MaxPool2D((2,2),strides=(2,2),padding='same'))
    #model.add(tf.keras.layers.BatchNormalization())
    
    #flatten and feed to dense layer
    model.add(tf.keras.layers.Flatten())
    model.add(tf.keras.layers.Dense(64,activation="relu",input_shape=(3072,)))
    model.add(tf.keras.layers.Dropout(0.3))
    
    #output layer that feeds to softmax
    model.add(tf.keras.layers.Dense(3,activation="softmax"))
    
    return model
    
# building the CNN
input_shape = (X_train.shape[1],X_train.shape[2],X_train.shape[3])

model = build_model(input_shape)

model.compile(optimizer = tf.keras.optimizers.Adam(),loss="sparse_categorical_crossentropy",metrics=["accuracy"])

# fitting the model
history=model.fit(X_train,y_train,validation_data=(X_validation,y_validation),batch_size=32,epochs=100)
model.summary()
import pandas as pd
pd.DataFrame(history.history["accuracy"]).to_csv('CNN_train_acc.csv')
pd.DataFrame(history.history["val_accuracy"]).to_csv('CNN_val_acc.csv')
#Evaluating the model
test_error,test_accuracy = model.evaluate(X_test,y_test,verbose=1)
test_error,test_accuracy
#model.save('CNN_model_calm_happy_angry.h5')
def plot_history(history):
    fig, axs = plt.subplots(2,figsize=(12,8))
    
    #create the accuracy subplot
    axs[0].plot(history.history["accuracy"],label="train accuracy")
    axs[0].plot(history.history["val_accuracy"],label="test accuracy")
    axs[0].set_ylabel("Accuracy")
    axs[0].legend(loc="lower right")
    axs[0].set_title("Accuracy eval")
    
    #create the error subplot
    axs[1].plot(history.history["loss"],label="train error")
    axs[1].plot(history.history["val_loss"],label="test error")
    axs[1].set_ylabel("Error")
    axs[1].set_xlabel("Epoch")
    axs[1].legend(loc="upper right")
    axs[1].set_title("Error eval")
    
    plt.show()   
    
plot_history(history)<jupyter_output><empty_output><jupyter_text>Evaluating the model performance<jupyter_code>def predict(model,X):
    #X = X[np.newaxis,...] #size of X (1,130,13,1) [Conversion to be done if entering one value],
    prediction = model.predict(X) #prediction contains 3 values - probability for each genre [[0.1, 0.2, ..]],
    #print(prediction),
    #extract index with max value,
    predicted_index = np.argmax(prediction,axis=1)
    return predicted_index
y_pred = predict(model,X_test)
confusion_matrix(y_test,y_pred)
y_train_pred = predict(model,X_train)
y_validation_pred = predict(model,X_validation)
confusion_matrix(y_train,y_train_pred)
confusion_matrix(y_validation,y_validation_pred)
# Emotions : calm : 0,happy : 1,angry : 2<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.13766098155238427, 'fraction_numerical': 0.022102332057083188, 'mean_word_length': 5.055848261327713, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 12, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '348159', 'license': 'MIT', 'n_tokens_mistral': 2118, 'n_tokens_neox': 2051, 'n_words': 413, 'path': 'SER_training_the_CNN_model.ipynb', 'repo_name': 'Jamsheeda-K/Speech_emotion_recognition', 'size': '5746'}