<jupyter_start><jupyter_text>Split and shuffle data<jupyter_code>directory = "recordings_converted"

# load all file names and generate paths
all_raw_data_files = os.listdir(directory)
all_raw_data_paths = [os.path.join(directory, raw_data_file) for raw_data_file in all_raw_data_files]

np.random.shuffle(all_raw_data_paths)

# Calculate number of files on each dataset
num_train_files = int(0.8 * len(all_raw_data_paths))
num_valid_files = int(0.1 * len(all_raw_data_paths))

# Split all data into train,valid and test set
train_ls = all_raw_data_paths[:num_train_files]
valid_ls = all_raw_data_paths[num_train_files:num_train_files + num_valid_files]
test_ls = all_raw_data_paths[num_train_files + num_valid_files:]

print('Raw files: {}'.format(len(all_raw_data_paths)))

print('Training set: {}'.format(len(train_ls)))
print('Validation set: {}'.format(len(valid_ls)))
print('Test set: {}'.format(len(test_ls)))<jupyter_output>Raw files: 1990
Training set: 1592
Validation set: 199
Test set: 199<jupyter_text>Plot raw mfcc<jupyter_code>#Get raw .wav data and sampling rate from librosa's load function
raw_w, sampling_rate = librosa.load(train_ls[5], mono=True)
    
print('Raw data shape: {}'.format(raw_w.shape))
    
#Obtain MFCC Features from raw data
mfcc_features = librosa.feature.mfcc(y=raw_w, sr=sampling_rate)
    
print('Selected utterance length: {}'.format(FRAME_SIZE))
print('Shape of MFCC features before cutting or padding: {}'.format(mfcc_features.shape))
    
#Cut or pad
if mfcc_features.shape[1] > FRAME_SIZE:
    mfcc_features = mfcc_features[:, :FRAME_SIZE]
else:
    mfcc_features = np.pad(mfcc_features, ((0, 0), (0, FRAME_SIZE - mfcc_features.shape[1])),
                                   mode='constant', constant_values=0)
#Visualize of raw data
fig, ax = plt.subplots(1, figsize=(20, 6))
ax.set_axis_off()
plt.plot(raw_w)
plt.title('Time series of raw data')
plt.show()

#Visualize of extracted mfcc features
fig, ax = plt.subplots(1, figsize=(20, 6))
ax.set_axis_off()

librosa.display.specshow(mfcc_features)
plt.title('MFCC features')
plt.colorbar()
plt.show()<jupyter_output>Raw data shape: (9436,)
Selected utterance length: 30
Shape of MFCC features before cutting or padding: (20, 19)<jupyter_text>Extract mfcc features and labels<jupyter_code>def extract_mfcc_features_labels(file_paths, frame_size):
    """ Extract MFCC features and labels of all raw data

    Args:
    file_paths: List of paths of raw files.
    frame_size: Length of data to cut or pad.

    Returns:
    all_mfcc_features: 2D array, shape: (number of samples, number of mfcc features)
    all_labels: 2D array, shape: (number of samples, number of labels)
    """
    # Initialize the results
    all_mfcc_features = []
    all_labels = []

    # For loop through all files' paths
    for file_path in file_paths:
        
        # Get raw .wav data and sampling rate from librosa's load function
        raw_w, sampling_rate = librosa.load(file_path, mono=True)
    
        # Obtain MFCC Features from raw data
        mfcc_features = librosa.feature.mfcc(y=raw_w, sr=sampling_rate)
    
        # Cut or pad
        if mfcc_features.shape[1] > frame_size:
            mfcc_features = mfcc_features[:, :frame_size]
        else:
            mfcc_features = np.pad(mfcc_features, ((0, 0), (0, frame_size - mfcc_features.shape[1])),
                                   mode='constant', constant_values=0)
            
        mfcc_features = mfcc_features.flatten()

        # Extract labels from file names.
        basename = os.path.basename(file_path)

        # One-hot encoding for labels
        raw_label = int(basename.split('_')[0])
        label = np.eye(10)[raw_label]

        all_mfcc_features.append(mfcc_features)
        all_labels.append(label)

    # Convert results from list of arrays to 2d arrays
    all_mfcc_features = np.asarray(all_mfcc_features)
    all_labels = np.asarray(all_labels)

    print('all_mfcc_features: {}'.format(all_mfcc_features.shape))
    print('all_labels: {}'.format(all_labels.shape))

    return all_mfcc_features, all_labels
print('Extract training set...')
train_mfcc, train_label = extract_mfcc_features_labels(train_ls, FRAME_SIZE)

print('Extract validation set...')
valid_mfcc, valid_label = extract_mfcc_features_labels(valid_ls, FRAME_SIZE)

print('Extract test set...')
test_mfcc, test_label = extract_mfcc_features_labels(test_ls,FRAME_SIZE)
def build_model(input_shape, hidden_sizes, num_classes):
    """ 
    Build model given input shape, hidden sizes and number of classes. 

    Args:
    input_shape: Tuple
    hidden_sizes: List of ints
    num_classes: number of classes/labels to output.

    Return:
    Tensorfow Model
    """

    # Initialize the model
    model = tf.keras.Sequential(name='Speech_model')

    # Build hidden layers  dense softmax
    for i, hidden_size in enumerate(hidden_sizes):
        if i == 0:
            hidden_layer = tf.keras.layers.Dense(units=hidden_size, activation='tanh', input_dim=input_shape[1])
        else:
            hidden_layer = tf.keras.layers.Dense(units=hidden_size, activation='tanh')
        model.add(hidden_layer)

    # Build output layer
    output_layer = tf.keras.layers.Dense(units=num_classes, activation='softmax', name='output_layer')
    model.add(output_layer)

    # Print model summary
    print(model.summary())

    return model
train_mfcc.shape
model = build_model(input_shape=train_mfcc.shape, hidden_sizes=[4096, 1024, 128], num_classes=10)<jupyter_output>Model: "Speech_model"
_________________________________________________________________
 Layer (type)                Output Shape              Param #   
=================================================================
 dense_4 (Dense)             (None, 4096)              2461696   
                                                                 
 dense_5 (Dense)             (None, 1024)              4195328   
                                                                 
 dense_6 (Dense)             (None, 128)               131200    
                                                                 
 output_layer (Dense)        (None, 10)                1290      
                                                                 
=================================================================
Total params: 6,789,514
Trainable params: 6,789,514
Non-trainable params: 0
_________________________________________________________________
None<jupyter_text>Compile model<jupyter_code>model.compile(optimizer = tf.optimizers.Nadam(learning_rate=0.00001),
              loss=tf.keras.losses.categorical_crossentropy,
              metrics=[tf.keras.metrics.categorical_accuracy])
def train_model(model, train_x, train_y, valid_x, valid_y, epochs, verbose, model_save_path, patience):
    # Reduce learning rate if plateu 
    lr_monitor = tf.keras.callbacks.ReduceLROnPlateau(monitor="val_loss", patience=10, factor=0.5, cooldown=1)
    
    # Save the best weights during the training process
    model_checkpoint = tf.keras.callbacks.ModelCheckpoint(filepath=model_save_path, save_best_only=True)

    # Stop the training when validation loss starts increasing
    early_stop = tf.keras.callbacks.EarlyStopping(patience=patience, monitor="val_loss")

    # Train model
    history = model.fit(
        x=train_x, 
        y=train_y, 
        epochs=epochs, 
        verbose=verbose,
        callbacks=[lr_monitor, model_checkpoint, early_stop],
        validation_data=(valid_x, valid_y)
    )

    return model, history
model, history = train_model(model, train_mfcc, train_label, valid_mfcc, valid_label, 
                             epochs=5000,
                             verbose=1,
                             model_save_path="save/model.h5", 
                             patience=20)<jupyter_output>Epoch 1/5000
50/50 [==============================] - 2s 25ms/step - loss: 1.9669 - categorical_accuracy: 0.3681 - val_loss: 1.5978 - val_categorical_accuracy: 0.5477 - lr: 1.0000e-05
Epoch 2/5000
50/50 [==============================] - 1s 21ms/step - loss: 1.1739 - categorical_accuracy: 0.7268 - val_loss: 1.2293 - val_categorical_accuracy: 0.6784 - lr: 1.0000e-05
Epoch 3/5000
50/50 [==============================] - 1s 22ms/step - loss: 0.8473 - categorical_accuracy: 0.8285 - val_loss: 1.0090 - val_categorical_accuracy: 0.7538 - lr: 1.0000e-05
Epoch 4/5000
50/50 [==============================] - 1s 21ms/step - loss: 0.6507 - categorical_accuracy: 0.8932 - val_loss: 0.8589 - val_categorical_accuracy: 0.8191 - lr: 1.0000e-05
Epoch 5/5000
50/50 [==============================] - 1s 21ms/step - loss: 0.5177 - categorical_accuracy: 0.9328 - val_loss: 0.7456 - val_categorical_accuracy: 0.8543 - lr: 1.0000e-05
Epoch 6/5000
50/50 [==============================] - 1s 21ms/step - loss: 0.420[...]<jupyter_text>Load trained model<jupyter_code>model = tf.keras.models.load_model("save/model.h5")<jupyter_output><empty_output><jupyter_text>Visualize loss and accuracy<jupyter_code>#Get loss and accuracy on training set
train_loss = history.history['loss']
train_acc = history.history['categorical_accuracy']

#Get loss and accuracy on training set
valid_loss = history.history['val_loss']
valid_acc = history.history['val_categorical_accuracy']

#Get training epochs
num_epochs = len(train_loss)

fig, ax = plt.subplots(1, figsize=(10, 5))
plt.plot(train_loss)
plt.plot(valid_loss)
plt.legend(['train_loss', 'valid_loss'], fontsize='large')
plt.xlabel('Epochs')
plt.ylabel('Loss')

fig, ax = plt.subplots(1, figsize=(10, 5))
plt.plot(train_acc, linestyle='--')
plt.plot(valid_acc, linestyle='--')
plt.legend(['train_accuracy', 'valid_accuracy'], fontsize='large')
plt.xlabel('Epochs')
plt.ylabel('Accuracy')
plt.show()<jupyter_output><empty_output><jupyter_text>Evaluate model<jupyter_code>loss, acc = model.evaluate(test_mfcc, test_label)

print('Test loss: {}'.format(loss))
print('Test accuracy: {}'.format(acc))
FILE_PATH = test_ls[9]

FILE_PATH = "chunk9.wav"

#Get raw .wav data, sampling rate from librosa's load function
raw_w, sampling_rate = librosa.load(FILE_PATH, mono=True)
    
#Obtain MFCC Features from raw data
mfcc_features = librosa.feature.mfcc(y=raw_w, sr=sampling_rate)

#Cut or pad
if mfcc_features.shape[1] > FRAME_SIZE:
    mfcc_features = mfcc_features[:, :FRAME_SIZE]
else:
    mfcc_features = np.pad(mfcc_features, 
                           ((0, 0), (0, FRAME_SIZE - mfcc_features.shape[1])),
                           mode='constant', 
                           constant_values=0)
    
#Visualize raw data
fig, ax = plt.subplots(1, figsize=(20, 6))
ax.set_axis_off()
plt.plot(raw_w)
plt.title('Time series of raw data')
plt.show()

#Visualize extracted mfcc features
fig, ax = plt.subplots(1, figsize=(20, 6))
ax.set_axis_off()

librosa.display.specshow(mfcc_features)
plt.title('MFCC features')
plt.colorbar()
plt.show()

#Extract MFCC features from single file path
mfcc_features = mfcc_features.flatten()

# Reshape mfcc_features to meet the requirements 1D numpy array, shape: (1, mfcc features)
mfcc_features = mfcc_features.reshape((1, -1))

#Predict label
pred_label = np.argmax(model.predict(mfcc_features))

print('File loaded: {}'.format(FILE_PATH))
print('Predicted label: {}'.format(pred_label))<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.13423471191233652, 'fraction_numerical': 0.033934252386002124, 'mean_word_length': 3.566989507667474, 'pattern_counts': {'":': 0, '<': 25, '<?xml version=': 0, '>': 28, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 10, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '7', 'dir': 'jupyter-structured-clean-dedup', 'id': '167565', 'license': 'MIT', 'n_tokens_mistral': 4047, 'n_tokens_neox': 3671, 'n_words': 805, 'path': 'Digit Recognition Neural Network.ipynb', 'repo_name': 'GeorgeCodeHub/Neural-Networks-for-digit-speech-recognition', 'size': '11316'}