<jupyter_start><jupyter_text>Urban Sound Tinkering w/ Pickles By: <NAME> The purpose of this notebook is to create the pickle files that will contain the 9-1 fold splits. Reading in .wav files, converting to spectrograms, and then running the model for each 9-1 split will take waaaay too long. This notebook also contains some minor exploration of audio at the beginning and some practice with OneHotEncoding towards the bottom.  Importing Libraries<jupyter_code>import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import tensorflow as tf
import librosa as lr
import librosa.display
from glob import glob
import IPython.display
from keras.utils import to_categorical
from keras.models import Sequential, Model
from keras.layers import Conv2D, MaxPool2D, Flatten, LSTM, Dropout, Dense
from keras.layers import Conv2D, AveragePooling2D
from keras.layers import MaxPooling2D, GlobalAveragePooling2D, BatchNormalization, Activation
from keras.callbacks import TensorBoard
from keras import regularizers, optimizers
from sklearn.metrics import confusion_matrix,classification_report
from sklearn import preprocessing
import pickle
import os
import seaborn as sns<jupyter_output><empty_output><jupyter_text>https://forums.fast.ai/t/deep-learning-with-audio-thread/38123 is super useful!<jupyter_code># Setting directory for wav files

# Using r takes in raw string (no need for // escape)
data_dir = r"D:/Machine_Learning/UrbanSound8K/UrbanSound8K/audio/fold1"

audio_files = glob(data_dir + '/*.wav')

len(audio_files)
print(audio_files[11])
audio, sfreq = lr.load(audio_files[55])

time = np.arange(0, len(audio)) / sfreq

time
sfreq
np.shape(audio)
fig, ax = plt.subplots()
ax.plot(time, audio)
ax.set(xlabel='Time (s)', ylabel='Sound Amplitude')
IPython.display.Audio(data=audio, rate=sfreq)
y, sr = lr.load(audio_files[55])
lr.feature.melspectrogram(y=y, sr=sr)
# array([[  2.891e-07,   2.548e-03, ...,   8.116e-09,   5.633e-09],
# [  1.986e-07,   1.162e-02, ...,   9.332e-08,   6.716e-09],
# ...,
# [  3.668e-09,   2.029e-08, ...,   3.208e-09,   2.864e-09],
# [  2.561e-10,   2.096e-09, ...,   7.543e-10,   6.101e-10]])

# Using a pre-computed power spectrogram

D = np.abs(lr.stft(y))**2
S = lr.feature.melspectrogram(S=D)

# Passing through arguments to the Mel filters
S = lr.feature.melspectrogram(y=y, sr=sr, n_mels=128,
                                    fmax=8000)

import matplotlib.pyplot as plt
plt.figure(figsize=(10, 4))
lr.display.specshow(lr.power_to_db(S,
                                             ref=np.max),
                         y_axis='mel', fmax=8000,
                         x_axis='time')
plt.colorbar(format='%+2.0f dB')
plt.title('Mel spectrogram')
plt.tight_layout()
print(np.shape(D))
print(np.shape(S))
plt.figure(figsize=(10, 4))
lr.display.specshow(lr.power_to_db(D, ref=np.max), y_axis='log')
plt.colorbar(format='%+2.0f dB')
plt.title('Mel spectrogram')
plt.tight_layout()
for file in range(1,4):
    audio, sfreq = lr.load(audio_files[file])

    time = np.arange(0, len(audio)) / sfreq
    
    fig, ax = plt.subplots()
    ax.plot(time, audio)
    ax.set(xlabel='Time (s)', ylabel='Sound Amplitude')
metadata = pd.read_csv(r"D:\Machine_Learning\UrbanSound8K\UrbanSound8K\metadata\UrbanSound8K.csv")

metadata.head(5)
metadata.set_index('slice_file_name', inplace = True)<jupyter_output><empty_output><jupyter_text>Need to make a classes dataframe for confusion matrix later!<jupyter_code># Selects those two columns
classes = metadata[['classID', 'class']]

# Drops duplicates, then sorts by classID
classes = classes.drop_duplicates(subset=['classID','class']).sort_values('classID')

# Make ClassID the index
classes.set_index('classID', inplace = True)

# Print
classes
metadata.head(5)<jupyter_output><empty_output><jupyter_text>Setting configurations<jupyter_code>configs = {}

configs['sampling_rate'] = 22050
configs['duration'] = 4
configs['hop_length'] = 882 # 20ms
configs['fmin'] = 20
configs['fmax'] = configs['sampling_rate'] // 2
configs['n_mels'] = 128
configs['n_fft'] = configs['n_mels'] * 20
configs['samples'] = configs['sampling_rate'] * configs['duration']
configs['dims'] = (configs['n_mels'], 1 + int(np.floor(configs['samples']/configs['hop_length'])), 1)<jupyter_output><empty_output><jupyter_text>This prepares the spectrograms before being fed into the model (will be saved to pickle file later)<jupyter_code>X_train = []
X_test = []
Y_train = []
Y_test = []
#_min = float('inf')
#_max = -float('inf')

for file in metadata.index:
    filename = 'D:/Machine_Learning/UrbanSound8K/UrbanSound8K/audio/fold'+str(metadata.at[file, 'fold'])+ '/'+file
    audio, sfreq = lr.load(filename)

    # If file is too long, cut it to first 4 seconds
    if len(audio) > configs['samples']:
        audio = audio[0:0+configs['samples']]

    # Else if it is too short, pad with zeroes and then center it
    elif len(audio) < configs['samples']:
        padding = configs['samples'] - len(audio)
        offset = padding // 2
        audio = np.pad(audio, (offset, configs['samples'] - len(audio) - offset), 'constant')

    # Computes melspectogram
    spectrogram = lr.feature.melspectrogram(audio, 
                                                sr=configs['sampling_rate'],
                                                n_mels=configs['n_mels'],
                                                hop_length=configs['hop_length'],
                                                n_fft=configs['n_fft'],
                                                fmin=configs['fmin'],
                                                fmax=configs['fmax'])
    spectrogram = lr.power_to_db(spectrogram)
    spectrogram = spectrogram.astype(np.float32)

    # Running min and max for mormalization
    #_min = min(np.amin(spectrogram), _min)
    #_max = max(np.amin(spectrogram), _max)
        
    if metadata.at[file, 'fold'] != 10:
        X_train.append(spectrogram)
        Y_train.append(metadata.at[file, 'classID'])
    
    else:
        X_test.append(spectrogram)
        Y_test.append(metadata.at[file, 'classID'])

# Converts the lists into numpy arrays
X_train, Y_train = np.array(X_train), np.array(Y_train)
X_test, Y_test = np.array(X_test), np.array(Y_test)

# I DID NOT NORMALIZE
#X = (X - _min) / (_max - _min)

# Reshapes numpy arrays to have 1 channel
# Also One hot encodes the labels
X_train = X_train.reshape(X_train.shape[0], X_train.shape[1], X_train.shape[2], 1)
Y_train = to_categorical(Y_train, num_classes = 10)

X_test = X_test.reshape(X_test.shape[0], X_test.shape[1], X_test.shape[2], 1)
Y_test = to_categorical(Y_test, num_classes = 10)
print(np.shape(X_train), "\n", 
      np.shape(X_test))
model = Sequential()
model.add(Conv2D(32, (3, 3), padding='same',
                 input_shape=configs['dims']))
model.add(Activation('relu'))
model.add(Conv2D(64, (3, 3)))
model.add(Activation('relu'))
model.add(MaxPooling2D(pool_size=(2, 2)))
model.add(Dropout(0.25))
model.add(Conv2D(64, (3, 3), padding='same'))
model.add(Activation('relu'))
model.add(Conv2D(64, (3, 3)))
model.add(Activation('relu'))
model.add(MaxPooling2D(pool_size=(2, 2)))
model.add(Dropout(0.5))
model.add(Conv2D(128, (3, 3), padding='same'))
model.add(Activation('relu'))
model.add(Conv2D(128, (3, 3)))
model.add(Activation('relu'))
model.add(MaxPooling2D(pool_size=(2, 2)))
model.add(Dropout(0.5))
model.add(Flatten())
model.add(Dense(512))
model.add(Activation('relu'))
model.add(Dropout(0.5))
model.add(Dense(10, activation='softmax'))
model.compile(optimizers.rmsprop(lr=0.0005, decay=1e-6),loss="categorical_crossentropy",metrics=["accuracy"])
model.summary()
model.compile(optimizer='adam', loss='categorical_crossentropy', metrics=['accuracy'])
model.fit(X_train, Y_train,
          batch_size=32,
          epochs=20,
          verbose=1,
          validation_data=(X_test, Y_test))
          #callbacks=[history])
model.save(r"D:\Machine_Learning\UrbanSound8K\UrbanSound8K\babys_first_model\baby_model.h5")
Y_predict = model.evaluate(X_test, Y_test)
print(Y_predict)

test_predictions = model.predict(X_test)
print(test_predictions)
Y_test
np.argmax(Y_test,axis=1)
np.argmax(test_predictions,axis=1)
# Transforming numbers into labels

# Get list of labels in order
my_list = classes["class"].tolist()

# Using sklearn labelencoder
le = preprocessing.LabelEncoder()
le.fit(my_list)

#Transforming Test matrices
labelled_test = le.inverse_transform(np.argmax(Y_test,axis=1))
labelled_predictions = le.inverse_transform(np.argmax(test_predictions,axis=1))
confusion = confusion_matrix(labelled_test, labelled_predictions)
confusion
def plot_confusion_matrix(y_true, y_pred, classes,
                          normalize=False,
                          title=None,
                          cmap=plt.cm.Blues):
    """
    This function prints and plots the confusion matrix.
    Normalization can be applied by setting `normalize=True`.
    """
    if not title:
        if normalize:
            title = 'Normalized confusion matrix'
        else:
            title = 'Confusion matrix, without normalization'

    # Compute confusion matrix
    cm = confusion_matrix(y_true, y_pred)
    # Only use the labels that appear in the data
    #classes = classes[unique_labels(y_true, y_pred)]
    if normalize:
        cm = cm.astype('float') / cm.sum(axis=1)[:, np.newaxis]
        print("Normalized confusion matrix")
    else:
        print('Confusion matrix, without normalization')

    print(cm)

    fig, ax = plt.subplots()
    im = ax.imshow(cm, interpolation='nearest', cmap=cmap)
    ax.figure.colorbar(im, ax=ax)
    # We want to show all ticks...
    ax.set(xticks=np.arange(cm.shape[1]),
           yticks=np.arange(cm.shape[0]),
           # ... and label them with the respective list entries
           xticklabels=classes, yticklabels=classes,
           title=title,
           ylabel='True label',
           xlabel='Predicted label')

    # Rotate the tick labels and set their alignment.
    plt.setp(ax.get_xticklabels(), rotation=45, ha="right",
             rotation_mode="anchor")

    # Loop over data dimensions and create text annotations.
    fmt = '.2f' if normalize else 'd'
    thresh = cm.max() / 2.
    for i in range(cm.shape[0]):
        for j in range(cm.shape[1]):
            ax.text(j, i, format(cm[i, j], fmt),
                    ha="center", va="center",
                    color="white" if cm[i, j] > thresh else "black")
    fig.tight_layout()
    return ax
plot_confusion_matrix(labelled_test, labelled_predictions, classes=my_list,
                      title='Confusion matrix, without normalization')
my_list<jupyter_output><empty_output><jupyter_text>This saves the spectograms into a pickle file for one 9-1 fold split!<jupyter_code>pickle.dump([X_train, Y_train, X_test, Y_test], open('D:/Machine_Learning/UrbanSound8K/UrbanSound8K/pickles/10_fold_10/10_fold_10.p', 'wb'))<jupyter_output><empty_output><jupyter_text>This is how we load in the pickle file for CV<jupyter_code>infile = open('D:/Machine_Learning/UrbanSound8K/UrbanSound8K/pickles/10_fold_10/10_fold_10.p','rb')
list1, list2, list3, list4 = pickle.load(infile)
infile.close()
my_list = classes["class"].tolist()
my_list<jupyter_output><empty_output><jupyter_text>Transfer Learning For Resnet50, the spectrograms need to be 3-channel. The solution prevalent online was to stack 3 greyscale images in numpy. A better solution would be to modify Resnet to accept single-channel, but that would require more time to figure out.<jupyter_code>#np.stack([array]*3, axis=-1)

# Converts single-channel tensor to 3-channel tensor by duplicating image 3 times
X_train_3d = np.squeeze(np.stack((X_train,) * 3, -1))
X_test_3d = np.squeeze(np.stack((X_test,) * 3, -1))
np.shape(X_test_3d)
from keras.applications.resnet50 import ResNet50

resnet_model = ResNet50(weights='imagenet', include_top=False, input_shape=(128,101,3))<jupyter_output>Using TensorFlow backend.<jupyter_text>Here is how to replace the top of Resnet50 w/ GAP2D + Dropout<jupyter_code>x = resnet_model.output
x = GlobalAveragePooling2D()(x)
x = Dropout(0.7)(x)
predictions = Dense(10, activation= 'softmax')(x)
final_model = Model(inputs = resnet_model.input, outputs = predictions)
final_model.compile(optimizers.rmsprop(lr=0.0005, decay=1e-6),loss="categorical_crossentropy",metrics=["accuracy"])
final_model.summary()
final_model.fit(X_train_3d, Y_train,
          batch_size=32,
          epochs=100,
          verbose=1,
          validation_data=(X_test_3d, Y_test))
          #callbacks=[history])
test_predictions_3d = final_model.predict(X_test_3d)
print(test_predictions_2)
#Transforming Test matrices
labelled_test = le.inverse_transform(np.argmax(Y_test,axis=1))
labelled_predictions_3d = le.inverse_transform(np.argmax(test_predictions_3d,axis=1))
plot_confusion_matrix(labelled_test, labelled_predictions_3d, classes=my_list,
                      title='Confusion matrix, without normalization')

plt.savefig('confused.png')
final_model.save(r'D:\Machine_Learning\UrbanSound8K\UrbanSound8K\resnet_model\final_model.h5')<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.13157894736842105, 'fraction_numerical': 0.027917620137299773, 'mean_word_length': 3.9475471698113207, 'pattern_counts': {'":': 0, '<': 38, '<?xml version=': 0, '>': 39, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 15, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '9', 'dir': 'jupyter-structured-clean-dedup', 'id': '151412', 'license': 'MIT', 'n_tokens_mistral': 4809, 'n_tokens_neox': 4509, 'n_words': 953, 'path': 'Urban_Sound_Tinkering.ipynb', 'repo_name': 'Path-A/Urban8kNotebooks', 'size': '13119'}