<jupyter_start><jupyter_text>MFCC: No Scaling Overfitting:* Regularization* Batch Normalization* Dropout Layers:    * tf.keras.layers.Dropout(rate=0.2)     * 0.20-0.3 in RNN and 0.4-.5 in CNNTry with ReLU layersor Elu or SELU layers:* tf.keras.layers.Dense(64, kernel_initializer='lecun_normal', activation='selu')* tf.keras.layers.Dense(64, kernel_initializer='he_normal',                                activation= 'elu')<jupyter_code>def plot_results(fit):
    plt.figure(figsize=(20,15))
    fig, ax = plt.subplots(2)
    # accuracy subplot
    ax[0].plot(fit.history["accuracy"], label="train")
    ax[0].plot(fit.history["val_accuracy"], label="test")    
    ax[0].set_ylabel("Accuracy")
    ax[0].legend(loc="lower right")
    ax[0].set_title("Accuracy")
    
    # Error subplot
    ax[1].plot(fit.history["loss"], label="train error")
    ax[1].plot(fit.history["val_loss"], label="test error")    
    ax[1].set_ylabel("Error")
    ax[1].set_xlabel("Epoch")
    ax[1].legend(loc="upper right")
    ax[1].set_title("Error")
    plt.tight_layout()
    plt.show()
train_times = []
trainX.shape<jupyter_output><empty_output><jupyter_text>Baseline Metrics---Following Book<jupyter_code>%%time
model = tf.keras.Sequential([  
    tf.keras.layers.Flatten(input_shape = [trainX.shape[1], trainX.shape[2], trainX.shape[3]]),
    tf.keras.layers.Dense(10)
])
## get output layer as logits for each class 
##(we can use softmax to get probabilities, but doing so makes model unstable:
# https://www.tensorflow.org/tutorials/quickstart/beginner
model.summary()
callback = tf.keras.callbacks.EarlyStopping(monitor='loss', patience=50)
model.compile(optimizer='adam',
              loss=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True),
              metrics=['accuracy'])

start = time.time()
fit = model.fit(trainX, trainY,
                validation_data = (testX,testY),
                epochs = 250,
                callbacks = [callback])
stop = time.time()
train_times.append(stop-start)


plot_results(fit)

probability_model = tf.keras.Sequential([model, 
                                         tf.keras.layers.Softmax()])
predictions = probability_model.predict(testX)
pred_labels = [np.argmax(prediction) for i,prediction in enumerate(predictions)]

from sklearn.metrics import ConfusionMatrixDisplay
from sklearn.metrics import accuracy_score

ConfusionMatrixDisplay.from_predictions(
    y_true = [inv_labels[e] for i,e in enumerate(testY)],
    y_pred = [inv_labels[e] for i,e in enumerate(pred_labels)], 
    xticks_rotation= "vertical")
print(accuracy_score(testY,pred_labels))
%%time
model = tf.keras.Sequential([  
    tf.keras.layers.Flatten(input_shape = [trainX.shape[1], trainX.shape[2], trainX.shape[3]]),
    tf.keras.layers.Dense(10)
])
## get output layer as logits for each class 
##(we can use softmax to get probabilities, but doing so makes model unstable:
# https://www.tensorflow.org/tutorials/quickstart/beginner
model.summary()
callback = tf.keras.callbacks.EarlyStopping(monitor='loss', patience=50)
model.compile(optimizer='adam',
              loss=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True),
              metrics=['accuracy'])

start = time.time()
fit = model.fit(trainX, trainY,
                validation_data = (testX,testY),
                epochs = 250,
                callbacks = [callback])
stop = time.time()
train_times.append(stop-start)


plot_results(fit)

probability_model = tf.keras.Sequential([model, 
                                         tf.keras.layers.Softmax()])
predictions = probability_model.predict(testX)
pred_labels = [np.argmax(prediction) for i,prediction in enumerate(predictions)]

from sklearn.metrics import ConfusionMatrixDisplay
from sklearn.metrics import accuracy_score

ConfusionMatrixDisplay.from_predictions(
    y_true = [inv_labels[e] for i,e in enumerate(testY)],
    y_pred = [inv_labels[e] for i,e in enumerate(pred_labels)], 
    xticks_rotation= "vertical")
print(accuracy_score(testY,pred_labels))<jupyter_output>2022-04-28 09:45:41.952658: W tensorflow/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudnn.so.8'; dlerror: libcudnn.so.8: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /opt/software/CUDA/11.4.2/nvvm/lib64:/opt/software/CUDA/11.4.2/extras/CUPTI/lib64:/opt/software/CUDA/11.4.2/lib
2022-04-28 09:45:41.952766: W tensorflow/core/common_runtime/gpu/gpu_device.cc:1850] Cannot dlopen some GPU libraries. Please make sure the missing libraries mentioned above are installed properly if you would like to use GPU. Follow the guide at https://www.tensorflow.org/install/gpu for how to download and setup the required libraries for your platform.
Skipping registering GPU devices...
2022-04-28 09:45:41.954004: I tensorflow/core/platform/cpu_feature_guard.cc:151] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations:  AVX2 FMA
To e[...]<jupyter_text>SimpleRNN---<jupyter_code>trainX.shape
trainY.shape
type(trainX)
%%time
model = tf.keras.Sequential([
    tf.keras.layers.SimpleRNN(64, input_shape=trainX.shape[1:3],
                              return_sequences=True),
    tf.keras.layers.SimpleRNN(64),
    tf.keras.layers.Dense(10)])
model.summary()
callback = tf.keras.callbacks.EarlyStopping(monitor='loss', patience=10)
model.compile(optimizer='adam',
              loss=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True),
              metrics=['accuracy'])

start = time.time()
fit = model.fit(trainX, trainY,
                validation_data = (testX,testY),
                epochs = 250, 
                callbacks = [callback])
stop = time.time()
train_times.append(stop-start)


plot_results(fit)

probability_model = tf.keras.Sequential([model, 
                                         tf.keras.layers.Softmax()])
predictions = probability_model.predict(testX)
pred_labels = [np.argmax(prediction) for i,prediction in enumerate(predictions)]

from sklearn.metrics import ConfusionMatrixDisplay
from sklearn.metrics import accuracy_score

ConfusionMatrixDisplay.from_predictions(
    y_true = [inv_labels[e] for i,e in enumerate(testY)],
    y_pred = [inv_labels[e] for i,e in enumerate(pred_labels)], 
    xticks_rotation= "vertical")
print(accuracy_score(testY,pred_labels))
%%time
model = tf.keras.Sequential([
    tf.keras.layers.SimpleRNN(64, input_shape=trainX.shape[1:3],
                              return_sequences=True),
    tf.keras.layers.SimpleRNN(64),
    tf.keras.layers.Dense(10)])
model.summary()
callback = tf.keras.callbacks.EarlyStopping(monitor='loss', patience=10)
model.compile(optimizer='adam',
              loss=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True),
              metrics=['accuracy'])

start = time.time()
fit = model.fit(trainX, trainY,
                validation_data = (testX,testY),
                epochs = 100, batch_size =10,
                callbacks = [callback])
stop = time.time()
train_times.append(stop-start)


plot_results(fit)

probability_model = tf.keras.Sequential([model, 
                                         tf.keras.layers.Softmax()])
predictions = probability_model.predict(testX)
pred_labels = [np.argmax(prediction) for i,prediction in enumerate(predictions)]

from sklearn.metrics import ConfusionMatrixDisplay
from sklearn.metrics import accuracy_score

ConfusionMatrixDisplay.from_predictions(
    y_true = [inv_labels[e] for i,e in enumerate(testY)],
    y_pred = [inv_labels[e] for i,e in enumerate(pred_labels)], 
    xticks_rotation= "vertical")
print(accuracy_score(testY,pred_labels))<jupyter_output>Model: "sequential_4"
_________________________________________________________________
 Layer (type)                Output Shape              Param #   
=================================================================
 simple_rnn_2 (SimpleRNN)    (None, 1293, 64)          12352     
                                                                 
 simple_rnn_3 (SimpleRNN)    (None, 64)                8256      
                                                                 
 dense_2 (Dense)             (None, 10)                650       
                                                                 
=================================================================
Total params: 21,258
Trainable params: 21,258
Non-trainable params: 0
_________________________________________________________________
Epoch 1/100
71/71 [==============================] - 17s 229ms/step - loss: 2.2696 - accuracy: 0.1737 - val_loss: 2.1516 - val_accuracy: 0.2542
Epoch 2/100
71/71 [===================[...]<jupyter_text>LSTM<jupyter_code>%%time
model = tf.keras.Sequential([
    tf.keras.layers.LSTM(64, input_shape=trainX.shape[1:3],
                         return_sequences=True),
    tf.keras.layers.LSTM(64),
    tf.keras.layers.Dense(64, activation = "relu"),

    tf.keras.layers.Dense(10),
])
model.summary()
callback = tf.keras.callbacks.EarlyStopping(monitor='loss', patience=10)
model.compile(optimizer=tf.keras.optimizers.Adam(learning_rate = 0.001),
              loss=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True),
              metrics=['accuracy'])

start = time.time()
fit = model.fit(trainX, trainY,
                validation_data = (testX,testY),
                epochs = 100, batch_size = 32,
                callbacks = [callback])
stop = time.time()
train_times.append(stop-start)


plot_results(fit)

probability_model = tf.keras.Sequential([model, 
                                         tf.keras.layers.Softmax()])
predictions = probability_model.predict(testX)
pred_labels = [np.argmax(prediction) for i,prediction in enumerate(predictions)]

from sklearn.metrics import ConfusionMatrixDisplay
from sklearn.metrics import accuracy_score

ConfusionMatrixDisplay.from_predictions(
    y_true = [inv_labels[e] for i,e in enumerate(testY)],
    y_pred = [inv_labels[e] for i,e in enumerate(pred_labels)], 
    xticks_rotation= "vertical")
print(accuracy_score(testY,pred_labels))
%%time
model = tf.keras.Sequential([
    tf.keras.layers.LSTM(64, input_shape=trainX.shape[1:3],
                         return_sequences=True),
    tf.keras.layers.LSTM(64, dropout=0.05, recurrent_dropout=0.1),
    tf.keras.layers.Dense(64, activation = "relu", 
                          kernel_regularizer = tf.keras.regularizers.l2(0.001)),
    tf.keras.layers.Dropout(rate=0.01),
    tf.keras.layers.Dense(32, activation = "relu", 
                          kernel_regularizer = tf.keras.regularizers.l2(0.001)),
    tf.keras.layers.Dropout(rate=0.01),
    tf.keras.layers.Dense(10, activation = "linear")
])
model.summary()
callback = tf.keras.callbacks.EarlyStopping(monitor='loss', patience=10)
model.compile(optimizer=tf.keras.optimizers.Adam(learning_rate = 0.001),
              loss=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True),
              metrics=['accuracy'])

start = time.time()
fit = model.fit(trainX, trainY,
                validation_data = (testX,testY),
                epochs = 100, batch_size = 32,
                callbacks = [callback])
stop = time.time()
train_times.append(stop-start)


plot_results(fit)

probability_model = tf.keras.Sequential([model, 
                                         tf.keras.layers.Softmax()])
predictions = probability_model.predict(testX)
pred_labels = [np.argmax(prediction) for i,prediction in enumerate(predictions)]

from sklearn.metrics import ConfusionMatrixDisplay
from sklearn.metrics import accuracy_score

ConfusionMatrixDisplay.from_predictions(
    y_true = [inv_labels[e] for i,e in enumerate(testY)],
    y_pred = [inv_labels[e] for i,e in enumerate(pred_labels)], 
    xticks_rotation= "vertical")
print(accuracy_score(testY,pred_labels))
%%time
model = tf.keras.Sequential([
    tf.keras.layers.LSTM(64, dropout=0.05, recurrent_dropout=0.1,
                         input_shape=trainX.shape[1:3],
                         return_sequences=True),
    tf.keras.layers.LSTM(64, dropout=0.05, recurrent_dropout=0.1),
    tf.keras.layers.Dense(64, activation = "relu", 
                          kernel_regularizer = tf.keras.regularizers.l2(0.001)),
    tf.keras.layers.Dropout(rate=0.01),
    tf.keras.layers.Dense(32, activation = "relu", 
                          kernel_regularizer = tf.keras.regularizers.l2(0.001)),
    tf.keras.layers.Dropout(rate=0.01),
    tf.keras.layers.Dense(10, activation = "linear")
])
model.summary()
callback = tf.keras.callbacks.EarlyStopping(monitor='loss', patience=10)
model.compile(optimizer=tf.keras.optimizers.Adam(learning_rate = 0.001),
              loss=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True),
              metrics=['accuracy'])

start = time.time()
fit = model.fit(trainX, trainY,
                validation_data = (testX,testY),
                epochs = 100, batch_size = 32,
                callbacks = [callback])
stop = time.time()
train_times.append(stop-start)


plot_results(fit)

probability_model = tf.keras.Sequential([model, 
                                         tf.keras.layers.Softmax()])
predictions = probability_model.predict(testX)
pred_labels = [np.argmax(prediction) for i,prediction in enumerate(predictions)]

from sklearn.metrics import ConfusionMatrixDisplay
from sklearn.metrics import accuracy_score

ConfusionMatrixDisplay.from_predictions(
    y_true = [inv_labels[e] for i,e in enumerate(testY)],
    y_pred = [inv_labels[e] for i,e in enumerate(pred_labels)], 
    xticks_rotation= "vertical")
print(accuracy_score(testY,pred_labels))<jupyter_output>Model: "sequential_10"
_________________________________________________________________
 Layer (type)                Output Shape              Param #   
=================================================================
 lstm_4 (LSTM)               (None, 1293, 64)          49408     
                                                                 
 lstm_5 (LSTM)               (None, 64)                33024     
                                                                 
 dense_8 (Dense)             (None, 64)                4160      
                                                                 
 dropout_2 (Dropout)         (None, 64)                0         
                                                                 
 dense_9 (Dense)             (None, 32)                2080      
                                                                 
 dropout_3 (Dropout)         (None, 32)                0         
                                                     [...]
{'fraction_non_alphanumeric': 0.13197796967430475, 'fraction_numerical': 0.027333922621880738, 'mean_word_length': 2.891005291005291, 'pattern_counts': {'":': 0, '<': 14, '<?xml version=': 0, '>': 14, 'https://': 3, 'lorem ipsum': 0, 'www.': 3, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 33, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '64211', 'license': 'MIT', 'n_tokens_mistral': 4828, 'n_tokens_neox': 4451, 'n_words': 799, 'path': 'models/LSTM_MelSpectrogram.ipynb', 'repo_name': 'yashmanne/Music_Genre_Classification', 'size': '14707'}