<jupyter_start><jupyter_text>Deep CNNs for Violin Melody Extraction from Polyphonic Music Signals Contributors: Mr. <NAME>, Dr. <NAME>  Import Required Modules<jupyter_code># import standard modules
import numpy as np
# import custom modules
import chooseRepresentation
import preprocessing
import training
import postprocessing
import evaluation
import predicting<jupyter_output><empty_output><jupyter_text>Define Paths<jupyter_code>path_proj = #fill in with path
path_data = path_proj + '/data'

path_ChooseFeatures = path_data + '/choose_Features'
path_violin = path_ChooseFeatures + '/violin_file.wav'

path_ME1 = path_data + '/ME1'
path_train_ME1 = path_ME1 + '/TRAINING'
path_trainAugmented_ME1 = path_ME1 + '/TRAINING_AUGMENTED'
path_test_ME1 = path_ME1 + '/TEST'

path_ME2 = path_data + '/ME2'
path_train_ME2 = path_ME2 + '/TRAINING'
path_trainAugmented_ME2 = path_ME2 + '/TRAINING_AUGMENTED'
path_test_ME2 = path_ME2 + '/TEST'

path_MTME = path_data + '/MTME'<jupyter_output><empty_output><jupyter_text>Set Fixed Parameters<jupyter_code>fs=16000
hopSize_Sec=0.010
noteMin='G3'
noteMax='F#7'
H=2
K=192
B=48
T=11
numClasses = 49<jupyter_output><empty_output><jupyter_text>Choose Input Representation  Step 1: Create solo violin File<jupyter_code>path_violin = chooseRepresentation.concatenateAudio(path_ChooseFeatures)
data, fs = chooseRepresentation.convertWavToArray(path_violin)<jupyter_output><empty_output><jupyter_text>Step 2: Plot Spectrograms of solo violin<jupyter_code>SNorm_Mel, SNorm_STFT, SNorm_CQT, SNorm_HCQT = chooseRepresentation.plotSpectrograms(data,fs)<jupyter_output><empty_output><jupyter_text>Step 3: Plot HCQT and Input Feature of polyphonic audio file<jupyter_code>path_exampleData = path_test_ME1 + '/VS01.wav'
exampleData,exampleFs = chooseRepresentation.convertWavToArrayNew(path_exampleData)
SFinal_HCQT = chooseRepresentation.plotHCQT(exampleData) 
inputFeatureExample = chooseRepresentation.plotInputFeature(exampleData)<jupyter_output><empty_output><jupyter_text>Preprocessing for Polyphonic Violin Melody Extraction CNN  Step 1: Clean Ground Truths<jupyter_code>preprocessing.cleanTestGTME(path_test_ME1)
preprocessing.cleanTrainGTME(path_train_ME1)<jupyter_output><empty_output><jupyter_text>Step 2: Downmix to mono and Downsample to 16kHz<jupyter_code>preprocessing.downmixAudio(path_test_ME1)
preprocessing.downmixAudio(path_train_ME1)
preprocessing.downsampleAudio(path_test_ME1,fs)
preprocessing.downsampleAudio(path_train_ME1,fs)<jupyter_output><empty_output><jupyter_text>Step 3: Augment training data by pitch shifting up by up to 3 semitones<jupyter_code>preprocessing.augmentDataME(path_train_ME1,path_trainAugmented_ME1,3)<jupyter_output><empty_output><jupyter_text>Step 4: Extract Input Features & Labels for Test Set<jupyter_code>XTest, yTest = preprocessing.extractFeaturesME(path_test_ME1, T, fs, hopSize_Sec, noteMin, H, K, B)
preprocessing.saveInputOutputArrays(XTest, yTest, 'XTest_ME.npy', 'yTest_ME.npy', path_ME1)<jupyter_output><empty_output><jupyter_text>Step 5: Get list of filenames in training dataset<jupyter_code>pieces = preprocessing.listAudioFilenames(path_train_ME1<jupyter_output><empty_output><jupyter_text>Step 6: Extract Input Features & Labels for each piece in Training Set (do one by one, changing i from 0 to len(pieces)-1)<jupyter_code>selPiece = pieces[i]
XTrain, yTrain = preprocessing.extractFeaturesME(path_trainAugmented_ME1+'/'+selPiece, T, fs, hopSize_Sec, noteMin, H, K, B)
preprocessing.saveInputOutputArrays(XTrain, yTrain, 'XTrain_ME_'+selPiece+'.npy', 'yTrain_ME_'+selPiece+'.npy', path_ME1)<jupyter_output><empty_output><jupyter_text>Step 7: Subsample training data by factor 10.00 to get appropriately sized data for RAM (do one by one, changing i from 0 to len(pieces)-1)<jupyter_code>XFull, yFull = preprocessing.loadArrays(path_ME1, 'XTrain_ME_'+selPiece+'.npy', 'yTrain_ME_'+selPiece+'.npy')
XSampled, ySampled = preprocessing.pickSubsetfromData(XFull, yFull, reductionFactor=10.00)
preprocessing.saveInputOutputArrays(XSampled, ySampled, 'XTrain_ME_'+selPiece+'subset'+'.npy', 'yTrain_ME_'+selPiece+'subset'+'.npy', path_ME1)<jupyter_output><empty_output><jupyter_text>Step 8: Concatenate subsampled data to form final training set<jupyter_code>XProv, yProv = preprocessing.loadArrays(path_ME1, 'XTrain_ME_'+pieces[0]+'subset'+'.npy','yTrain_ME_'+pieces[0]+'subset'+'.npy')
for i in range(1,len(pieces),1):
    selPiece = pieces[i]
    XLoaded, yLoaded = preprocessing.loadArrays(path_ME1, 'XTrain_ME_'+selPiece+'subset'+'.npy','yTrain_ME_'+selPiece+'subset'+'.npy')
    XProv = np.concatenate((XProv, XLoaded), axis=0)
    yProv = np.concatenate((yProv, yLoaded), axis=0)

preprocessing.saveInputOutputArrays(XProv, yProv, 'XTrain_ME'+'.npy', 'yTrain_ME'+'.npy', path_ME1)<jupyter_output><empty_output><jupyter_text>Step 9: Check that all label data is within required range<jupyter_code>y_TrainMECNN = preprocessing.loadLabelArray(path_ME1, 'yTrain_ME.npy')
counter=0
for el in y_TrainMECNN:
    if el > 48:
        print("NOTE TOO HIGH AT "+str(el))
    counter = counter+1<jupyter_output><empty_output><jupyter_text>Step 10: plot distribution of f0 labels for full and final training set<jupyter_code>ySubset = preprocessing.loadLabelArray(path_ME1,'yTrain_ME'+'.npy')
yProv = preprocessing.loadLabelArray(path_ME1,'yTrain_ME_'+pieces[0]+'.npy')
for i in range(1,len(pieces),1):
    selPiece = pieces[i]
    yLoaded = preprocessing.loadLabelArray(path_ME1,'yTrain_ME_'+selPiece+'.npy')
    yProv = np.concatenate((yProv, yLoaded), axis=0)

preprocessing.plotLabelDistributions(yProv,ySubset,'poly')<jupyter_output><empty_output><jupyter_text>Preprocessing for Monophonic Violin Melody Extraction CNN  Step 1: Clean Ground Truths<jupyter_code>preprocessing.cleanTestGTME(path_test_ME2)
preprocessing.cleanTrainGTME(path_train_ME2)<jupyter_output><empty_output><jupyter_text>Step 2: Downmix to mono and Downsample to 16kHz<jupyter_code>preprocessing.downmixAudio(path_test_ME2)
preprocessing.downmixAudio(path_train_ME2)
preprocessing.downsampleAudio(path_test_ME2,fs)
preprocessing.downsampleAudio(path_train_ME2,fs)<jupyter_output><empty_output><jupyter_text>Step 3: Augment training data by pitch shifting up by up to 3 semitones<jupyter_code>preprocessing.augmentDataME(path_train_ME2,path_trainAugmented_ME2,3)<jupyter_output><empty_output><jupyter_text>Step 4: Get list of filenames in training dataset<jupyter_code>pieces = preprocessing.listAudioFilenames(path_train_ME2)<jupyter_output><empty_output><jupyter_text>Step 5: Extract Input Features & Labels for each piece in Training Set (do one by one, changing i from 0 to len(pieces)-1)<jupyter_code>selPiece = pieces[i]
XTrain, yTrain = preprocessing.extractFeaturesME(path_trainAugmented_ME2+'/'+selPiece, T, fs, hopSize_Sec, noteMin, H, K, B)
preprocessing.saveInputOutputArrays(XTrain, yTrain, 'XTrain_ME_'+selPiece+'.npy', 'yTrain_ME_'+selPiece+'.npy', path_ME2)<jupyter_output><empty_output><jupyter_text>Step 6: Subsample training data by factor 2.22 to get appropriately sized data for RAM (do one by one, changing i from 0 to len(pieces)-1)<jupyter_code>selPiece = pieces[i]
XFull, yFull = preprocessing.loadArrays(path_ME2, 'XTrain_ME_'+selPiece+'.npy', 'yTrain_ME_'+selPiece+'.npy')
XSampled, ySampled = preprocessing.pickSubsetfromData(XFull, yFull, reductionFactor=2.22)
preprocessing.saveInputOutputArrays(XSampled, ySampled, 'XTrain_ME_'+selPiece+'subset'+'.npy', 'yTrain_ME_'+selPiece+'subset'+'.npy', path_ME2)<jupyter_output><empty_output><jupyter_text>Step 7: Concatenate subsampled data to form final training set<jupyter_code>XProv, yProv = preprocessing.loadArrays(path_ME2, 'XTrain_ME_'+pieces[0]+'subset'+'.npy','yTrain_ME_'+pieces[0]+'subset'+'.npy')
for i in range(1,len(pieces),1):
    selPiece = pieces[i]
    XLoaded, yLoaded = preprocessing.loadArrays(path_ME2, 'XTrain_ME_'+selPiece+'subset'+'.npy','yTrain_ME_'+selPiece+'subset'+'.npy')
    XProv = np.concatenate((XProv, XLoaded), axis=0)
    yProv = np.concatenate((yProv, yLoaded), axis=0)

preprocessing.saveInputOutputArrays(XProv, yProv, 'XTrain_ME'+'.npy', 'yTrain_ME'+'.npy', path_ME2)<jupyter_output><empty_output><jupyter_text>Step 8: Check that all label data is within required range<jupyter_code>y_TrainMECNN = preprocessing.loadLabelArray(path_ME2, 'yTrain_ME.npy')
counter=0
for el in y_TrainMECNN:
    if el > 48:
        print("NOTE TOO HIGH AT "+str(el))
    counter = counter+1<jupyter_output><empty_output><jupyter_text>Step 9: Plot distribution of f0 labels for full and final training set<jupyter_code>ySubset = preprocessing.loadLabelArray(path_ME2,'yTrain_ME'+'.npy')
yProv = preprocessing.loadLabelArray(path_ME2,'yTrain_ME_'+pieces[0]+'.npy')
for i in range(1,len(pieces),1):
    selPiece = pieces[i]
    yLoaded = preprocessing.loadLabelArray(path_ME2,'yTrain_ME_'+selPiece+'.npy')
    yProv = np.concatenate((yProv, yLoaded), axis=0)

preprocessing.plotLabelDistributions(yProv,ySubset,'mono')<jupyter_output><empty_output><jupyter_text>Build, Train and Evaluate Polyphonic Melody Extraction Single CNN  Step 1: Load Training and Test Set Arrays<jupyter_code>XTrain_PolyMECNN, yTrain_PolyMECNN = preprocessing.loadArrays(path_ME1, 'XTrain_ME'+'.npy','yTrain_ME'+'.npy')
XTest_PolyMECNN, yTest_PolyMECNN = preprocessing.loadArrays(path_ME1, 'XTest_ME'+'.npy','yTest_ME'+'.npy')<jupyter_output><empty_output><jupyter_text>Step 2: One hot encode labels<jupyter_code>yTrainOHE_PolyMECNN, yTestOHE_PolyMECNN = training.encodeLabels(yTrain_PolyMECNN,yTest_PolyMECNN,numClasses)<jupyter_output><empty_output><jupyter_text>Step 3: Execute random initialisation of CNN<jupyter_code>seed = 100
initialisation = training.initialiseWithSeed(seed)<jupyter_output><empty_output><jupyter_text>Step 4: Build Model Architecture<jupyter_code>PolyMECNN_Built = training.buildSingleMECNN(rows=K,columns=T,channels=H)<jupyter_output><empty_output><jupyter_text>Step 5: Set model Type and Number, and save model diagram (optional)<jupyter_code>modelType = 'PolyMECNN'
modelNum = 1
training.saveModelDiagram(PolyMECNN_Built, nameModel=modelType)<jupyter_output><empty_output><jupyter_text>Step 6: Set hyperparameters for training configuration<jupyter_code>E, batchSize, optimiser, lossFunction = training.setHyperparamsPolyMECNN()<jupyter_output><empty_output><jupyter_text>Step 7: Compile Model with loss function and optimiser<jupyter_code>PolyMECNN_Compiled = training.compileModel(PolyMECNN_Built, lossFunction, optimiser)<jupyter_output><empty_output><jupyter_text>Step 8: Train CNN Model<jupyter_code>PolyMECNN_Trained, PolyMECNN_History = training.trainModel(PolyMECNN_Compiled, 
                                                           XTrain_PolyMECNN, yTrainOHE_PolyMECNN, 
                                                           E, batchSize,  
                                                           nameModel=modelType, numModel=modelNum, pathSave=path_ME1,
                                                           vsplit=False, vsplitfactor=0.1,
                                                           XVal=XTest_PolyMECNN, yVal=yTestOHE_PolyMECNN)<jupyter_output><empty_output><jupyter_text>Step 9: Plot evolution of loss and accuracy<jupyter_code>training.plotModelHistory(PolyMECNN_History, 
                          modelType, modelNum,
                          E)<jupyter_output><empty_output><jupyter_text>Step 10: Evaluate Model on Test Set<jupyter_code>PolyMECNN_Loaded = training.loadpretrainedmodel(path_ME1+'/'+modelType+'_'+str(modelNum)+'.h5')
PolyMECNN_TestLoss, PolyMECNN_TestAcc = training.evaluateModel(PolyMECNN_Loaded, batchSize, 
                                                                       XTest_PolyMECNN, yTestOHE_PolyMECNN)<jupyter_output><empty_output><jupyter_text>Build, Train and Evaluate Monophonic Melody Extraction Single CNN  Step 1: Load Training and Test Set Arrays<jupyter_code>XTrain_MonoMECNN, yTrain_MonoMECNN = preprocessing.loadArrays(path_ME2, 'XTrain_ME'+'.npy','yTrain_ME'+'.npy')<jupyter_output><empty_output><jupyter_text>Step 2: One hot encode labels<jupyter_code>yTrainOHE_MonoMECNN = training.encodeLabelsSingle(yTrain_MonoMECNN,numClasses)<jupyter_output><empty_output><jupyter_text>Step 3: Execute random initialisation of CNN<jupyter_code>seed = 100
initialisation = training.initialiseWithSeed(seed)<jupyter_output><empty_output><jupyter_text>Step 4: Build Model Architecture<jupyter_code>MonoMECNN_Built = training.buildSingleMECNN(rows=K,columns=T,channels=H)<jupyter_output><empty_output><jupyter_text>Step 5: Set Model Type and Number, and save model diagram (optional)<jupyter_code>modelType = 'MonoMECNN'
modelNum = 1
training.saveModelDiagram(MonoMECNN_Built, nameModel=modelType)<jupyter_output><empty_output><jupyter_text>Step 6: Set hyperparameters for training configuration<jupyter_code>E, batchSize, optimiser, lossFunction = training.setHyperparamsMonoMECNN()<jupyter_output><empty_output><jupyter_text>Step 7: Compile Model with loss function and optimiser<jupyter_code>MonoMECNN_Compiled = training.compileModel(MonoMECNN_Built, lossFunction, optimiser)<jupyter_output><empty_output><jupyter_text>Step 8: Train CNN Model<jupyter_code>MonoMECNN_Trained, MonoMECNN_History = training.trainModel(MonoMECNN_Compiled, 
                                                           XTrain_MonoMECNN, yTrainOHE_MonoMECNN, 
                                                           E, batchSize,  
                                                           nameModel=modelType, numModel=modelNum, pathSave=path_ME2,
                                                           vsplit=True, vsplitfactor=0.1)<jupyter_output><empty_output><jupyter_text>Step 9: Plot evolution of loss and accuracy<jupyter_code>training.plotModelHistory(MonoMECNN_History, 
                          modelType, modelNum,
                          E)<jupyter_output><empty_output><jupyter_text>Step 10: Evaluate Model on Test Set<jupyter_code>MonoMECNN_Loaded = training.loadpretrainedmodel(path_ME2+'/'+modelType+'_'+str(modelNum)+'.h5')<jupyter_output><empty_output><jupyter_text>Plot Confusion Matrices and get predictions for Single CNN  Step 1: Set Model Type, Number and path<jupyter_code>modelType = 'PolyMECNN'
modelNum = 1
selPath = path_ME1<jupyter_output><empty_output><jupyter_text>Step 2: Load labels and one hot encode<jupyter_code>XTestPoly, yTestPoly = preprocessing.loadArrays(path_ME1, 'XTest_ME'+'.npy','yTest_ME'+'.npy')
yTestPolyOHE = training.encodeLabelsSingle(yTestPoly, numClasses)<jupyter_output><empty_output><jupyter_text>Step 3: Compute confusion matrix for CNN's predictions on test set<jupyter_code>loadedCNN = training.loadpretrainedmodel(selPath+'/'+modelType+'_'+str(modelNum)+'.h5')
lowestMIDIGT, labelListGT, classNamesGT = training.getClassNames(yTestPoly)
yPredPoly = predicting.predictOutputSingle(loadedCNN, XTestPoly)
lowestMIDIPred, labelListPred, classNamesPred = training.getClassNames(yPredPoly)
ConfusionMatrix, Predictions = training.plotConfusionMatrixSingle(loadedCNN, modelType, modelNum,
                                                                  XTestPoly, yTestPolyOHE, 
                                                                  classNamesPred)<jupyter_output><empty_output><jupyter_text>Post-processing for Single CNN  Step 1: Save array of all f0 Training Labels<jupyter_code>pieces = preprocessing.listAudioFilenames(path_train_ME1)
f0TrainingLabels = postprocessing.buildf0labelStream(path_train_ME1, pieces)
preprocessing.saveLabelArray(f0TrainingLabels, 'f0TrainingLabels'+'.npy', path_data)<jupyter_output><empty_output><jupyter_text>Step 2: Load Test Set Arrays<jupyter_code>XTestPoly, yTestPoly = preprocessing.loadArrays(path_ME1, 
                                                'XTest_ME'+'.npy', 'yTest_ME'+'.npy')<jupyter_output><empty_output><jupyter_text>Step 3: Load pre-trained CNN Model<jupyter_code>Singlemodel = training.loadpretrainedmodel(path_ME1+'/'+'PolyMECNN_1.h5')<jupyter_output><empty_output><jupyter_text>Step 4: Load f0 Training Labels<jupyter_code>f0TrainingLabels = preprocessing.loadLabelArray(path_data, 'f0TrainingLabels'+'.npy')<jupyter_output><empty_output><jupyter_text>Step 5: Get Smoothed Trajectory<jupyter_code>yPredPolySmoothed = postprocessing.getsmoothedf0Traj(f0TrainingLabels, yTestPoly, Singlemodel, False, XTestPoly, numClasses)<jupyter_output><empty_output><jupyter_text>Step 6: Compare accuracy obtained with smoothed over raw f0 trajectory on test set<jupyter_code>yPredPolyRaw = predicting.predictOutputSingle(Singlemodel, XTestPoly)
checksSmoothed, degreeSimilaritySmoothed = postprocessing.checkSimilarity(yTestPoly, yPredPolySmoothed)
checksRaw, degreeSimilarityRaw = postprocessing.checkSimilarity(yTestPoly, yPredPolyRaw)<jupyter_output><empty_output><jupyter_text>Preprocessing for Violin Melody Extraction Multi-Task CNN   Step 1: Subsample training data for Polyphonic VME CNN to form Target Task training set<jupyter_code>XPoly, yPoly = preprocessing.loadArrays(path_ME1, 'XTrain_ME'+'.npy', 'yTrain_ME'+'.npy')
XPolyHalved, yPolyHalved = preprocessing.pickNumFeatures(XPoly, yPoly, 150000)
preprocessing.saveInputOutputArrays(XPolyHalved, yPolyHalved, 'XTrainPoly_MTME'+'.npy', 'yTrainPoly_MTME'+'.npy', path_MTME)<jupyter_output><empty_output><jupyter_text>Step 2: Subsample training data for Polyphonic VME CNN to form Auxiliary Task training set<jupyter_code>XMono, yMono = preprocessing.loadArrays(path_ME2, 'XTrain_ME'+'.npy', 'yTrain_ME'+'.npy')
XMonoHalved, yMonoHalved = preprocessing.pickNumFeatures(XMono, yMono, 150000)
preprocessing.saveInputOutputArrays(XMonoHalved, yMonoHalved, 'XTrainMono_MTME'+'.npy', 'yTrainMono_MTME'+'.npy', path_MTME)<jupyter_output><empty_output><jupyter_text>Build, Train and Evaluate Violin Melody Extraction  Multi-Task CNN (Do each of 5 configurations one-by-one)  Step 1: Load Training and Test Set Arrays<jupyter_code>XTrainPoly_MTMECNN, yTrainPoly_MTMECNN = preprocessing.loadArrays(path_MTME, 'XTrainPoly_MTME'+'.npy',
                                                                  'yTrainPoly_MTME'+'.npy')
XTrainMono_MTMECNN, yTrainMono_MTMECNN = preprocessing.loadArrays(path_MTME, 'XTrainMono_MTME'+'.npy',
                                                                  'yTrainMono_MTME'+'.npy')
XTestPoly_MTMECNN, yTestPoly_MTMECNN = preprocessing.loadArrays(path_ME1, 'XTest_ME'+'.npy','yTest_ME'+'.npy')<jupyter_output><empty_output><jupyter_text>Step 2: One hot encode labels<jupyter_code>yTrainPolyOHE_MTMECNN = training.encodeLabelsSingle(yTrainPoly_MTMECNN, numClasses)
yTrainMonoOHE_MTMECNN = training.encodeLabelsSingle(yTrainMono_MTMECNN, numClasses)
yTestPolyOHE_MTMECNN = training.encodeLabelsSingle(yTestPoly_MTMECNN, numClasses)<jupyter_output><empty_output><jupyter_text>Step 3: Execute random initialisation of CNN<jupyter_code>seed = 100
initialisation = training.initialiseWithSeed(seed)<jupyter_output><empty_output><jupyter_text>Step 4: Build model architecture<jupyter_code>MTMECNN_Built = training.buildMTMECNN(rows=K,columns=T,channels=H)<jupyter_output><empty_output><jupyter_text>Step 5: Set model type and number (do one by one, changing i from 0 to 4), and save model diagram (optional)<jupyter_code>modelType = 'MTMECNN'
modelNum = [1, 2, 3, 4, 5]
selModelNum = modelNum[i]
training.saveModelDiagram(MTMECNN_Built, nameModel=modelType)<jupyter_output><empty_output><jupyter_text>Step 6: Set hyperparameters for training configuration<jupyter_code>E, batchSize, optimiser, lossFunction = training.setHyperparamsMTMECNN()
lossWeights = training.getLossWeights(selModelNum)<jupyter_output><empty_output><jupyter_text>Step 7: Compile Model with loss function and optimiser<jupyter_code>MTMECNN_Compiled = training.compileMultiTaskModel(MTMECNN_Built, lossFunction, lossWeights, optimiser)<jupyter_output><empty_output><jupyter_text>Step 8: Train CNN Model<jupyter_code>MTMECNN_Trained, MTMECNN_History = training.trainMultiTaskModel(MTMECNN_Compiled, 
                                                                XTrainPoly_MTMECNN, yTrainPolyOHE_MTMECNN, 
                                                                XTrainMono_MTMECNN, yTrainMonoOHE_MTMECNN,
                                                                E, batchSize,  
                                                                nameModel=modelType, numModel=selModelNum, pathSave=path_MTME,
                                                                vsplitfactor=0.1)<jupyter_output><empty_output><jupyter_text>Step 9: Plot evolution of loss and accuracy<jupyter_code>training.plotMultiTaskModelHistory(MTMECNN_History, 
                                   modelType, selModelNum,
                                   E)<jupyter_output><empty_output><jupyter_text>Step 10: Evaluate Model on Test Set<jupyter_code>MTMECNN_Loaded = training.loadpretrainedmodel(path_MTME+'/'+modelType+'_'+str(selModelNum)+'.h5')
MTMECNN_Metrics, MTMECNN_Score, MTMECNN_TestLoss, MTMECNN_TestAcc = training.evaluateMultiTaskModel(MTMECNN_Loaded, batchSize, 
                                                                                                    XTestPoly_MTMECNN, yTestPolyOHE_MTMECNN)<jupyter_output><empty_output><jupyter_text>Plot Confusion Matrices and get predictions for Multi-Task CNN  Step 1: Set model type, number and path<jupyter_code>modelType = 'MTMECNN'
modelNum = [1, 2, 3, 4, 5]
selPath = path_MTME<jupyter_output><empty_output><jupyter_text>Step 2: Load labels and one hot encode<jupyter_code>XTestMT, yTestMT = preprocessing.loadArrays(path_ME1, 'XTest_ME'+'.npy','yTest_ME'+'.npy')
yTestMTOHE = training.encodeLabelsSingle(yTestMT, numClasses)<jupyter_output><empty_output><jupyter_text>Step 3: Compute confusion matrices for each CNN's predictions on test set<jupyter_code>for num in modelNum:
    loadedCNN = training.loadpretrainedmodel(selPath+'/'+modelType+'_'+str(num)+'.h5')
    lowestMIDIGT, labelListGT, classNamesGT = training.getClassNames(yTestMT)
    yPredMT = predicting.predictOutputMT(loadedCNN, [XTestMT, XTestMT])
    lowestMIDIPred, labelListPred, classNamesPred = training.getClassNames(yPredMT)
    ConfusionMatrix = training.plotConfusionMatrixMT(loadedCNN, modelType, num,
                                                     [XTestMT, XTestMT],
                                                     [yTestMTOHE, yTestMTOHE], 
                                                     classNamesPred)<jupyter_output><empty_output><jupyter_text>Post-processing for Multi-Task CNN  Step 1: Load Test Set Arrays<jupyter_code>XTestPoly, yTestPoly = preprocessing.loadArrays(path_ME1, 
                                                'XTest_ME'+'.npy', 'yTest_ME'+'.npy')<jupyter_output><empty_output><jupyter_text>Step 2: Load pre-trained CNN Model (do one-by-one, changing i from 0 to 4)<jupyter_code>MTmodelfilenames = ['MTMECNN_1.h5', 'MTMECNN_2.h5', 'MTMECNN_3.h5', 'MTMECNN_4.h5', 'MTMECNN_5.h5']
MTmodel = training.loadpretrainedmodel(path_MTME+'/'+MTmodelfilenames[i])<jupyter_output><empty_output><jupyter_text>Step 3: Load f0 Training Labels<jupyter_code>f0TrainingLabels = preprocessing.loadLabelArray(path_data, 'f0TrainingLabels'+'.npy')<jupyter_output><empty_output><jupyter_text>Step 4: Get Smoothed Trajectory<jupyter_code>yPredPolySmoothed = postprocessing.getsmoothedf0Traj(f0TrainingLabels, yTestPoly, MTmodel, True, XTestPoly, numClasses)<jupyter_output><empty_output><jupyter_text>Step 5: Compare accuracy obtained with smoothed over raw f0 trajectory on test set<jupyter_code>yPredPolyRaw = predicting.predictOutputMT(MTmodel, XTestPoly)
checksSmoothed, degreeSimilaritySmoothed = postprocessing.checkSimilarity(yTestPoly, yPredPolySmoothed)
checksRaw, degreeSimilarityRaw = postprocessing.checkSimilarity(yTestPoly, yPredPolyRaw)<jupyter_output><empty_output><jupyter_text>Evaluate melody extraction systems  Step 1: Load Test Set Arrays<jupyter_code>XTestPoly, yTestPoly = preprocessing.loadArrays(path_ME1, 
                                                'XTest_ME'+'.npy', 'yTest_ME'+'.npy')<jupyter_output><empty_output><jupyter_text>Step 2: Define label file, models to evaluate and model names<jupyter_code>labelFile = 'f0TrainingLabels.npy'
modelsEval = ['PolyMECNN_1.h5','MTMECNN_1.h5','MTMECNN_2.h5','MTMECNN_3.h5','MTMECNN_4.h5','MTMECNN_5.h5']
modelNames = ['Melodia','Deep Salience Map CNN',
             'Violin Melody Extraction Single CNN without smoothing','Violin Melody Extraction Single CNN with smoothing',
             'Violin Melody Extraction MT-CNN 1 without smoothing','Violin Melody Extraction MT-CNN 1 with smoothing',
             'Violin Melody Extraction MT-CNN 2 without smoothing','Violin Melody Extraction MT-CNN 2 with smoothing',
             'Violin Melody Extraction MT-CNN 3 without smoothing','Violin Melody Extraction MT-CNN 3 with smoothing',
             'Violin Melody Extraction MT-CNN 4 without smoothing','Violin Melody Extraction MT-CNN 4 with smoothing',
             'Violin Melody Extraction MT-CNN 5 without smoothing','Violin Melody Extraction MT-CNN 5 with smoothing',]<jupyter_output><empty_output><jupyter_text>Step 3: Get ground truth arrays<jupyter_code>f0TrajGT = evaluation.getF0TrajfromF0Labels(yTestPoly)
f0CentsGT, voicingGT = evaluation.getf0CentsVoicingArrays(f0TrajGT)<jupyter_output><empty_output><jupyter_text>Step 4: Evaluate Melodia<jupyter_code>timesMelodia, f0TrajMelodia = evaluation.getMelodiaf0Traj(path_test_ME1, fs, noteMin, noteMax, hopSize_Sec, yTestPoly)
f0CentsMelodia, voicingMelodia = evaluation.getf0CentsVoicingArrays(f0TrajMelodia)
VR_Melodia, VFA_Melodia, RPA_Melodia, RCA_Melodia, OA_Melodia = evaluation.getMEmetrics(voicingGT, f0CentsGT, 
                                                                                        voicingMelodia, f0CentsMelodia, 
                                                                                        modelNames[0])<jupyter_output><empty_output><jupyter_text>Step 5: Evaluate DSM CNN<jupyter_code>timesDSMCNN, f0TrajDSMCNN = evaluation.getDSMCNNf0Traj(path_test_ME1, hopSize_Sec, fs, noteMin, noteMax, yTestPoly)
f0CentsDSMCNN, voicingDSMCNN = evaluation.getf0CentsVoicingArrays(f0TrajDSMCNN)
VR_DSMCNN, VFA_DSMCNN, RPA_DSMCNN, RCA_DSMCNN, OA_DSMCNN = evaluation.getMEmetrics(voicingGT, f0CentsGT, 
                                                                                   voicingDSMCNN, f0CentsDSMCNN, 
                                                                                   modelNames[1])<jupyter_output><empty_output><jupyter_text>Step 6: Evaluate PolyMECNN_1 Raw<jupyter_code>timesPolyMECNN1R, f0TrajPolyMECNN1R = evaluation.getVMECNNf0Traj(XTestPoly, 
                                                                 modelsEval[0], path_ME1, labelFile, path_data, 
                                                                 smooth=False, isMT=False)
f0CentsPolyMECNN1R, voicingPolyMECNN1R = evaluation.getf0CentsVoicingArrays(f0TrajPolyMECNN1R)
VR_PolyMECNN1R, VFA_PolyMECNN1R, RPA_PolyMECNN1R, RCA_PolyMECNN1R, OA_PolyMECNN1R = evaluation.getMEmetrics(voicingGT, f0CentsGT, 
                                                                                                            voicingPolyMECNN1R, f0CentsPolyMECNN1R, 
                                                                                                            modelNames[2])<jupyter_output><empty_output><jupyter_text>Step 7: Evaluate PolyMECNN_1 Smoothed<jupyter_code>timesPolyMECNN1S, f0TrajPolyMECNN1S = evaluation.getVMECNNf0Traj(XTestPoly, 
                                                                 modelsEval[0], path_ME1, labelFile, path_data, 
                                                                 smooth=True, isMT=False)
f0CentsPolyMECNN1S, voicingPolyMECNN1S = evaluation.getf0CentsVoicingArrays(f0TrajPolyMECNN1S)
VR_PolyMECNN1S, VFA_PolyMECNN1S, RPA_PolyMECNN1S, RCA_PolyMECNN1S, OA_PolyMECNN1S = evaluation.getMEmetrics(voicingGT, f0CentsGT, 
                                                                                                            voicingPolyMECNN1S, f0CentsPolyMECNN1S, 
                                                                                                            modelNames[3])<jupyter_output><empty_output><jupyter_text>Step 8: Evaluate MTMECNN_1 Raw<jupyter_code>timesMTMECNN1R, f0TrajMTMECNN1R = evaluation.getVMECNNf0Traj(XTestPoly, 
                                                             modelsEval[1], path_MTME, labelFile, path_data, 
                                                             smooth=False, isMT=True)
f0CentsMTMECNN1R, voicingMTMECNN1R = evaluation.getf0CentsVoicingArrays(f0TrajMTMECNN1R)
VR_MTMECNN1R, VFA_MTMECNN1R, RPA_MTMECNN1R, RCA_MTMECNN1R, OA_MTMECNN1R = evaluation.getMEmetrics(voicingGT, f0CentsGT, 
                                                                                                  voicingMTMECNN1R, f0CentsMTMECNN1R, 
                                                                                                  modelNames[4])<jupyter_output><empty_output><jupyter_text>Step 9: Evaluate MTMECNN_1 Smoothed<jupyter_code>timesMTMECNN1S, f0TrajMTMECNN1S = evaluation.getVMECNNf0Traj(XTestPoly, 
                                                             modelsEval[1], path_MTME, labelFile, path_data, 
                                                             smooth=True, isMT=True)
f0CentsMTMECNN1S, voicingMTMECNN1S = evaluation.getf0CentsVoicingArrays(f0TrajMTMECNN1S)
VR_MTMECNN1S, VFA_MTMECNN1S, RPA_MTMECNN1S, RCA_MTMECNN1S, OA_MTMECNN1S = evaluation.getMEmetrics(voicingGT, f0CentsGT, 
                                                                                                  voicingMTMECNN1S, f0CentsMTMECNN1S, 
                                                                                                  modelNames[5])<jupyter_output><empty_output><jupyter_text>Step 10: Evaluate MTMECNN_2 Raw<jupyter_code>timesMTMECNN2R, f0TrajMTMECNN2R = evaluation.getVMECNNf0Traj(XTestPoly, 
                                                             modelsEval[2], path_MTME, labelFile, path_data, 
                                                             smooth=False, isMT=True)
f0CentsMTMECNN2R, voicingMTMECNN2R = evaluation.getf0CentsVoicingArrays(f0TrajMTMECNN2R)
VR_MTMECNN2R, VFA_MTMECNN2R, RPA_MTMECNN2R, RCA_MTMECNN2R, OA_MTMECNN2R = evaluation.getMEmetrics(voicingGT, f0CentsGT, 
                                                                                                  voicingMTMECNN2R, f0CentsMTMECNN2R, 
                                                                                                  modelNames[6])<jupyter_output><empty_output><jupyter_text>Step 11: Evaluate MTMECNN_2 Smoothed<jupyter_code>timesMTMECNN2S, f0TrajMTMECNN2S = evaluation.getVMECNNf0Traj(XTestPoly, 
                                                             modelsEval[2], path_MTME, labelFile, path_data, 
                                                             smooth=True, isMT=True)    
f0CentsMTMECNN2S, voicingMTMECNN2S = evaluation.getf0CentsVoicingArrays(f0TrajMTMECNN2S)
VR_MTMECNN2S, VFA_MTMECNN2S, RPA_MTMECNN2S, RCA_MTMECNN2S, OA_MTMECNN2S = evaluation.getMEmetrics(voicingGT, f0CentsGT, 
                                                                                                  voicingMTMECNN2S, f0CentsMTMECNN2S, 
                                                                                                  modelNames[7])<jupyter_output><empty_output><jupyter_text>Step 12: Evaluate MTMECNN_3 Raw<jupyter_code>timesMTMECNN3R, f0TrajMTMECNN3R = evaluation.getVMECNNf0Traj(XTestPoly, 
                                                             modelsEval[3], path_MTME, labelFile, path_data, 
                                                             smooth=False, isMT=True)
f0CentsMTMECNN3R, voicingMTMECNN3R = evaluation.getf0CentsVoicingArrays(f0TrajMTMECNN3R)
VR_MTMECNN3R, VFA_MTMECNN3R, RPA_MTMECNN3R, RCA_MTMECNN3R, OA_MTMECNN3R = evaluation.getMEmetrics(voicingGT, f0CentsGT, 
                                                                                                  voicingMTMECNN3R, f0CentsMTMECNN3R, 
                                                                                                  modelNames[8])<jupyter_output><empty_output><jupyter_text>Step 13: Evaluate MTMECNN_3 Smoothed<jupyter_code>timesMTMECNN3S, f0TrajMTMECNN3S = evaluation.getVMECNNf0Traj(XTestPoly, 
                                                             modelsEval[3], path_MTME, labelFile, path_data, 
                                                             smooth=True, isMT=True)
f0CentsMTMECNN3S, voicingMTMECNN3S = evaluation.getf0CentsVoicingArrays(f0TrajMTMECNN3S)
VR_MTMECNN3S, VFA_MTMECNN3S, RPA_MTMECNN3S, RCA_MTMECNN3S, OA_MTMECNN3S = evaluation.getMEmetrics(voicingGT, f0CentsGT, 
                                                                                                  voicingMTMECNN3S, f0CentsMTMECNN3S, 
                                                                                                  modelNames[9])<jupyter_output><empty_output><jupyter_text>Step 14: Evaluate MTMECNN_4 Raw<jupyter_code>timesMTMECNN4R, f0TrajMTMECNN4R = evaluation.getVMECNNf0Traj(XTestPoly, 
                                                             modelsEval[4], path_MTME, labelFile, path_data, 
                                                             smooth=False, isMT=True)
f0CentsMTMECNN4R, voicingMTMECNN4R = evaluation.getf0CentsVoicingArrays(f0TrajMTMECNN4R)
VR_MTMECNN4R, VFA_MTMECNN4R, RPA_MTMECNN4R, RCA_MTMECNN4R, OA_MTMECNN4R = evaluation.getMEmetrics(voicingGT, f0CentsGT, 
                                                                                                  voicingMTMECNN4R, f0CentsMTMECNN4R, 
                                                                                                  modelNames[10])<jupyter_output><empty_output><jupyter_text>Step 15: Evaluate MTMECNN_4 Smoothed<jupyter_code>timesMTMECNN4S, f0TrajMTMECNN4S = evaluation.getVMECNNf0Traj(XTestPoly, 
                                                             modelsEval[4], path_MTME, labelFile, path_data, 
                                                             smooth=True, isMT=True)    
f0CentsMTMECNN4S, voicingMTMECNN4S = evaluation.getf0CentsVoicingArrays(f0TrajMTMECNN4S)
VR_MTMECNN4S, VFA_MTMECNN4S, RPA_MTMECNN4S, RCA_MTMECNN4S, OA_MTMECNN4S = evaluation.getMEmetrics(voicingGT, f0CentsGT, 
                                                                                                  voicingMTMECNN4S, f0CentsMTMECNN4S, 
                                                                                                  modelNames[11])<jupyter_output><empty_output><jupyter_text>Step 16: Evaluate MTMECNN_5 Raw<jupyter_code>timesMTMECNN5R, f0TrajMTMECNN5R = evaluation.getVMECNNf0Traj(XTestPoly, 
                                                             modelsEval[5], path_MTME, labelFile, path_data, 
                                                             smooth=False, isMT=True)
f0CentsMTMECNN5R, voicingMTMECNN5R = evaluation.getf0CentsVoicingArrays(f0TrajMTMECNN5R)
VR_MTMECNN5R, VFA_MTMECNN5R, RPA_MTMECNN5R, RCA_MTMECNN5R, OA_MTMECNN5R = evaluation.getMEmetrics(voicingGT, f0CentsGT, 
                                                                                                  voicingMTMECNN5R, f0CentsMTMECNN5R, 
                                                                                                  modelNames[12])<jupyter_output><empty_output><jupyter_text>Step 17: Evaluate MTMECNN_5 Smoothed<jupyter_code>timesMTMECNN5S, f0TrajMTMECNN5S = evaluation.getVMECNNf0Traj(XTestPoly, 
                                                             modelsEval[5], path_MTME, labelFile, path_data, 
                                                             smooth=True, isMT=True)
f0CentsMTMECNN5S, voicingMTMECNN5S = evaluation.getf0CentsVoicingArrays(f0TrajMTMECNN5S)
VR_MTMECNN5S, VFA_MTMECNN5S, RPA_MTMECNN5S, RCA_MTMECNN5S, OA_MTMECNN5S = evaluation.getMEmetrics(voicingGT, f0CentsGT, 
                                                                                                  voicingMTMECNN5S, f0CentsMTMECNN5S, 
                                                                                                  modelNames[13])<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.08495351962047878, 'fraction_numerical': 0.018016288699372034, 'mean_word_length': 3.2528279883381925, 'pattern_counts': {'":': 0, '<': 367, '<?xml version=': 0, '>': 369, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 219, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '91', 'dir': 'jupyter-structured-clean-dedup', 'id': '69230', 'license': 'Apache-2.0', 'n_tokens_mistral': 12268, 'n_tokens_neox': 11711, 'n_words': 1778, 'path': 'main.ipynb', 'repo_name': 'davidwfong/ViolinMelodyCNNs', 'size': '36479'}