<jupyter_start><jupyter_text>Prepare Loaders<jupyter_code>#path_dataset = '../input/'
path_dataset = '/home/edoardobucheli/Datasets/FSDKaggle2018'

path_train = os.path.join(path_dataset,'audio_train_16k')
path_test = os.path.join(path_dataset,'audio_test_16k')<jupyter_output><empty_output><jupyter_text>Load Label Data<jupyter_code>train_data = pd.read_csv(os.path.join(path_dataset,'train_post_competition.csv'))
test_data = pd.read_csv(os.path.join(path_dataset,'test_post_competition_scoring_clips.csv'))
from utilities import get_all_classes_dict, get_classes_to_meta_dict, get_labels
num_to_label, label_to_num, n_classes = get_all_classes_dict(train_data)
label_to_meta, label_num_to_meta = get_classes_to_meta_dict(label_to_num)
data_cur = train_data[train_data['manually_verified']==1]
data_noi = train_data[train_data['manually_verified']==0]
meta_labels_all, labels_all = get_labels(train_data,label_to_meta, label_to_num)
meta_labels_cur, labels_cur = get_labels(data_cur,label_to_meta, label_to_num)
meta_labels_noi, labels_noi = get_labels(data_noi,label_to_meta, label_to_num)
meta_labels_test, labels_test = get_labels(test_data,label_to_meta, label_to_num)
n_meta_classes = len(np.unique(meta_labels_all))<jupyter_output><empty_output><jupyter_text>Load Data<jupyter_code>pickle_test = './preprocessed_test/PS-257-HL256-WF16k-64k'
with open(pickle_test, 'rb') as fp:
    x_test = pickle.load(fp)
x_test_private = [f for i,f in enumerate(x_test) if test_data['usage'][i] == 'Private']
labels_test_private = [f for i,f in enumerate(labels_test) if test_data['usage'][i] == 'Private']
meta_labels_test_private = [f for i,f in enumerate(meta_labels_test) if test_data['usage'][i] == 'Private']
new_x_test = []

for this_x in x_test_private:
    if this_x.shape[1] == 251:
        this_x = this_x[:,:250]
    new_x_test.append(this_x)
    
x_test_private = new_x_test
pickle_test_32 = './preprocessed_test/PS-257-HL256-WF32k-64k'

with open(pickle_test_32, 'rb') as fp:
    x_test_32k = pickle.load(fp)
x_test_32k_private = [f for i,f in enumerate(x_test_32k) if test_data['usage'][i]=='Private']
temp_x_test = []

for this_x in x_test_32k_private:
    if this_x.shape[1] == 251:
        this_x = this_x[:,:250]
    temp_x_test.append(this_x)
    
x_test_32k_private = temp_x_test
sr = 16000
file_length = 64000
hop_length = 256
freq_res = 257
frames = int(np.ceil(file_length/hop_length))<jupyter_output><empty_output><jupyter_text>Load Network<jupyter_code>from networks.CNNetworks2D import malley_cnn_120
from tensorflow.keras.optimizers import Adam
input_shape = ([freq_res,frames])
lr = 0.001
mc_model = malley_cnn_120(input_shape,n_meta_classes)
mc_model.compile(optimizer=Adam(lr),loss = 'sparse_categorical_crossentropy',metrics = ['accuracy'])
#mc_model.save_weights('./weights_mc_malley.h5')
mc_model.load_weights('./outputs_mc/best_weights/malley-MC-16k-PS-257-256[0].h5')
mc_model.summary()<jupyter_output>_________________________________________________________________
Layer (type)                 Output Shape              Param #   
=================================================================
input_1 (InputLayer)         (None, 257, 250)          0         
_________________________________________________________________
expand_dims (Lambda)         (None, 257, 250, 1)       0         
_________________________________________________________________
conv2d (Conv2D)              (None, 257, 250, 64)      1408      
_________________________________________________________________
activation (Activation)      (None, 257, 250, 64)      0         
_________________________________________________________________
max_pooling2d (MaxPooling2D) (None, 85, 250, 64)       0         
_________________________________________________________________
conv2d_1 (Conv2D)            (None, 85, 250, 128)      57472     
_________________________________________________________________
activation[...]<jupyter_text>Make Inference<jupyter_code>from utilities import create_quick_test_2d
from utilities import evaluate_complete_files
y_scores_MC,edo = evaluate_complete_files(x_test_private,meta_labels_test_private,mc_model,[257,250])
y_hat_MC = np.argmax(y_scores_MC,axis = 1)
np.mean(y_hat_MC==meta_labels_test_private)
x_test_2 = create_quick_test_2d(x_test_private,freq_res,frames)
mc_model.evaluate(x_test_2,meta_labels_test_private)
#x_test_2 = create_quick_test_2d(x_test,freq_res,frames)
#mc_model.evaluate(x_test_2,meta_labels_test)
#y_scores_MC = mc_model.predict(x_test_2)
#y_hat_MC = np.argmax(y_scores_MC, axis = 1)<jupyter_output><empty_output><jupyter_text>Plot Confusion Matrix<jupyter_code>from utilities import plot_cm
plot_cm(meta_labels_test_private,y_hat_MC,figsize = (7,7))
plt.savefig('./cm_MC-Private.eps')
del(mc_model)<jupyter_output><empty_output><jupyter_text>Separate Given Inference<jupyter_code>c0_indx = [i for i,f in enumerate(y_scores_MC) if f[0] >= 0.2]
c1_indx = [i for i,f in enumerate(y_scores_MC) if f[1] >= 0.2]
c2_indx = [i for i,f in enumerate(y_scores_MC) if f[2] >= 0.2]
c3_indx = [i for i,f in enumerate(y_scores_MC) if f[3] >= 0.2]
c4_indx = [i for i,f in enumerate(y_scores_MC) if f[4] >= 0.2]
c5_indx = [i for i,f in enumerate(y_scores_MC) if f[5] >= 0.2]<jupyter_output><empty_output><jupyter_text>Create General Class Vector<jupyter_code>all_scores =np.zeros((len(x_test_private),47))<jupyter_output><empty_output><jupyter_text>Cluster 0<jupyter_code>x0 = [x_test_private[f] for f in c0_indx]
y0 = [labels_test_private[f] for f in c0_indx]
with open('./Clustering_V1_mappings/c0_mapping', 'rb') as fp:
    c0_label_mapping = pickle.load(fp)
    
c0_label_mapping_inv = dict([[v,k] for [k,v] in c0_label_mapping.items()])
c0_label_mapping_inv[len(c0_label_mapping_inv)] = 41

c0_labels = list(c0_label_mapping.keys())
y0_2 = []

for f in y0:
    if f in c0_labels:
        y0_2.append(c0_label_mapping[f])
    else:
        y0_2.append(len(c0_labels))
sr = 16000
file_length = 64000
hop_length = 256
freq_res = 257
frames = int(np.ceil(file_length/hop_length))
input_shape = [freq_res,frames]
modelc0 = malley_cnn_120(input_shape,len(c0_label_mapping)+1)
modelc0.load_weights('./outputs/best_weights/malley-C0-16k-PS-257-HL256[0].h5')
modelc0.compile(optimizer=Adam(lr),loss = 'sparse_categorical_crossentropy',metrics = ['accuracy'])
y_scores_c0,edo= evaluate_complete_files(x0,y0_2,modelc0,[freq_res,frames])
y_hat_c0 = np.argmax(y_scores_c0,axis = 1)
np.mean(y_hat_c0==y0_2)
x0_2 = create_quick_test_2d(x0,freq_res,frames)
modelc0.evaluate(x0_2,y0_2)<jupyter_output>309/309 [==============================] - 1s 5ms/step<jupyter_text>Save Scores in General Scores<jupyter_code>y_scores_c0_1 = modelc0.predict(x0_2)
y_hat_c0_1 = np.argmax(y_scores_c0,axis = 1)
y_hat_c0_orig = [c0_label_mapping_inv[f] for f in y_hat_c0]
for i,this_scores in zip(c0_indx,y_scores_c0):
    for j,f in enumerate(this_scores):
        all_scores[i,c0_label_mapping_inv[j]] = f * (1-this_scores[-1])#*y_scores_MC[i][0]
labels = [num_to_label[f] for f in c0_labels]
labels.append('Unknown')
plot_cm(y0_2,y_hat_c0,figsize = (10,10), labels = labels)
plt.savefig('./cm_MC0-Private.eps')
del(modelc0)<jupyter_output><empty_output><jupyter_text>Cluster 1<jupyter_code>x1 = [x_test_32k_private[f] for f in c1_indx]
y1 = [labels_test_private[f] for f in c1_indx]
with open('./Clustering_V1_mappings/c1_mapping', 'rb') as fp:
    c1_label_mapping = pickle.load(fp)

c1_label_mapping_inv = dict([[v,k] for [k,v] in c1_label_mapping.items()])
c1_label_mapping_inv[len(c1_label_mapping_inv)] = 42
    
c1_labels = list(c1_label_mapping.keys())
y1_2 = []

for f in y1:
    if f in c1_labels:
        y1_2.append(c1_label_mapping[f])
    else:
        y1_2.append(len(c1_labels))
sr = 32000
file_length = 64000
hop_length = 256
freq_res = 257
frames = int(np.ceil(file_length/hop_length))

input_shape = [freq_res,frames]
modelc1 = malley_cnn_120(input_shape,len(c1_labels)+1)
modelc1.load_weights('./outputs/best_weights/malley-C1-32k-PS-257-HL256[0].h5')
modelc1.compile(optimizer=Adam(lr),loss = 'sparse_categorical_crossentropy',metrics = ['accuracy'])
y_scores_c1,edo= evaluate_complete_files(x1,y1_2,modelc1,[freq_res,frames])
y_hat_c1 = np.argmax(y_scores_c1,axis = 1)
np.mean(y_hat_c1==y1_2)
x1_2 = create_quick_test_2d(x1,freq_res,frames)
modelc1.evaluate(x1_2,y1_2)<jupyter_output>158/158 [==============================] - 1s 7ms/step<jupyter_text>Save Scores in General Scores<jupyter_code>#y_scores_c1 = modelc1.predict(x1_2)
#y_hat_c1 = np.argmax(y_scores_c1,axis = 1)
y_hat_c1_orig = [c1_label_mapping_inv[f] for f in y_hat_c1]
for i,this_scores in zip(c1_indx,y_scores_c1):
    for j,f in enumerate(this_scores):
        all_scores[i,c1_label_mapping_inv[j]] = f*(1-this_scores[-1])#*y_scores_MC[i][1]
labels = [num_to_label[f] for f in c1_labels]
labels.append('Unknown')
plot_cm(y1_2,y_hat_c1,figsize = (7,7), labels = labels)
plt.savefig('./cm_MC1-private.eps')

del(modelc1)<jupyter_output><empty_output><jupyter_text>Cluster 2<jupyter_code>x2 = [x_test_32k_private[f] for f in c2_indx]
y2 = [labels_test_private[f] for f in c2_indx]
with open('./Clustering_V1_mappings/c2_mapping', 'rb') as fp:
    c2_label_mapping = pickle.load(fp)
    
c2_label_mapping_inv = dict([[v,k] for [k,v] in c2_label_mapping.items()])
c2_label_mapping_inv[len(c2_label_mapping_inv)] = 43
    
c2_labels = list(c2_label_mapping.keys())
y2_2 = []

for f in y2:
    if f in c2_labels:
        y2_2.append(c2_label_mapping[f])
    else:
        y2_2.append(len(c2_labels))
sr = 32000
file_length = 64000
hop_length = 256
freq_res = 257
frames = int(np.ceil(file_length/hop_length))

input_shape = [freq_res,frames]
modelc2 = malley_cnn_120(input_shape,len(c2_labels)+1)
modelc2.load_weights('./outputs/best_weights/malley-C2-32k-PS-257-HL256[0].h5')
modelc2.compile(optimizer=Adam(lr),loss = 'sparse_categorical_crossentropy',metrics = ['accuracy'])
y_scores_c2,edo= evaluate_complete_files(x2,y2_2,modelc2,[freq_res,frames])
y_hat_c2 = np.argmax(y_scores_c2,axis = 1)
np.mean(y_hat_c2==y2_2)
x2_2 = create_quick_test_2d(x2,freq_res,frames)
modelc2.evaluate(x2_2,y2_2)<jupyter_output>586/586 [==============================] - 2s 4ms/step<jupyter_text>Save Scores in General Scores<jupyter_code>#y_scores_c2 = modelc2.predict(x2_2)
#y_hat_c2 = np.argmax(y_scores_c2,axis = 1)
y_hat_c2_orig = [c2_label_mapping_inv[f] for f in y_hat_c2]
for i,this_scores in zip(c2_indx,y_scores_c2):
    for j,f in enumerate(this_scores):
        all_scores[i,c2_label_mapping_inv[j]] = f*(1-this_scores[-1])#*y_scores_MC[i][2]
labels = [num_to_label[f] for f in c2_labels]
labels.append('Unknown')
plot_cm(y2_2,y_hat_c2,figsize = (15,15), labels = labels, xrotation = 90)
plt.savefig('./cm_MC2-private.eps')

del(modelc2)<jupyter_output><empty_output><jupyter_text>Cluster 3<jupyter_code>x3 = [x_test_32k_private[f] for f in c3_indx]
y3 = [labels_test_private[f] for f in c3_indx]
with open('./Clustering_V1_mappings/c3_mapping', 'rb') as fp:
    c3_label_mapping = pickle.load(fp)
    
c3_label_mapping_inv = dict([[v,k] for [k,v] in c3_label_mapping.items()])
c3_label_mapping_inv[len(c3_label_mapping_inv)] = 44
    
c3_labels = list(c3_label_mapping.keys())
y3_2 = []

for f in y3:
    if f in c3_labels:
        y3_2.append(c3_label_mapping[f])
    else:
        y3_2.append(len(c3_labels))
sr = 32000
file_length = 64000
hop_length = 256
freq_res = 257
frames = int(np.ceil(file_length/hop_length))

input_shape = [freq_res,frames]
modelc3 = malley_cnn_120(input_shape,len(c3_labels)+1)
modelc3.load_weights('./outputs/best_weights/malley-C3-32k-PS-257-HL256[0].h5')
modelc3.compile(optimizer=Adam(lr),loss = 'sparse_categorical_crossentropy',metrics = ['accuracy'])
y_scores_c3,edo= evaluate_complete_files(x3,y3_2,modelc3,[freq_res,frames])
y_hat_c3 = np.argmax(y_scores_c3,axis = 1)
np.mean(y_hat_c3==y3_2)
x3_2 = create_quick_test_2d(x3,freq_res,frames)
modelc3.evaluate(x3_2,y3_2)<jupyter_output>170/170 [==============================] - 1s 4ms/step<jupyter_text>Save Scores in General Scores<jupyter_code>#y_scores_c3 = modelc3.predict(x3_2)
#y_hat_c3 = np.argmax(y_scores_c3,axis = 1)
y_hat_c3_orig = [c3_label_mapping_inv[f] for f in y_hat_c3]
for i,this_scores in zip(c3_indx,y_scores_c3):
    for j,f in enumerate(this_scores):
        all_scores[i,c3_label_mapping_inv[j]] = f*(1-this_scores[-1])#*y_scores_MC[i][3]
labels = [num_to_label[f] for f in c3_labels]
labels.append('Unknown')
plot_cm(y3_2,y_hat_c3,figsize = (7,7), labels = labels, xrotation = 45)
plt.savefig('./cm_MC3-private.eps')

del(modelc3)<jupyter_output><empty_output><jupyter_text>Cluster 4<jupyter_code>x4 = [x_test_private[f] for f in c4_indx]
y4 = [labels_test_private[f] for f in c4_indx]
c4_scores = [f[4] for i,f in enumerate(y_scores_MC) if i in c4_indx]
for i, score in zip(c4_indx,c4_scores):
    all_scores[i,label_to_num['Applause']] = score<jupyter_output><empty_output><jupyter_text>Cluster 5<jupyter_code>x5 = [x_test_private[f] for f in c5_indx]
y5 = [labels_test_private[f] for f in c5_indx]
with open('./Clustering_V1_mappings/c5_mapping', 'rb') as fp:
    c5_label_mapping = pickle.load(fp)

c5_label_mapping_inv = dict([[v,k] for [k,v] in c5_label_mapping.items()])
c5_label_mapping_inv[len(c5_label_mapping_inv)] = 45
    
c5_labels = list(c5_label_mapping.keys())
c5_labels = list(c5_label_mapping.keys())
y5_2 = []

for f in y5:
    if f in c5_labels:
        y5_2.append(c5_label_mapping[f])
    else:
        y5_2.append(len(c5_labels))
sr = 16000
file_length = 64000
hop_length = 256
freq_res = 257
frames = int(np.ceil(file_length/hop_length))

input_shape = [freq_res,frames]
modelc5 = malley_cnn_120(input_shape,len(c5_labels)+1)
modelc5.load_weights('./outputs/best_weights/malley-C5-16k-PS-257-HL256[0].h5')
modelc5.compile(optimizer=Adam(lr),loss = 'sparse_categorical_crossentropy',metrics = ['accuracy'])
y_scores_c5,edo= evaluate_complete_files(x5,y5_2,modelc5,[freq_res,frames])
y_hat_c5 = np.argmax(y_scores_c5,axis = 1)
np.mean(y_hat_c5==y5_2)
x5_2 = create_quick_test_2d(x5,freq_res,frames)
modelc5.evaluate(x5_2,y5_2)<jupyter_output>104/104 [==============================] - 0s 5ms/step<jupyter_text>Save Scores in General Scores<jupyter_code>#y_scores_c5 = modelc5.predict(x5_2)
#y_hat_c5 = np.argmax(y_scores_c5,axis = 1)
y_hat_c5_orig = [c5_label_mapping_inv[f] for f in y_hat_c5]
for i,this_scores in zip(c5_indx,y_scores_c5):
    for j,f in enumerate(this_scores):
        all_scores[i,c5_label_mapping_inv[j]] = f * (1-this_scores[-1])#y_scores_MC[i][5]
labels = [num_to_label[f] for f in c5_labels]
labels.append('Unknown')
plot_cm(y5_2,y_hat_c5,figsize = (7,7), labels = labels)
plt.savefig('./cm_MC5-private.eps')

del(modelc5)<jupyter_output><empty_output><jupyter_text>Final Inference<jupyter_code>from utilities import mapk
y_hat_final = np.argmax(all_scores[:,:41],axis = 1)
tha_best = []

for this_score in all_scores[:,:41]:
    tha_best.append(sorted(zip(this_score, np.arange(len(this_score))), reverse=True)[:3])
y_top_3 = [[a,b,c] for [(_,a),(_,b),(_,c)] in tha_best] 
mapk([[f] for f in labels_test_private],y_top_3,k=3)
np.mean(y_hat_final==labels_test_private)
num_to_label[41] = 'Unknown0'
num_to_label[42] = 'Unknown1'
num_to_label[43] = 'Unknown2'
num_to_label[44] = 'Unknown3'
num_to_label[45] = 'Unknown4'
num_to_label[46] = 'Unknown5'
with open('./preprocessed_test/WF-16k-64k','rb') as fp:
    waves = pickle.load(fp)
waves_private = [f for i,f in enumerate(waves) if test_data['usage'][i] == 'Private']
n=np.random.randint(0,301)
x = np.arange(47)

plt.figure(figsize = (16,5))
plt.stem(all_scores[n])
ticks = plt.xticks(ticks = x,labels= [num_to_label[f] for f in x], rotation = 90)
plt.title('Real: {} {}'.format(num_to_label[labels_test_private[n]],n))
librosa.display.specshow(x_test_32k_private[n])
Audio(waves_public[n],rate = 16000)
labels = list(num_to_label.values())
plot_cm(labels_test_private,y_hat_final,figsize = (20,20), labels = labels, xrotation=90)
plt.savefig('./cm_all_41classes-private.eps')<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.14439870259481039, 'fraction_numerical': 0.053330838323353294, 'mean_word_length': 7.0406218655967905, 'pattern_counts': {'":': 0, '<': 75, '<?xml version=': 0, '>': 81, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '20', 'dir': 'jupyter-structured-clean-dedup', 'id': '27594', 'license': 'MIT', 'n_tokens_mistral': 7145, 'n_tokens_neox': 6658, 'n_words': 954, 'path': 'FSDKaggle2018_Complete-Private.ipynb', 'repo_name': 'ebucheli/FreeSound_GPAT', 'size': '16032'}