<jupyter_start><jupyter_text>Training CNN models:<jupyter_code>import os
import numpy as np
import matplotlib.pyplot as plt
import librosa
import librosa.display
import IPython.display as ipd
from sklearn.cluster import MiniBatchKMeans
import joblib
import keras
from keras.models import Sequential
from keras.layers import Dense, Conv2D, Flatten
from keras.callbacks import ModelCheckpoint
from DataGenerator import DataGenerator_Sup
from utility_functions import create_CNN_model, plot_confusion_matrix<jupyter_output>Using TensorFlow backend.<jupyter_text>Get a list of all files:<jupyter_code>Ad_folder = '../Data/audio_ads' # positive samples location
Music_folder = '../Data/Music'  # negative samples (music) location
Podcast_folder = '../Data/podcasts'  # negative samples (podcasts) location

pos_files = []
for r,d,f in os.walk(Ad_folder):
    for filename in f:
        if '.mp3' in filename:
            pos_files.append(os.path.join(Ad_folder,filename))

music_files = []
for r,d,f in os.walk(Music_folder):
    for filename in f:
        if '.mp3' or '.au' in filename:
            music_files.append(os.path.join(r,filename))

podcast_files = []
for r,d,f in os.walk(Podcast_folder):
    for filename in f:
        if '.wav' in filename:
            podcast_files.append(os.path.join(r,filename))<jupyter_output><empty_output><jupyter_text>How many mp3 files do we have?<jupyter_code>neg_files = music_files + podcast_files
n_pos_files = len(pos_files)
n_neg_files = len(neg_files)

print('We have ' + str(n_pos_files) + ' positive examples')
print('We have ' + str(len(music_files)) + ' music examples')
print('We have ' + str(len(podcast_files)) + ' podcast examples')

music_duration = 30/60.0 # duration of files in minutes
podcast_duration = 12/60.0 # duration of files in minutes
ads_duration = 30/60.0 # average duration of ad files in minutes

pos_minutes = round(ads_duration*n_pos_files,2)
neg_minutes = round(music_duration*len(music_files) + podcast_duration*len(podcast_files),2)
pos_fraction = str(round(neg_minutes/pos_minutes,2)) # the fraction of positives to take for balancing
print('--------------------------------')
print('In total, ' + str(pos_minutes) + ' minutes of positive and ' + str(neg_minutes) + ' minutes of negative')<jupyter_output>We have 2293 positive examples
We have 1013 music examples
We have 204739 podcast examples
--------------------------------
In total, 1146.5 minutes of positive and 41454.3 minutes of negative<jupyter_text>Create data generators:  Decide which part of the data to use and how data will be distributed:<jupyter_code>minutes2use = 1000 # total minutes to use for training and testing
train_fraction = 0.9 # fraction of the data to train on

pos_minutes2use = int(minutes2use/2.0)
neg_minutes2use = int(minutes2use/2.0)

music_minutes2use = (neg_minutes2use/2)*0
podcast_minutes2use = (neg_minutes2use/2)*2
assert music_minutes2use + podcast_minutes2use == neg_minutes2use, 'music + podcast != neg'

# calculate train and test minutes splitting
# train
pos_train_minutes = round(train_fraction*pos_minutes2use, 2) # number of pos audio training minutes
music_train_minutes = round(train_fraction*music_minutes2use, 2) # number of minutes for music training
podcast_train_minutes = round(train_fraction*podcast_minutes2use, 2) # number of minutes for podcast training
# test
pos_test_minutes = pos_minutes2use - pos_train_minutes
music_test_minutes = music_minutes2use - music_train_minutes
if podcast_minutes2use:
    podcast_test_minutes = podcast_minutes2use - podcast_train_minutes
else:
    podcast_test_minutes = 0

print('Using ' + str(pos_train_minutes) + ' minutes of positives for training and '
      + str(pos_test_minutes) + ' for testing')
print('Using ' + str(music_train_minutes) + ' minutes of music and ' 
      + str(podcast_train_minutes) +' minutes of podcasts for training')
print('Using ' + str(music_test_minutes) + ' minutes of music and ' 
      + str(podcast_test_minutes) +' minutes of podcasts for testing')

# calculate number of files
# train
n_music_train_files = int(music_train_minutes/music_duration)
n_podcast_train_files = int(podcast_train_minutes/podcast_duration)
n_pos_train_files = int(pos_train_minutes/ads_duration)
# test
n_music_test_files = int(music_test_minutes/music_duration)
n_podcast_test_files = int(podcast_test_minutes/podcast_duration)
n_pos_test_files = int(pos_test_minutes/ads_duration)
print('---------------------------------------------------------------')
print('This translates into:')
print( str(n_music_train_files) + ' files of music' + ' and ' 
      + str(n_podcast_train_files) + ' files of podcasts for training')
print( str(n_music_test_files) + ' files of music' + ' and ' 
      + str(n_podcast_test_files) + ' files of podcasts for testing')
print(str(n_pos_train_files) + ' poitive files for training')
print(str(n_pos_test_files) + ' poitive files for testing')

assert len(music_files) >= n_music_train_files + \
n_music_test_files, 'There are not enough music files for that!'
assert len(podcast_files) >= n_podcast_train_files + \
n_podcast_test_files, 'There are not enough podcast files for that!'<jupyter_output>Using 450.0 minutes of positives for training and 50.0 for testing
Using 0.0 minutes of music and 450.0 minutes of podcasts for training
Using 0.0 minutes of music and 50.0 minutes of podcasts for testing
---------------------------------------------------------------
This translates into:
0 files of music and 2250 files of podcasts for training
0 files of music and 250 files of podcasts for testing
900 poitive files for training
100 poitive files for testing<jupyter_text>Create a generator based on the above:<jupyter_code>train_files = [] # a list of training files 
test_files = [] # a list of test files 

# shuffle files
np.random.seed(1)
np.random.shuffle(pos_files)
np.random.seed(2)
np.random.shuffle(music_files)
np.random.seed(3)
np.random.shuffle(podcast_files)

'''Collect a balanced list of files + add labels'''
# Training list
for f in pos_files[:n_pos_train_files]:
    train_files.append([f,1])
for f in music_files[:n_music_train_files]:
    train_files.append([f,0])
for f in podcast_files[:n_podcast_train_files]:
    train_files.append([f,0])

# Test list
for f in pos_files[n_pos_train_files:n_pos_train_files + n_pos_test_files]:
    test_files.append([f,1])
for f in music_files[n_music_train_files:n_music_train_files + n_music_test_files]:
    test_files.append([f,0])
for f in podcast_files[n_podcast_train_files:n_podcast_train_files + n_podcast_test_files]:
    test_files.append([f,0])

train_generator = DataGenerator_Sup(train_files, dataset='train', CNN=True)
test_generator = DataGenerator_Sup(test_files, batch_size=40, dataset='test', CNN=True)<jupyter_output><empty_output><jupyter_text>Check generators:<jupyter_code>len(train_generator)
len(test_generator)
if 1:
    X, Y = train_generator.__getitem__(2) # get item
    print(X.shape)
    print(Y.shape)
    print('Positive example fraction in batch is ' + str((sum(Y == 1)/Y.shape[0])[0]))<jupyter_output>(51, 13, 130, 1)
(51, 1)
Positive example fraction in batch is 0.6470588235294118<jupyter_text>Train CNN model: Create and compile model:<jupyter_code>model = create_CNN_model()
filepath = 'models/weights_LeNet5ish_1000_only_podcasts_and_ads_3epochs.hdf5'
checkpoint = ModelCheckpoint(filepath)
model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])<jupyter_output>WARNING:tensorflow:From /home/ubuntu/anaconda3/envs/tensorflow_p36/lib/python3.6/site-packages/tensorflow/python/framework/op_def_library.py:263: colocate_with (from tensorflow.python.framework.ops) is deprecated and will be removed in a future version.
Instructions for updating:
Colocations handled automatically by placer.
_________________________________________________________________
Layer (type)                 Output Shape              Param #   
=================================================================
conv2d_1 (Conv2D)            (None, 13, 130, 16)       160       
_________________________________________________________________
conv2d_2 (Conv2D)            (None, 13, 130, 16)       2320      
_________________________________________________________________
flatten_1 (Flatten)          (None, 27040)             0         
_________________________________________________________________
dense_1 (Dense)              (None, 32)                865312    
______________[...]<jupyter_text>Train model:<jupyter_code>H = model.fit_generator(generator = train_generator, epochs = 3, callbacks = [checkpoint])
history_list.append(H)
plt.rcParams.update({'font.size': 16})
plt.figure(figsize= (10,4))
plt.subplot(1,2,1)
plt.plot(H.history['loss'], linewidth=3, color = 'b')
plt.title('Loss')
plt.xlabel('epoch')

plt.subplot(1,2,2)
plt.plot(H.history['acc'], linewidth=3, color = 'g')
plt.title('Accuracy')
plt.xlabel('epoch')
plt.tight_layout()
# history_list = []
# history_list.append(H)
# Error files:
train_generator.err_files<jupyter_output><empty_output><jupyter_text>Check Accuracy:<jupyter_code>filepath = 'models/weights_LeNet5ish_1000_only_podcasts_and_ads_3epochs.hdf5'
# recreate a model for evaluation
eval_model = create_CNN_model()
# load weights
eval_model.load_weights(filepath)
# compile
eval_model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])<jupyter_output>WARNING:tensorflow:From /home/ubuntu/anaconda3/envs/tensorflow_p36/lib/python3.6/site-packages/tensorflow/python/framework/op_def_library.py:263: colocate_with (from tensorflow.python.framework.ops) is deprecated and will be removed in a future version.
Instructions for updating:
Colocations handled automatically by placer.
_________________________________________________________________
Layer (type)                 Output Shape              Param #   
=================================================================
conv2d_1 (Conv2D)            (None, 13, 130, 16)       160       
_________________________________________________________________
conv2d_2 (Conv2D)            (None, 13, 130, 16)       2320      
_________________________________________________________________
flatten_1 (Flatten)          (None, 27040)             0         
_________________________________________________________________
dense_1 (Dense)              (None, 32)                865312    
______________[...]<jupyter_text>What metrics do we use for evaluation?<jupyter_code>eval_model.metrics_names<jupyter_output><empty_output><jupyter_text>How does the model perform on the test?<jupyter_code>eval_model.evaluate_generator(generator = test_generator, steps = 2)
X,Y = test_generator.__getitem__(1)
Y_pred = eval_model.predict(X)
plt.rcParams.update({'font.size': 14})
plot_confusion_matrix(Y, Y_pred>0.8, ['Non Ad', 'Ad'])
loss = []
acc = []
for l in range(len(history_list)):
    loss += history_list[l].history['loss']
    acc += history_list[l].history['acc']
plt.rcParams.update({'font.size': 16})
plt.figure(figsize= (10,4))
plt.subplot(1,2,1)
plt.plot(loss, linewidth=3, color = 'b')
plt.title('Loss')
plt.xlabel('epoch')

plt.subplot(1,2,2)
plt.plot(acc, linewidth=3, color = 'g')
plt.title('Accuracy')
plt.xlabel('epoch')
plt.tight_layout()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.12646328485278469, 'fraction_numerical': 0.027492018446257538, 'mean_word_length': 5.0111940298507465, 'pattern_counts': {'":': 0, '<': 39, '<?xml version=': 0, '>': 42, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 3, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '11', 'dir': 'jupyter-structured-clean-dedup', 'id': '92410', 'license': 'MIT', 'n_tokens_mistral': 3802, 'n_tokens_neox': 3493, 'n_words': 834, 'path': 'notebooks/Train_CNN_model_ads_vs_podcasts.ipynb', 'repo_name': 'ohadmich/Adetector', 'size': '11276'}