<jupyter_start><jupyter_text>Sound Event Detection using Approach 2 This colab demonstrates how to extract the AudioSet embeddings, using a VGGish deep neural network (DNN). Based on the directions at: https://github.com/tensorflow/models/tree/master/research/audioset<jupyter_code>!lscpu
!nvidia-smi
#Google drive access
import os
from google.colab import drive
drive.mount('/content/gdrive',force_remount = True)

#Directory
root_path = 'gdrive/My Drive/SoundEventDetection/modelTraining/'
os.chdir(root_path)
%%capture
#Install necessary software
!pip install six
!pip install h5py
!pip install pydub
!pip install numpy
!pip install scipy
!pip install keras
!pip install future
!pip install resampy
!pip install ipython
!pip install soundfile
!pip install pysoundfile
!pip install scikit-learn
!apt-get install libsndfile1dense_15

!pip install python==3.6
!pip install matplotlib
!pip install cudnn==7.1.2
!pip install cudatoolkit==9
!pip install tensorflow-gpu==1.12.0

#Install: cuda-repo-ubuntu1604-9-0-local_9.0.176-1_amd64-deb
!dpkg -i cuda-repo-ubuntu1604-9-0-local_9.0.176-1_amd64-deb
!apt-key add /var/cuda-repo-9-0-local/7fa2af80.pub
!apt-get update
!apt-get install cuda=9.0.176-1
#OR
#!wget https://developer.nvidia.com/compute/cuda/9.0/Prod/local_installers/cuda-repo-ubuntu1604-9-0-local_9.0.176-1_amd64-deb

#VGGish model checkpoint, in TensorFlow checkpoint format.
import os 
os.chdir("trained_models")
!wget https://storage.googleapis.com/audioset/vggish_model.ckpt
os.chdir("..")
%%capture
#Important DO NOT DELETE
#import six
import sys
#import h5py
#import math
#import glob
import json
#import time
import numpy as np
import pandas as pd
import soundfile as sf
import tensorflow as tf
import matplotlib.pyplot as plt
#from pydub.playback import play
#from pydub import AudioSegment
#from scipy.io import wavfile
#from scipy.io.wavfile import write

sys.path.insert(1, os.path.join(sys.path[0], '../'))

#ML imports
#from keras.layers import Input, Dense, BatchNormalization, Dropout, Lambda, Activation, Concatenate
from keras.layers import Lambda
from keras.optimizers import Adam
from keras.models import load_model
from keras.models import Model
import keras.backend as K

#External .py scripts
from lib import mel_features
from lib import vggish_input
from lib import vggish_params
from lib import vggish_postprocess
from lib import vggish_slim
from lib import utilities
from lib import data_generator
from lib.train_functions import evaluateCore, trainCore, average_pooling, max_pooling, attention_pooling, pooling_shape, train, writeToFile
%%capture
trainingType = ['decision_level_max_pooling', 
                'decision_level_average_pooling', 
                'decision_level_single_attention', 
                'decision_level_multi_attention', 
                'feature_level_attention']

balanced = ['no_balance', 'balance_in_batch']


def loadModel(num1,num2):
  args = {
          "data_dir" : "packed_features/",
          "workspace" : "workspace/",
          "mini_data" : False,
          "balance_type" : balanced[num2], #'no_balance', 'balance_in_batch'
          "model_type" : trainingType[num1], 
          "learning_rate" : 1e-3,
          "filename":utilities.get_filename("work/")
  }

  #Output directories
  sub_dir = os.path.join(args["filename"],
                        'balance_type={}'.format(args["balance_type"]),
                        'model_type={}'.format(args["model_type"]))
  
  modelFiles = [['200k_DLMP_B','200k_DLMP_U'],['200k_DLAP_B','200k_DLAP_U'],
                ['200k_DLSA_B','200k_DLSA_U'],['200k_DLMA_B','200k_DLMA_U'],
                ['200k_FLA_B','200k_FLA_U']]
                

  #CNN Architecture
  model = train(args, 1)
  optimizer = Adam(lr=args["learning_rate"])
  model.compile(loss='binary_crossentropy', optimizer=optimizer)

  #Model dir
  #models_dir ="workspace/models/work/balance_type=balance_in_batch/model_type=feature_level_attention"
  models_dir = "trained_models/"+modelFiles[num1][num2]+".h5"

  #latest_file ="workspace/models/work/balance_type=no_balance/model_type=decision_level_multi_attention/md_4000_iters.h5"
  print(models_dir)
  model.load_weights(models_dir)
  print(type(model))
  return model
  
model = loadModel(2,1)
'''
from keras.models import load_model
model = load_model( "trained_models/"+"200k_DLSA_U"+".h5")
'''
#start_time = time.time()
def getTime(num, start_time):
  print(str(num)+ ":" + str(time.time() - start_time))
  return time.time()
model.summary()
#Get labels
class_labels = pd.read_csv("lib/class.csv").drop(columns=['index','mid'])['display_name'].to_numpy()

def getLabeling(class_labels,num):
   return class_labels[num]
def scale_att(inputs, **kwargs):
  [out, att] = inputs
  epsilon = 1e-7
  att = K.clip(att, epsilon, 1. - epsilon)
  normalized_att = att / K.sum(att, axis=1)[:, None, :]
  return out * normalized_att

#Predict on input
def predictOnVggishInput(df, model):
  results = []
  attention_results = []
  #Handling edge case of 10 seconds
  length = df.shape[0]
  if(length==10):
    length = 11
  print("length " + str(length))
  for endNumber in range(10,length,10):
    startNumber = endNumber - 10
    df_final = [df[startNumber:endNumber]]
    df_final = np.asarray(df_final)
    df_final = (np.float32(df_final) - 128.) / 128.
    #Normal output
    output = model.predict(df_final)
    results.append(output.astype(np.float32))
    #Attention output

    #Normalize the attn
    try:
      attention = model.get_layer('dense_15').output
      output = model.get_layer('dense_14').output
    except:
      attention = model.get_layer('dense_5').output
      output = model.get_layer('dense_4').output

    normalized_attention = Lambda(scale_att)([output,attention])

    intermediate_model=Model(inputs=model.input,outputs=normalized_attention)

    attention_layer_prediction=intermediate_model.predict(df_final)
    attention_results.append(attention_layer_prediction)
    
  return results,attention_results
def plotResults(df):
  #Create plot
  plt.rcParams["figure.figsize"] = (20,10)
  groups = df.groupby('Class')
  # Plot
  fig, ax = plt.subplots()
  ax.margins(0.05) # Optional, just adds 5% padding to the autoscaling
  for name, group in groups:
      ax.plot(group.StartTime, group.Probability, marker='o', linestyle='-', ms=6, label=getLabeling(class_labels,name))

  f = "output/dogprobsld.csv"
  df = pd.read_csv(f)
  plt.plot(df[df['Class']==0]['Second'],df[df['Class']==0]['Class']+1.1, marker='.', linestyle='', ms=5,color='C0', label="Speech SLD") 
  plt.plot(df[df['Class']==74]['Second'],df[df['Class']==74]['Class']-72.8, marker='.', linestyle='', ms=5,color='C3', label="Dog SLD")

  ax.legend()
  plt.show()
def vggish_fullprocess(wav_file,data,sr,switch):

  # wav_file = "output/dog.wav" #tempSave
  checkpoint = "trained_models/vggish_model.ckpt"
  pca_params = "trained_models/vggish_pca_params.npz"

  if switch==1:
    examples_batch = vggish_input.wavfile_to_examples(wav_file)
  else:
    examples_batch = vggish_input.waveform_to_examples(data,sr)

  # Prepare a postprocessor to munge the model embeddings.
  pproc = vggish_postprocess.Postprocessor(pca_params)
  with tf.Graph().as_default(), tf.Session() as sess:
    # Define the model in inference mode, load the checkpoint, and
    # locate input and outputgetTime tensors.
    vggish_slim.define_vggish_slim(training=False)
    vggish_slim.load_vggish_slim_checkpoint(sess, checkpoint)
    features_tensor = sess.graph.get_tensor_by_name(vggish_params.INPUT_TENSOR_NAME)
    embedding_tensor = sess.graph.get_tensor_by_name(vggish_params.OUTPUT_TENSOR_NAME)

    # Run inference and postprocessing.
    [embedding_batch] = sess.run([embedding_tensor],feed_dict={features_tensor: examples_batch})

  postprocessed_batch = pproc.postprocess(embedding_batch)
  return postprocessed_batch

# #Load vggish output
# filename = 'output/data.h5'

wav_file='output/dog.wav'
wav_data, sr = sf.read(wav_file, dtype='int16')
#assert wav_data.dtype == np.int16, 'Bad sample type: %r' % wav_data.dtype
samples = wav_data / 32768.0  # Convert to [-1.0, +1.0]
if len(samples.shape) > 1:
  data = np.mean(samples, axis=1)
else:
  data = samples
data = np.asarray(data)
result_list = []
attention_list = []

step = int(data.shape[0]/sr)
#step = 30
for segment in np.arange(0,int(data.shape[0]/sr),step):
  if((data.shape[0]/sr)-segment<step):
    break
  data_segment = data[sr*segment:sr*(segment+step)]
  data_embedding = vggish_fullprocess(wav_file,data_segment,sr,2)
  result,attention_results = predictOnVggishInput(data_embedding, model)

  result_list.append(result)
  attention_list.append(attention_results)
#Take the bucketed results and make them ready for plotting
#attention_table = pd.DataFrame(columns=['Second','Class','Probability'])

attention_new_list = []

for attention_segment in attention_list:
   for bucket in attention_segment:
     for second in bucket[0]:
       attention_new_list.append(second)

attention_table = np.array(attention_new_list).transpose()
print(attention_table.shape)
def thresholding_algo(y, lag, threshold, influence):
    signals = np.zeros(len(y))
    filteredY = np.array(y)
    avgFilter = [0]*len(y)
    stdFilter = [0]*len(y)
    avgFilter[lag - 1] = np.mean(y[0:lag])
    stdFilter[lag - 1] = np.std(y[0:lag])
    for i in range(lag, len(y)):
        if abs(y[i] - avgFilter[i-1]) > threshold * stdFilter [i-1]:
            if y[i] > avgFilter[i-1]:
                signals[i] = 1
            else:
                signals[i] = -1

            filteredY[i] = influence * y[i] + (1 - influence) * filteredY[i-1]
            avgFilter[i] = np.mean(filteredY[(i-lag+1):i+1])
            stdFilter[i] = np.std(filteredY[(i-lag+1):i+1])
        else:
            signals[i] = 0
            filteredY[i] = y[i]
            avgFilter[i] = np.mean(filteredY[(i-lag+1):i+1])
            stdFilter[i] = np.std(filteredY[(i-lag+1):i+1])

    return dict(signals = np.asarray(signals),
                avgFilter = np.asarray(avgFilter),
                stdFilter = np.asarray(stdFilter))
#Plot the attention

classes = [0,137,300,307,353]

classes = [74]

# Settings: lag = 30, threshold = 5, influence = 0

lag = 15

threshold = 2

influence = 0.01

from scipy.signal import savgol_filter
# Run algo with settings from above
#plt.plot(abs(np.gradient(attention_table[0],edge_order=1)))
#plt.plot(attention_table[classes,:],marker='o', linestyle='-', ms=6, label=getLabeling(class_labels,classes))
#plt.plot(abs(np.gradient(attention_table[74],edge_order=1)))
plt.rcParams["figure.figsize"] = (20,10)
fig, ax = plt.subplots()
ax.margins(0.05) # Optional, just adds 5% padding to the autoscaling
for className in classes:
    ax.plot(attention_table[className], marker='o', linestyle='-', ms=6, label=getLabeling(class_labels,className))
    ax.plot(abs(np.gradient(attention_table[className],edge_order=1))-0.3, marker='o', linestyle='-', ms=6, label=str('Grad_')+getLabeling(class_labels,className))
    yhat = savgol_filter(attention_table[className], 5, 1)
    #ax.plot(yhat-0.3, marker='o', linestyle='-', ms=6, label=str('S')+getLabeling(class_labels,className))
    k=0.6
    ax.plot(list(map(lambda x: -0.4 if x>(-0.59) else -0.6, yhat-k)), marker='o', linestyle='-', ms=6,color='C3', label=str('SGrad')+getLabeling(class_labels,className))

    #result = thresholding_algo(attention_table[className], lag=lag, threshold=threshold, influence=influence)
    #ax.plot(list(map(lambda x: abs(x),result['signals'])), marker='o', linestyle='-', ms=6, label=str('grad_')+getLabeling(class_labels,className))

f = "output/dogprobsld.csv"
df = pd.read_csv(f)
#plt.plot(df[df['Class']==0]['Second'],df[df['Class']==0]['Class']-0.7, marker='o', linestyle='', ms=6,color='C3', label="Speech SLD") 
#plt.plot(df[df['Class']==74]['Second'],df[df['Class']==74]['Class']-74.7, marker='o', linestyle='', ms=6,color='C3', label="Dog SLD")

ax.legend()
plt.xlim(0,180)
#plt.ylim(-0.62,)
plt.show()
# Manipulate results
result_table = pd.DataFrame(columns=['StartTime','Class','Probability'])
i = 0
threshold = 0.1
for result_segment in result_list:
  for result in result_segment:
    result = result[0]
    
    classes = np.arange(len(result))[result>threshold]
    probs = result[result>threshold]
    for j in np.arange(len(probs)):
      result_table = result_table.append({'StartTime' : 0+i,'Class' : classes[j] , 'Probability' : probs[j]}, ignore_index=True)
    i=i+1

  result_table["Class"] = result_table["Class"].astype(int)

# Change column type
# result_table['Class'] = pd.Categorical(result_table.Class)
plotResults(result_table)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1435349940688019, 'fraction_numerical': 0.025227362593910636, 'mean_word_length': 4.663233318405732, 'pattern_counts': {'":': 2, '<': 6, '<?xml version=': 0, '>': 11, 'https://': 3, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '386440', 'license': 'Apache-2.0', 'n_tokens_mistral': 4752, 'n_tokens_neox': 4464, 'n_words': 859, 'path': 'modelTraining/3_Approach_2.ipynb', 'repo_name': 'PatrickLewandowski/SoundEventDetection', 'size': '12645'}