<jupyter_start><jupyter_text>Preprocessing we create silence files, pad short files and make training and validation sets<jupyter_code>import numpy as np
import librosa
import librosa.display
import os
import csv
%matplotlib inline
import matplotlib.pyplot as plt
import soundfile as sf
!ls /dev/shm/kmrozowski/
data_path = '/dev/shm/kmrozowski/'
train_dir = data_path + './data/train/audio/' #download files from kaggle

classes = ['yes', 'no', 
           'up', 'down', 
           'left', 'right', 
           'on', 'off', 
           'stop', 'go',
         #   'zero', 'one', 'two', 'three', 'four', 'five', 'six', 'seven', 'eight', 'nine',
           'silence']<jupyter_output><empty_output><jupyter_text>RUN ONLY ONCE in your lifetime! This cell moves the background noises folder out of the audio directory. We will create silence samples from these files after.<jupyter_code>%%bash
mv {data_path}/data/train/audio/_background_noise_ data/train
ls {data_path}/data/train<jupyter_output>LICENSE
README.md
_background_noise_
audio
testing_list.txt
validation_list.txt<jupyter_text>Split all the \_background\_noises\_ into 1-sec audio tracks<jupyter_code>def split_arr(arr):
    """
    split an array into chunks of length 16000
    Returns:
        list of arrays
    """
    return np.split(arr, np.arange(16000, len(arr), 16000))
def create_silence():
   """
   reads wav files in background noises folder, 
   splits them and saves to silence folder in train_dir
   """
   sampleRate = 16000 # hertz
   for file in os.listdir('data/train/_background_noise_/'):
      if 'wav' in file:
         sig, rate = librosa.load('data/train/_background_noise_/' + file, sr = 16000)        
         sig_arr = split_arr(sig)
         if not os.path.exists(train_dir+'silence/'):
               os.makedirs(train_dir+'silence/')
         for ind, arr in enumerate(sig_arr):
               filename = 'frag%d' %ind + '_%s' %file # example: frag0_running_tap.wav
               sf.write(train_dir+'silence/'+filename, arr, sampleRate)
  
create_silence()<jupyter_output><empty_output><jupyter_text>It is probably a good idea to make more silence samples yourself. Perhaps just by recording walking or driving around without speaking. The silence class is underrepresented.  three lists with file names. one for training set, one for validation set, one for all. Plus a dictionary with file counts per class.<jupyter_code>folders = os.listdir(train_dir)
# put folders in same order as in the classes list, used when making sets
all_classes = [x for x in classes]
for ind, cl in enumerate(folders):
    if cl not in classes:
        all_classes.append(cl)
print(all_classes)
with open(data_path + './data/train/validation_list.txt') as val_list:
    validation_list = [row[0] for row in csv.reader(val_list)]
assert len(validation_list) == 6798, 'file not loaded'
print(len(validation_list))

"""
#if you want to add the files in testing_list.txt to the validation list:
with open(data_path + './data/train/testing_list.txt') as test_list:
    testing_list = [row[0] for row in csv.reader(test_list)]
assert len(testing_list) == 6835, 'file not loaded'
#combine into validation set
validation_list.extend(testing_list)
"""

#add silence files to validation_list
for i, file in enumerate(os.listdir(train_dir + 'silence/')):
    if i%10==0:
        validation_list.append('silence/'+file)

training_list = []
all_files_list = []
class_counts = {}

for folder in folders:
    files = os.listdir(train_dir + folder)
    for i, f in enumerate(files):
        all_files_list.append(folder + '/' + f)
        path = folder + '/' + f
        if path not in validation_list:
            training_list.append(folder + '/' + f)        
        class_counts[folder] = i

#remove filenames from validation_list that don't exist anymore (due to eda)
validation_list = list(set(validation_list).intersection(all_files_list))
print(len(validation_list))
path = '/dev/shm/kmrozowski/'
np.savetxt(path + 'validation_list.csv',validation_list, delimiter =" ",  fmt ='% s') 
assert len(validation_list)+len(training_list)==len(all_files_list), 'error'
# check random file name
print(training_list[345], 'size training set: ',len(training_list), 'size validation set: ', len(validation_list))
print(class_counts)
# {'tree': 1732, 'sheila': 1733, 'bird': 1730, 'no': 2374, 'four': 2371, 'zero': 2375, 'up': 2374, 'five': 2356, 'cat': 1732, 'yes': 2376, 'eight': 2351, 'off': 2356, 'seven': 2376, 'house': 1749, 'happy': 1741, 'three': 2355, 'left': 2352, 'two': 2372, 'bed': 1712, 'nine': 2363, 'dog': 1745, 'down': 2358, 'wow': 1744, 'right': 2366, 'on': 2366, 'one': 2369, 'go': 2371, 'marvin': 1745, 'stop': 2379, 'six': 2368, 'silence': 401}<jupyter_output>{'silence': 401, 'zero': 2375, 'yes': 2376, 'wow': 1744, 'up': 2374, 'two': 2372, 'tree': 1732, 'three': 2355, 'stop': 2379, 'six': 2368, 'sheila': 1733, 'seven': 2376, 'right': 2366, 'one': 2369, 'on': 2366, 'off': 2356, 'no': 2374, 'nine': 2363, 'marvin': 1745, 'left': 2352, 'house': 1749, 'happy': 1741, 'go': 2371, 'four': 2371, 'five': 2356, 'eight': 2351, 'down': 2358, 'dog': 1745, 'cat': 1732, 'bird': 1730, 'bed': 1712}<jupyter_text>Plot a wav file<jupyter_code>x, r = librosa.load(train_dir + 'yes/bfdb9801_nohash_0.wav', sr = 16000)
print('min: ',np.min(x), 
      '\nmax: ', np.max(x), 
      '\nmean: ', np.mean(x),
      '\nmedian: ', np.median(x),
      '\nvariance: ', np.var(x),
      '\nlength: ', len(x))
plt.plot(x)<jupyter_output>min:  -0.1182251 
max:  0.10827637 
mean:  7.842428e-06 
median:  0.0 
variance:  0.0003236237 
length:  10923<jupyter_text>Turning all wav files into spectrograms<jupyter_code>def make_spec(file, file_dir = train_dir, flip = False, ps = False, st = 4):
    """
    create a melspectrogram from the amplitude of the sound
    
    Args:
        file (str): filename
        file_dir (str): directory path
        flip (bool): reverse time axis
        ps (bool): pitch shift
        st (int): half-note steps for pitch shift
    Returns:
        np.array with shape (122,85) (time, freq)
    """
    sig, rate = librosa.load(file_dir + file, sr = 16000)
    if len(sig) < 16000: # pad shorter than 1 sec audio with ramp to zero
        sig = np.pad(sig, (0,16000-len(sig)), 'linear_ramp')
    if ps:
        sig = librosa.effects.pitch_shift(sig, rate, st)
    D = librosa.amplitude_to_db(librosa.stft(sig[:16000], n_fft = 512, 
                                             hop_length = 128, 
                                             center = False), ref = np.max)
    S = librosa.feature.melspectrogram(S=D, n_mels = 85).T
    if flip:
        S = np.flipud(S)
    return S.astype(np.float32)
librosa.display.specshow(make_spec('yes/bfdb9801_nohash_0.wav'), 
                         x_axis='mel',
                         fmax=8000, 
                         y_axis='time', 
                         sr = 16000,
                         hop_length = 128)
make_spec('yes/bfdb9801_nohash_0.wav').shape
def create_sets(file_list = training_list):
    X_array = np.zeros([len(file_list),122,85])
    Y_array = np.zeros([len(file_list)])    
    for ind, file in enumerate(file_list):
        if ind%2000 == 0:
            print(ind, file)
        try:
            X_array[ind] = make_spec(file)
        except ValueError:
            print(ind, file, ValueError)
        Y_array[ind] = all_classes.index(file.rsplit('/')[0])
        
    return X_array, Y_array        
        
def print_sets(file_list = training_list):
    for ind, file in enumerate(file_list):
        if ind%2000 == 0:
            print(ind, file, file.rsplit('/')[0], all_classes.index(file.rsplit('/')[0]))
print_sets()
X_train, Y_train_all = create_sets() # takes a while
# 0 yes/8a28231e_nohash_3.wav
# 2000 yes/21307344_nohash_0.wav
# 4000 silence/frag55_dude_miaowing.wav
# 6000 nine/283d7a53_nohash_1.wav
# 8000 four/31d31fa0_nohash_0.wav
# 10000 wow/d84829e0_nohash_0.wav
# 12000 five/173e6bbf_nohash_0.wav
# 14000 stop/bbd0bbd0_nohash_4.wav
# 16000 off/29fb33da_nohash_0.wav
# 18000 zero/39543cfd_nohash_1.wav
# 20000 tree/6a203e0e_nohash_3.wav
# 22000 up/01b4757a_nohash_0.wav
# 24000 six/21cbe292_nohash_0.wav
# 26000 one/3b852f6f_nohash_0.wav
# 28000 down/b959cd0c_nohash_4.wav
# 30000 three/23abe1c9_nohash_1.wav
# 32000 three/f953e1af_nohash_1.wav
# 34000 right/75915c90_nohash_0.wav
# 36000 eight/85b877b5_nohash_0.wav
# 38000 no/88053e92_nohash_0.wav
# 40000 two/a1c63f25_nohash_0.wav
# 42000 two/c1d39ce8_nohash_8.wav
# 44000 seven/72ca6a6d_nohash_0.wav
# 46000 on/0137b3f4_nohash_3.wav
# 48000 left/1b4c9b89_nohash_4.wav
# 50000 dog/dfb6450b_nohash_0.wav
# 52000 go/0137b3f4_nohash_2.wav
# 54000 sheila/1ecfb537_nohash_0.wav
# 56000 bed/129c7d8d_nohash_0.wav
# 58000 bird/b9f46737_nohash_0.wav
# all unknown are index 11
Y_train = np.where(Y_train_all < 11, Y_train_all, 11)
print(max(Y_train_all), max(Y_train))
print(len(Y_train) == len(Y_train_all), 12 in Y_train)
X_train.shape
Y_train_all.shape
Y_train.shape
librosa.display.specshow(X_train[6500], 
                         x_axis='mel', 
                         fmax=8000, 
                         y_axis='time', 
                         sr = 16000,
                         hop_length = 128)<jupyter_output><empty_output><jupyter_text>Histogram of trainset values<jupyter_code>print('min: ',np.min(X_train), 
      '\nmax: ', np.max(X_train), 
      '\nmean: ', np.mean(X_train),
      '\nmedian: ', np.median(X_train),
      '\nvariance: ', np.var(X_train))
plt.hist(X_train.flatten(), bins = 50)<jupyter_output><empty_output><jupyter_text>save the training sets, add channel dimension for keras, normalize around zero<jupyter_code>np.save('data/X_train.npy', np.expand_dims(X_train, -1) + np.mean(X_train))
np.save('data/Y_train.npy', Y_train.astype(np.int))
np.save('data/Y_train_all.npy', Y_train_all.astype(np.int))
print(len(validation_list))
X_val, Y_val_all = create_sets(file_list = validation_list)
Y_val = np.where(Y_val_all < 11, Y_val_all, 11)
print(Y_val.shape)<jupyter_output>(6839,)<jupyter_text>Histogram of validation data values<jupyter_code>plt.hist(X_val.flatten(), bins = 50)<jupyter_output><empty_output><jupyter_text>To speed up eventual training of the network I decided to do most of the preprocessing separately and save individual train and validation sets as numpy arrays in .npy files.<jupyter_code>np.save(data_path + 'data/X_val.npy', np.expand_dims(X_val, -1) + np.mean(X_train))
np.save(data_path + 'data/Y_val.npy', Y_val.astype(np.int64))
np.save(data_path + 'data/Y_val_all.npy', Y_val_all.astype(np.int64))<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.13379817205314237, 'fraction_numerical': 0.07387166682370677, 'mean_word_length': 3.608771168041685, 'pattern_counts': {'":': 0, '<': 40, '<?xml version=': 0, '>': 37, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 5, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '10', 'dir': 'jupyter-structured-clean-dedup', 'id': '597424', 'license': 'MIT', 'n_tokens_mistral': 4518, 'n_tokens_neox': 3920, 'n_words': 899, 'path': 'speech-recognition/preproc.ipynb', 'repo_name': 'kpmrozowski/wut-mini-deep-learning', 'size': '10613'}