<jupyter_start><jupyter_text>Demonstration NotebookTrain a model to recognize notes from input sounds Part 1: Data Preparation and ExplorationBy <NAME> \For Bnaura&copy; 2021 <NAME>  Contents1. [Import Libraries](lib_import)1. [Data Import](data_import)1. [Data Exploration](data_explore)1. [Data Augmentation](data_aug)1. [Feature Extraction](feat_extract)1. [Save Features](feat_save) TO DO- Add column to database for train=1, test=0- Generalize training functions to look for any files matching corresponding note tag  - Add _Male2 recordings- Submodule repo into simple_gui<jupyter_code>%load_ext autoreload
%autoreload 2<jupyter_output><empty_output><jupyter_text>1. Import Libraries<jupyter_code>import sys
import os
import datetime
import time

import numpy as np
import pandas as pd

import matplotlib.pyplot as plt
import ipywidgets as widgets

from sklearn.model_selection import train_test_split
from sklearn.metrics import accuracy_score
from sklearn import svm
from sklearn.preprocessing import LabelEncoder

import xgboost as xgb

import pickle
import sqlite3

from scipy.io import wavfile as wav
from IPython.display import Audio

# Add custom modules to path
module_path = os.path.abspath(os.path.join('..'))
if module_path not in sys.path:
    sys.path.append(module_path)

import util.music_util as mus 
from util.ml_util import feat_extract, load_training_data
from util import DATA_FOLDER, SCALE<jupyter_output>pygame 1.9.6
Hello from the pygame community. https://www.pygame.org/contribute.html
../1_audio\hum\Hum_Db4.wav does not exist
../1_audio\hum\Hum_Eb4.wav does not exist
../1_audio\hum\Hum_Gb4.wav does not exist
../1_audio\hum\Hum_Ab4.wav does not exist
../1_audio\hum\Hum_Bb4.wav does not exist
../1_audio\hum\Hum_B4.wav does not exist<jupyter_text>2. Data Import<jupyter_code>X, y, fs = load_training_data(SCALE)<jupyter_output>C:\Users\benja\Documents\Python\note-recognition\util\ml_util.py:124: WavFileWarning: Chunk (non-data) not understood, skipping it.
  fs, signal = wav.read(training_data[note])<jupyter_text>3. Data Exploration  OverviewSanity check the dimensions of X and y<jupyter_code>print('X is a {} with shape {}'.format(type(X), X.shape))
print('X has {} rows, and expected # of rows from notes = {}'.format(X.shape[0], len(SCALE)))
print('X sample length is {}'.format(X.shape[1]))
print('y has {} samples, and expected # of samples from notes = {}'.format(len(y), len(SCALE)))<jupyter_output>X is a <class 'numpy.ndarray'> with shape (6, 120000)
X has 6 rows, and expected # of rows from notes = 6
X sample length is 120000
y has 6 samples, and expected # of samples from notes = 6<jupyter_text>Plot Waveform<jupyter_code># Pick a note to plot
idx_test = 1
signal_test = X[idx_test,:]
note_test = SCALE[idx_test]

# Create array of time samples, converting number of samples and dividing by sample of frequency 
t_array = np.arange(X.shape[1])/fs

# Plot waveform over short time period to see sine
plt.subplot(121)
plt.xlim(0.1,0.9)
plt.xlabel('Time (s)')
plt.title('Hummed {}: {} Hz'.format(note_test, mus.note_to_freq[note_test]))
plt.plot(t_array, signal_test)

# Plot waveform over shorter time period to see sine
plt.subplot(122)
plt.xlim(0.5 ,0.55)
plt.xlabel('Time (s)')
plt.title('Hummed {}: {} Hz'.format(note_test, mus.note_to_freq[note_test]))
plt.plot(t_array, signal_test)

plt.show()<jupyter_output><empty_output><jupyter_text>Plot Frequency<jupyter_code># Take FFT
ftransform = np.fft.fft(signal_test)/len(signal_test)
ftransform = ftransform[range(int(len(signal_test)/2))]
tp_count = len(signal_test)
vals = np.arange(tp_count/2)
t_period = tp_count/fs

# Scale to Hz frequency values for plotting
freqs = vals/t_period

plt.plot(freqs, abs(ftransform))
plt.xlim((0, 2*mus.note_to_freq[note_test]))
plt.xlabel('Frequency (Hz)')
plt.title('Hummed {}: {} Hz'.format(note_test, mus.note_to_freq[note_test]))
plt.show()<jupyter_output><empty_output><jupyter_text>4. Data Augmentation Synthesize dataset `aug_factor` times larger<jupyter_code>aug_factor = 20

# Initialize matrix where each row contains a noisy sample
X_aug = np.empty((aug_factor*X.shape[0], X.shape[1]))
y_aug = pd.DataFrame(np.empty((aug_factor*X.shape[0], 1)), columns=['note'])<jupyter_output><empty_output><jupyter_text>4.1 Add time shifts Demonstrate time shift code<jupyter_code>arr1 = np.array(np.arange(1,6))
print(arr1)
print(mus.add_timeshifts(arr1, samp_shift_max=4, debug=True))<jupyter_output>[1 2 3 4 5]
Shape of input array = (5,)
Input samp_shift_max = 4
Random prepad_len = 2
[0. 0. 1. 2. 3.]<jupyter_text>Apply time shifts to augmented training data<jupyter_code># Add random time shifts noise to each element
T_SHIFT_MAX = 0.1 # seconds
DEBUG=False

#X_aug = X_aug.apply(lambda x: mus.add_timeshifts(x, 
#                                         samp_shift_max=np.int(T_SHIFT_MAX*fs), 
#                                         debug=DEBUG), axis=1) 

for idx in range(X.shape[0]):
            
    # Initialize truth values in synthesized set
    y_aug.loc[aug_factor*idx:aug_factor*(idx+1)]=SCALE[idx]
    
    for sample in range(aug_factor):
        X_aug[sample+aug_factor*idx,:] = mus.add_timeshifts(X[idx,:], samp_shift_max=np.int(T_SHIFT_MAX*fs), debug=DEBUG)<jupyter_output><empty_output><jupyter_text>4.2 Add noise Add noise to original recordings<jupyter_code>NOISE_AMPL = 100

# Add white noise to each element
X_aug = pd.DataFrame(X_aug).apply(lambda x: mus.add_noise(x, 
                                    ampl=NOISE_AMPL),
                                    axis=1).values<jupyter_output><empty_output><jupyter_text>Sanity check the dimensions of the augmented training set<jupyter_code>print('X_aug has shape {}, with aug_factor = {}'.format(X_aug.shape, aug_factor))
print('Original X has shape {}'.format(X.shape))
print('y_aug has shape {}'.format(y_aug.shape))<jupyter_output>X_aug has shape (120, 120000), with aug_factor = 20
Original X has shape (6, 120000)
y_aug has shape (120, 1)<jupyter_text>Plot example with added noise and shifts<jupyter_code>idx_test = 1
signal_test1 = X_aug[idx_test*aug_factor,:]
signal_test2 = X_aug[idx_test*aug_factor+5,:]
note_test = SCALE[idx_test]

# Plot waveform over shorter time period to see sine
plt.subplot(212)
plt.xlim(0.5 ,0.55)
plt.xlabel('Time (s)')
plt.title('Examples with noise - Hummed {}: {} Hz'.format(note_test, mus.note_to_freq[note_test]))
plt.plot(t_array, signal_test1)
plt.plot(t_array, signal_test2)

plt.show()<jupyter_output><empty_output><jupyter_text>5. Feature Extraction<jupyter_code>X_feat = feat_extract(X_aug, fs, mus.note_to_freq, SCALE, debug=True)
X_feat.head()<jupyter_output><empty_output><jupyter_text>Explore extracted features<jupyter_code>NOTE_TEST = 'C'
plt.stem(X_feat[y_aug['note'].str.contains(NOTE_TEST)].mean())
#plt.bar(SCALE, X_feat[y_aug['note'].str.contains(NOTE_TEST)].mean())
plt.title('Relative power across {} note training samples'.format(NOTE_TEST))
plt.xlabel('MFCC filter #')
plt.show()
NOTE1 = 'E4'
NOTE2 = 'G4'
#plt.scatter(X_feat[y_aug['note'].str.contains(NOTE1)][NOTE1], 
#            X_feat[y_aug['note'].str.contains(NOTE1)][NOTE2])
#plt.scatter(X_feat[y_aug['note'].str.contains(NOTE2)][NOTE1], 
#            X_feat[y_aug['note'].str.contains(NOTE2)][NOTE2])

feat1 = 'mfcc_logfbank_1'
feat2 = 'mfcc_logfbank_2'

plt.scatter(X_feat[y_aug['note'].str.contains(NOTE1)][feat1], 
            X_feat[y_aug['note'].str.contains(NOTE1)][feat2])
plt.scatter(X_feat[y_aug['note'].str.contains(NOTE2)][feat1], 
            X_feat[y_aug['note'].str.contains(NOTE2)][feat2])

plt.ylabel('Relative power at {}'.format(NOTE2))
plt.xlabel('Relative power at {}'.format(NOTE1))
plt.legend([NOTE1, NOTE2])
plt.title('Feature separation between notes {} vs {}'.format(NOTE1, NOTE2))
plt.show()<jupyter_output><empty_output><jupyter_text>6. Save Features  Using SQL<jupyter_code>FEAT_DB_NAME = os.path.join(DATA_FOLDER, 'features.db')
TABLE_NAME = 'X_all'
conn = sqlite3.connect(FEAT_DB_NAME)
c = conn.cursor()

# Create table
c.execute('''CREATE TABLE IF NOT EXISTS {}
             ({} real, {} real, {} real, 
             {} real, {} real, {} real)'''.format(TABLE_NAME, X_feat.columns[0],
                                                    X_feat.columns[1],
                                                 X_feat.columns[2],
                                                 X_feat.columns[3],
                                                 X_feat.columns[4],
                                                 X_feat.columns[5]))

# Commit the changes to db
conn.commit()

# Insert multiple records in a single query
c.executemany('INSERT INTO {} VALUES(?,?,?,?,?,?);'.format(TABLE_NAME), X_feat.values);
conn.commit()

# Close the connection
conn.close()<jupyter_output><empty_output><jupyter_text>Using CSV Split data for training and testing and save off features<jupyter_code>X_train, X_test, y_train, y_test = train_test_split(X_feat, y_aug, random_state=1)

if not(os.path.exists(DATA_FOLDER)):
    os.mkdir(DATA_FOLDER)

X_train.to_csv(os.path.join(DATA_FOLDER, 'X_train.csv'), index=False)
X_test.to_csv(os.path.join(DATA_FOLDER, 'X_test.csv'), index=False)

y_train.to_csv(os.path.join(DATA_FOLDER, 'y_train.csv'), index=False)
y_test.to_csv(os.path.join(DATA_FOLDER, 'y_test.csv'), index=False)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.13221541438245726, 'fraction_numerical': 0.020638503708481134, 'mean_word_length': 4.687041564792176, 'pattern_counts': {'":': 0, '<': 63, '<?xml version=': 0, '>': 63, 'https://': 1, 'lorem ipsum': 0, 'www.': 1, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 16, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '16', 'dir': 'jupyter-structured-clean-dedup', 'id': '508719', 'license': 'MIT', 'n_tokens_mistral': 3524, 'n_tokens_neox': 3311, 'n_words': 715, 'path': '0_notebooks/1_Data_Prep.ipynb', 'repo_name': 'bwalsh62/note-recognition', 'size': '9309'}