<jupyter_start><jupyter_text>Load Data hiThe reason of why SR is 44100 is that the sample rate of above files is 44.1kbpsa wav file sample has 884736. if sample is divided by sample rate, the value is timethe time is fixed by 20.06<jupyter_code>def load(files, sr=SR):
    [raw, sr] = librosa.load(files[0], sr=sr)
    for f in files[1:]:
        [array, sr] = librosa.load(f, sr=sr)
        raw = np.hstack((raw, array))
    print(raw.shape)
    return raw
none_raw = load(none_files)
uav_raw = load(uav_files)
loaded_raw = load(loaded_files)<jupyter_output>(86495188,)
(54636926,)
(62515322,)<jupyter_text>Feature extraction  steps 1. Resampling  2. *VAD*( Voice Activity Detection) 3. Maybe padding with 0 to make signals be equal length 4. Log spectrogram (or *MFCC*, or *PLP*) 5. Features normalization with *mean* and *std* 6. Stacking of a given number of frames to get temporal information  1. Resamplingif you see the graph, there are few at high frequency. this is mean that data is big but it's no useless. so To small the data, do Resampling. In general, use 0~8000Hz   2. VADSometimes, Files have silence. It is not necessary. So, We need to find sound of Drone except silence.But, Not yet implemented  3. padding with 0 to make signals be equal lengthIf we have a lot of sound files, we need to pad some datas. But These files's time is longger than 1 second. So It dosn't need to pad  4. Log spectrogram (or MFCC, or PLP)The upper picture is resampled data. The lower picture is original data.In MFCC Feature, There is no big difference.<jupyter_code>from scipy.stats import skew
#returns mfcc features with mean and standard deviation along time

def mfcc4(raw, label, chunk_size=8192, window_size=4096, sr=22050, n_mfcc=16, n_frame=16):
    mfcc = np.empty((0, n_mfcc, n_frame))
    y = []
    print(raw.shape)
    for i in range(0, len(raw), chunk_size//2):
        mfcc_slice = librosa.feature.mfcc(raw[i:i+chunk_size], sr=sr, n_mfcc=n_mfcc) #n_mfcc,17
        if mfcc_slice.shape[1] < 17:
            print("small end:", mfcc_slice.shape)
            continue
        mfcc_slice = mfcc_slice[:,:-1]
        mfcc_slice = mfcc_slice.reshape((1, mfcc_slice.shape[0], mfcc_slice.shape[1]))
        mfcc = np.vstack((mfcc, mfcc_slice))
        y.append(label)
    y = np.array(y)
    return mfcc, y

def mfcc(raw, chunk_size=8192, sr=SR, n_mfcc=N_MFCC):
    mfcc = np.empty((N_MFCC, 0))
    for i in range(0, len(raw), chunk_size):
        mfcc_slice = librosa.feature.mfcc(raw[i:i+chunk_size], sr=sr, n_mfcc=n_mfcc)
        mfcc = np.hstack((mfcc, mfcc_slice))
    print(mfcc.shape)
    return mfcc

def mfcc2(raw, chunk_size=8192, sr=SR, n_mfcc=N_MFCC):
    t1 = time.time()
    raw_mel = librosa.feature.melspectrogram(y=raw, sr=sr)
    t2 = time.time()
    print('1:', t2-t1)
    s = librosa.power_to_db(raw_mel)
    t3 = time.time()
    print('2:', t3-t2)
    mfcc = librosa.feature.mfcc(S=s, sr=sr, n_mfcc=n_mfcc)
    t4 = time.time()
    print('3:', t4-t3)
    print(mfcc.shape)
    return mfcc

#mfcc_loaded, y_loaded = mfcc4(loaded_raw, 2)
#mfcc_uav, y_uav = mfcc4(uav_raw, 1)
#mfcc_none, y_none = mfcc4(none_raw, 0)

mfcc_none = mfcc2(none_raw)
mfcc_uav = mfcc2(uav_raw)
mfcc_loaded = mfcc2(loaded_raw)

y_none =np.zeros(mfcc_none.shape[1], dtype=int)
y_uav = np.ones(mfcc_uav.shape[1], dtype=int)
y_loaded = np.ones(mfcc_loaded.shape[1], dtype=int)*2

print(len(mfcc_uav),len(mfcc_loaded),len(mfcc_none))
print(y_none.shape, y_none[0])
print(y_uav.shape, y_uav[0])
print(y_loaded.shape, y_loaded[0])<jupyter_output>1: 9.426102876663208
2: 0.7309417724609375
3: 0.2666757106781006
(30, 168936)
1: 5.780373573303223
2: 0.46323442459106445
3: 0.17847299575805664
(30, 106713)
1: 6.492225646972656
2: 0.4812812805175781
3: 0.204542875289917
(30, 122101)
30 30 30
(168936,) 0
(106713,) 1
(122101,) 2<jupyter_text>5. Features normalization with *mean* and *std*  6. Stacking of a given number of frames to get temporal information<jupyter_code>X_mfcc = np.hstack((mfcc_none, mfcc_uav))
X_mfcc = np.hstack((X_mfcc, mfcc_loaded))
X_mfcc = X_mfcc.T

y = np.hstack((y_none,  y_uav))
y = np.hstack((y,y_loaded))

#X = np.concatenate((mfcc_loaded, mfcc_uav, mfcc_none), axis=0)
#y = np.hstack((y_loaded, y_uav, y_none))
print(X_mfcc.shape, y.shape)


n_labels = y.shape[0]
y_encoded = np.zeros((n_labels, n_unique_labels))
y_encoded[np.arange(n_labels), y] = 1
print(y_encoded.shape)
#print(y_encoded[0], y_encoded[40000],y_encoded[100000])
'''
mfcc_uav_list = mfcc_uav.tolist()
mfcc_uav_list = mfcc_uav_list
fig = plt.figure(figsize=(15,9))
ax = fig.add_subplot(1,1,1)
ax.plot(np.linspace(0,len(mfcc_uav_list), len(mfcc_uav_list)),mfcc_uav_list)
'''
dataX = X_mfcc
dataY = y_encoded
#print(y_encoded)
print(dataX.shape, dataY.shape)
def makeHot(dataX, dataY, sequence_length):
    X_hot_list= []
    Y_hot_tmp = dataY[sequence_length-1:]

    for i in range(0, dataX.shape[0] - sequence_length+1):
        _x = dataX[i:i + sequence_length]
        #if i<10:
            #print(_x, "->", Y_hot_tmp[i])
        X_hot_list.append(_x)

    X_hot = np.array(X_hot_list[:])
    Y_hot= Y_hot_tmp.reshape((len(Y_hot_tmp),n_unique_labels))
    return X_hot[:], Y_hot[:]

#X_hot = np.array(dataX[:])
#Y_hot = np.array(dataY[:])
X_hot, Y_hot = makeHot( dataX, dataY, sequence_length)
print(X_hot.shape, Y_hot.shape)

from sklearn import model_selection
X_train, X_test, y_train, y_test = model_selection.train_test_split(X_hot, Y_hot, test_size=0.2, random_state=42)
X_train, X_val, y_train, y_val = model_selection.train_test_split(X_train, y_train, test_size=0.2, random_state=42)
print(X_train.shape, X_test.shape,X_val.shape)
print(y_train.shape, y_test.shape,y_val.shape)
np.save('../data/Xy/X_train2', X_train)
np.save('../data/Xy/X_test2', X_test)
np.save('../data/Xy/X_val2', X_val)
np.save('../data/Xy/y_train2', y_train)
np.save('../data/Xy/y_test2', y_test)
np.save('../data/Xy/y_val2', y_val)
X_train = np.load('../data/Xy/X_train2.npy')
X_test = np.load('../data/Xy/X_test2.npy')
X_val = np.load('../data/Xy/X_val2.npy')
y_train = np.load('../data/Xy/y_train2.npy')
y_test = np.load('../data/Xy/y_test2.npy')
y_val = np.load('../data/Xy/y_val2.npy')
class Data:
    def __init__(self,X,Y,BatchSize):
        self.X = X
        self.Y = Y
        self.len = len(Y)
        self.bs = BatchSize
        
        self.bs_i = 0
    def getBatchData(self):
        s = self.bs_i
        e = self.bs_i + self.bs
        if e> self.len:
            e -= self.len
            result =  np.vstack((self.X[s:],self.X[:e])), np.vstack((self.Y[s:],self.Y[:e]))
        else:
            result =  self.X[s:e], self.Y[s:e]
            
        self.bs_i = e
        return result
        
dataX = [1,2,3,4,5,6,7,8]
dataY = [11,12,13,14,15,16,17,18]
D = Data(dataX, dataY,3)
x, y = D.getBatchData()
print(x,y)
x, y = D.getBatchData()
print(x,y)
batch_size = 2048
traindata = Data(X_train,y_train,batch_size)
testdata = Data(X_test,y_test,batch_size)
valdata = Data(X_val,y_val,batch_size)<jupyter_output><empty_output><jupyter_text>Tensorflow RNN  Train<jupyter_code>batch_size = 2048
num_classes = N_MFCC            #분류할 사전의 크기 

learning_rate = 0.01
sequence_length = sequence_length #9

output_dim = n_unique_labels
layers = 3

model_path = '../models/RNN/my_RNN_model_test'

print(batch_size, num_classes)
print(learning_rate, sequence_length)
print(output_dim, layers)
X = tf.placeholder(tf.float32, [None, sequence_length,num_classes], name="X")
Y = tf.placeholder(tf.float32, [None, output_dim], name="Y")

keep_prob = tf.placeholder(tf.float32)

cell = tf.contrib.rnn.BasicLSTMCell(num_units=num_classes, state_is_tuple=True)
#cell = tf.contrib.rnn.DropoutWrapper(cell, output_keep_prob=keep_prob)
cell = tf.contrib.rnn.MultiRNNCell([cell]*2, state_is_tuple= True)

BatchSize = tf.placeholder(tf.int32, [], name='BatchSize')
initial_state = cell.zero_state(BatchSize, tf.float32)
outputs, _states = tf.nn.dynamic_rnn(cell, X,initial_state=initial_state,dtype=tf.float32)

dense1 = tf.layers.dense(inputs=outputs[:,-1], units=sequence_length*output_dim, activation=tf.nn.relu)

dense2 = tf.layers.dense(inputs=dense1, units=sequence_length*output_dim, activation=tf.nn.relu)
dropout2 = tf.nn.dropout(dense1, keep_prob=keep_prob)

dense3 = tf.layers.dense(inputs=dense2, units=output_dim, activation=tf.nn.relu)

Y_pred= tf.layers.dense(inputs=dense3, units=output_dim)
cost = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits_v2(logits=Y_pred, labels=Y))
lr = tf.placeholder(tf.float32,shape=(), name='learning_rate')
train = tf.train.AdamOptimizer(lr).minimize(cost)

traindata.X[0]
print(traindata.Y[:10])
x, y = traindata.getBatchData()
print(y)
from sklearn.metrics import accuracy_score
init = tf.global_variables_initializer()
cost_history = np.empty(shape=[1],dtype=float)
step_loss = 999999.0
saver = tf.train.Saver()
training_epochs = 100
# Training step

sess = tf.InteractiveSession()
sess.run(init)
#learning_rate_ = [i*0.001 for i in range(20,10,-1)]
#for learning_rate in [0.02, 0.01]:
#    feed = {lr:learning_rate, BatchSize: batch_size}
N = int(len(valdata.Y) / batch_size) + 1
for i in range(training_epochs):
    feed = {lr:learning_rate, BatchSize: batch_size, keep_prob : 1.0}
    for n in range(N):
        x,y = traindata.getBatchData()
        feed[X], feed[Y] = x, y
        step_loss_prev = step_loss
        _, step_loss = sess.run([train, cost], feed_dict=feed)
        cost_history = np.append(cost_history,step_loss)
        
    y_pred = sess.run(tf.argmax(Y_pred,1),feed_dict={
        X: valdata.X, BatchSize: len(valdata.Y), keep_prob:1.0})
    y_true =  sess.run(tf.argmax(valdata.Y,1))
    accuracy_val = accuracy_score(y_pred, y_true)
    print("[step: {}] loss: {}".format(i, step_loss), "\tvalidation: {:.3f}%".format(accuracy_val * 100))    
    
print('')
saver.save(sess, model_path)
sess.close()
sess = tf.InteractiveSession()
saver.restore(sess, model_path)
y_pred = sess.run(tf.argmax(Y_pred,1),feed_dict=
                  {X: testdata.X, BatchSize: len(testdata.Y), keep_prob : 1.0})
y_true = sess.run(tf.argmax(testdata.Y,1))
print(y_pred.shape, y_true.shape)
sess.close()
from sklearn.metrics import precision_recall_fscore_support
from sklearn.metrics import accuracy_score
from sklearn.metrics import classification_report
from sklearn.metrics import confusion_matrix

fig = plt.figure(figsize=(10,8))
plt.plot(cost_history)
plt.ylabel("Cost")
plt.xlabel("Iterations") 
plt.axis([0,len(cost_history),0,np.max(cost_history)])
plt.show()


p,r,f,s = precision_recall_fscore_support(y_true, y_pred, average='micro')
print("F-Score:", round(f,3))
print("Accuracy: ", accuracy_score(y_true, y_pred))

print(classification_report(y_true, y_pred))
print(confusion_matrix(y_true, y_pred))<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.14477958236658933, 'fraction_numerical': 0.04705336426914153, 'mean_word_length': 4.537512846865365, 'pattern_counts': {'":': 0, '<': 17, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '361035', 'license': 'MIT', 'n_tokens_mistral': 4721, 'n_tokens_neox': 4424, 'n_words': 862, 'path': 'notebook/RNN/RNN.ipynb', 'repo_name': 'gunhoo/Drone-Tracking', 'size': '10775'}