<jupyter_start><jupyter_text>LSTM-L Architecture based on https://arxiv.org/pdf/1711.07128.pdf<jupyter_code>import warnings
warnings.filterwarnings("ignore")

import sys
import os
import tensorflow as tf

# sys.path.append("../libs")
sys.path.insert(1, '../')

from libs import input_data
from libs import models
from libs import trainer
from libs import freeze
flags=tf.app.flags
flags=tf.app.flags
#Important Directories
flags.DEFINE_string('data_dir','..\\..\\_inputs\\raw','Train Data Folder')
flags.DEFINE_string('summaries_dir','..\\..\\summaries','Summaries Folder')
flags.DEFINE_string('train_dir','..\\..\\logs&checkpoint','Directory to write event logs and checkpoint')
flags.DEFINE_string('models_dir','..\\..\\models','Models Folder')
#Task Specific Parameters
flags.DEFINE_string('wanted_words','bed,bird,cat,dog,down,eight,five,four,go,happy,house,left,marvin,nine,no,off,on,one,right,seven,sheila,six,stop,three,tree,two,up,wow,yes,zero','Wanted Words')
flags.DEFINE_float('validation_percentage',10,'Validation Percentage')
flags.DEFINE_float('testing_percentage',10,'Testing Percentage')
flags.DEFINE_integer('sample_rate',16000,'Sample Rate')
flags.DEFINE_integer('clip_duration_ms',1000,'Clip Duration in ms')
flags.DEFINE_float('window_size_ms',40,'How long each spectogram timeslice is')
flags.DEFINE_float('window_stride_ms',20.0,'How far to move in time between frequency windows.')
flags.DEFINE_integer('dct_coefficient_count',40,'How many bins to use for the MFCC')
flags.DEFINE_float('time_shift_ms',100.0,'Range to randomly shift the training audio by in time.')

FLAGS=flags.FLAGS
model_architecture='lstm_l'
start_checkpoint=None
logging_interval=10
eval_step_interval=500
save_step_interval=1
silence_percentage=10.0
unknown_percentage=15.0
background_frequency=1
background_volume=0.3
learning_rate='0.0005,0.0001,0.00002,0.0001,0.00002' #Always seperated by comma, trains with each of the learning rate for the given number of iterations
train_steps='10000,10000,10000,15000,5000' #Declare  the training steps for which the learning rates will be used
batch_size=256
remaining_args = FLAGS([sys.argv[0]] + [flag for flag in sys.argv if flag.startswith("--")])
assert(remaining_args == [sys.argv[0]])
train_dir=os.path.join(FLAGS.data_dir,'train','audio')
model_settings = models.prepare_model_settings(
      len(input_data.prepare_words_list(FLAGS.wanted_words.split(','))),
      FLAGS.sample_rate, FLAGS.clip_duration_ms, FLAGS.window_size_ms,
      FLAGS.window_stride_ms, FLAGS.dct_coefficient_count)
audio_processor = input_data.AudioProcessor(
      train_dir, silence_percentage, unknown_percentage,
      FLAGS.wanted_words.split(','), FLAGS.validation_percentage,
      FLAGS.testing_percentage, model_settings,use_silence_folder=True)
def get_train_data(args):
    sess=args
    time_shift_samples = int((FLAGS.time_shift_ms * FLAGS.sample_rate) / 1000)
    train_data, train_ground_truth = audio_processor.get_data(
        batch_size, 0, model_settings, background_frequency,
        background_volume, time_shift_samples, 'training', sess)
    return train_data, train_ground_truth

def get_val_data(args):
    '''
    Input: (sess,offset)
    '''
    sess,i=args
    validation_data, validation_ground_truth = (
        audio_processor.get_data(batch_size, i, model_settings, 0.0,
                                     0.0, 0, 'validation', sess))
    return validation_data,validation_ground_truth
# def get_test_data(args):
#     '''
#     Input: (sess,offset)
#     '''
#     sess,i=args
#     test_data, test_ground_truth = audio_processor.get_data(
#         batch_size, i, model_settings, 0.0, 0.0, 0, 'testing', sess)
#     return test_data,test_ground_truth
def main(_):
    sess=tf.InteractiveSession()
    # Placeholders
    data_size = model_settings['data_size']
    label_count = model_settings['label_count']
    data_input = tf.placeholder(
      tf.float32, [None, data_size], name='data_input')
    ground_truth_input = tf.placeholder(
      tf.float32, [None, label_count], name='groundtruth_input')
    set_size = audio_processor.set_size('validation')
    label_count = model_settings['label_count']
    
    # Create Model
    
    logits, dropout_prob = models.create_model(
      data_input,
      model_settings,
      model_architecture,
      is_training=True)
    
    #Start Training
    extra_args=(dropout_prob,label_count,batch_size,set_size)
    trainer.train(sess,logits,data_input,ground_truth_input,get_train_data,
                  get_val_data,train_steps,learning_rate,eval_step_interval, logging_interval=logging_interval,
                  start_checkpoint=start_checkpoint,checkpoint_interval=save_step_interval,
                  model_name=model_architecture,train_dir=FLAGS.train_dir,
                  summaries_dir=FLAGS.summaries_dir,args=extra_args)
tf.app.run(main=main)<jupyter_output>W0119 23:08:28.659476 13444 deprecation.py:323] From ..\libs\models.py:506: LSTMCell.__init__ (from tensorflow.python.ops.rnn_cell_impl) is deprecated and will be removed in a future version.
Instructions for updating:
This class is equivalent as tf.keras.layers.LSTMCell, and will be replaced by that in Tensorflow 2.0.
W0119 23:08:28.675407 13444 deprecation.py:323] From ..\libs\models.py:511: dynamic_rnn (from tensorflow.python.ops.rnn) is deprecated and will be removed in a future version.
Instructions for updating:
Please use `keras.layers.RNN(cell)`, which is equivalent to this API
W0119 23:08:28.737274 13444 deprecation.py:506] From C:\ProgramData\Anaconda3\lib\site-packages\tensorflow\python\ops\init_ops.py:1251: calling VarianceScaling.__init__ (from tensorflow.python.ops.init_ops) with dtype is deprecated and will be removed in a future version.
Instructions for updating:
Call initializer instance with the dtype argument instead of passing it to the constructor
W0119 23:08:28.7[...]<jupyter_text>Freeze<jupyter_code># save_checkpoint='..\\..\\logs&checkpoint\\lstm_l\\ckpt-50000'
# save_path=os.path.join(FLAGS.models_dir,model_architecture,'%s-batched.pb'%os.path.basename(save_checkpoint))
# freeze.freeze_graph(FLAGS,model_architecture,save_checkpoint,save_path,batched=True,model_size_info=model_size_info)
# save_path=os.path.join(FLAGS.models_dir,model_architecture,'%s-batched.pb'%os.path.basename(save_checkpoint))
# freeze.freeze_graph(FLAGS,model_architecture,save_checkpoint,save_path,batched=True)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.11640540121061617, 'fraction_numerical': 0.03492162036318485, 'mean_word_length': 5.582226762002043, 'pattern_counts': {'":': 0, '<': 8, '<?xml version=': 0, '>': 8, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '273086', 'license': 'MIT', 'n_tokens_mistral': 2311, 'n_tokens_neox': 2097, 'n_words': 382, 'path': 'FinalProject_SpeechRecognition/src/notebooks/07 Training - LSTM L.ipynb', 'repo_name': 'lev1khachatryan/ASDS_DL', 'size': '6443'}