<commit_before># Copyright 2016 <NAME>. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================

import tensorflow as tf
import numpy as np
import time
import os

import vocabulary
from speech_input import InputBatchLoader
from speech_model import Wav2LetterModel
from preprocess import SpeechCorpusReader

tf.app.flags.DEFINE_float("learning_rate", 1e-3, "Learning rate.")
tf.app.flags.DEFINE_float("learning_rate_decay_factor", 0.9,
                          "Learning rate decays by this much (multiplication).")
tf.app.flags.DEFINE_float("momentum", 0.9, "Optimizer momentum")
tf.app.flags.DEFINE_bool('disable_learning_rate_decay', False, 'Do not adapt the learning rate')
tf.app.flags.DEFINE_float("max_gradient_norm", 5.0, "Clip gradients to this norm.")
tf.app.flags.DEFINE_bool('relu', False, 'Use ReLU activation instead of tanh')
tf.app.flags.DEFINE_bool('power', False, 'Use a power spectrogram instead of mfccs as input')
tf.app.flags.DEFINE_integer("batch_size", 64,
                            "Batch size to use during training.")
tf.app.flags.DEFINE_string("data_dir", "data/", "Data directory")
tf.app.flags.DEFINE_string("train_dir", "train/", "Training directory")
tf.app.flags.DEFINE_string("log_dir", "log/", "Logging directory for summaries")
tf.app.flags.DEFINE_string("run_name", "", "Give this training a name to appear in tensorboard")
tf.app.flags.DEFINE_integer("limit_training_set", 0,
                            "Train on a smaller training set, limited to the specified size")
tf.app.flags.DEFINE_integer("steps_per_checkpoint", 200,
                            "How many training steps to do per checkpoint.")

FLAGS = tf.app.flags.FLAGS


def create_model(session, input_size, speech_input):
  """Create speechT model and initialize or load parameters in session."""
  model = Wav2LetterModel(speech_input,
                          input_size,
                          vocabulary.SIZE + 1,
                          FLAGS.learning_rate,
                          FLAGS.learning_rate_decay_factor,
                          FLAGS.max_gradient_norm,
                          FLAGS.log_dir,
                          FLAGS.relu,
                          FLAGS.run_name,
                          FLAGS.momentum,
                          run_type='train')
  ckpt = tf.train.get_checkpoint_state(FLAGS.train_dir)
  if ckpt and tf.gfile.Exists(ckpt.model_checkpoint_path):
    print("Reading model parameters from %s" % ckpt.model_checkpoint_path)
    model.saver.restore(session, ckpt.model_checkpoint_path)
    model.init_session(session, init_variables=False)
  else:
    print("Created model with fresh parameters.")
    model.init_session(session, init_variables=True)
  return model


def train():
  # Create training sub-directory if not specified otherwise
  if FLAGS.run_name and FLAGS.train_dir == 'train/':
    FLAGS.train_dir += FLAGS.run_name + '/'

  # Determine feature type
  feature_type = 'power' if FLAGS.power else 'mfcc'

  # Create training directory if it does not exist<commit_msg>Added flag to reset learning rate to specified value<commit_after># Copyright 2016 <NAME>. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================

import tensorflow as tf
import numpy as np
import time
import os

import vocabulary
from speech_input import InputBatchLoader, BaseInputLoader
from speech_model import Wav2LetterModel, SpeechModel
from preprocess import SpeechCorpusReader

tf.app.flags.DEFINE_float("learning_rate", 1e-3, "Learning rate.")
tf.app.flags.DEFINE_bool("reset_learning_rate", False, "Reset the learning rate to the default or provided value")
tf.app.flags.DEFINE_float("learning_rate_decay_factor", 0.9,
                          "Learning rate decays by this much (multiplication).")
tf.app.flags.DEFINE_float("momentum", 0.9, "Optimizer momentum")
tf.app.flags.DEFINE_bool('disable_learning_rate_decay', False, 'Do not adapt the learning rate')
tf.app.flags.DEFINE_float("max_gradient_norm", 5.0, "Clip gradients to this norm.")
tf.app.flags.DEFINE_bool('relu', False, 'Use ReLU activation instead of tanh')
tf.app.flags.DEFINE_bool('power', False, 'Use a power spectrogram instead of mfccs as input')
tf.app.flags.DEFINE_integer("batch_size", 64,
                            "Batch size to use during training.")
tf.app.flags.DEFINE_string("data_dir", "data/", "Data directory")
tf.app.flags.DEFINE_string("train_dir", "train/", "Training directory")
tf.app.flags.DEFINE_string("log_dir", "log/", "Logging directory for summaries")
tf.app.flags.DEFINE_string("run_name", "", "Give this training a name to appear in tensorboard")
tf.app.flags.DEFINE_integer("limit_training_set", 0,
                            "Train on a smaller training set, limited to the specified size")
tf.app.flags.DEFINE_integer("steps_per_checkpoint", 200,
                            "How many training steps to do per checkpoint.")

FLAGS = tf.app.flags.FLAGS


def create_model(session: tf.Session, input_size: int, speech_input: BaseInputLoader) -> SpeechModel:
  """Create speechT model and initialize or load parameters in session."""
  model = Wav2LetterModel(speech_input,
                          input_size,
                          vocabulary.SIZE + 1,
                          FLAGS.learning_rate,
                          FLAGS.learning_rate_decay_factor,
                          FLAGS.max_gradient_norm,
                          FLAGS.log_dir,
                          FLAGS.relu,
                          FLAGS.run_name,
                          FLAGS.momentum,
                          run_type='train')
  ckpt = tf.train.get_checkpoint_state(FLAGS.train_dir)
  if ckpt and tf.gfile.Exists(ckpt.model_checkpoint_path):
    print("Reading model parameters from %s" % ckpt.model_checkpoint_path)
    model.saver.restore(session, ckpt.model_checkpoint_path)
    model.init_session(session, init_variables=False)
    if FLAGS.reset_learning_rate:
      session.run(model.learning_rate.assign(FLAGS.learning_rate))
  else:
    print("Created model with fresh parameters.")
    model.init_session(session, init_variables=True)
  return model


def train():
  # Create training sub-directory if not specified otherwise
  if FLAGS.run_name and FLAGS.train_dir == 'train/':
    FLAGS.train_dir += FLAGS.run_name + '/'

  # Determine feature type
  feature_type = 'power' if FLAGS.power else 'mfcc'

  # Create training directory if it does not exist
{'fraction_non_alphanumeric': 0.11203264876250658, 'fraction_numerical': 0.006582411795681938, 'mean_word_length': 3.2751828925154753, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 6, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1229870', 'n_tokens_mistral': 2233, 'n_tokens_neox': 2066, 'n_words': 638}