<commit_before># Copyright 2016 <NAME>. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================

import tensorflow as tf
import abc


class SpeechModel:
  def __init__(self, input_loader, input_size, num_classes, learning_rate, learning_rate_decay_factor,
               max_gradient_norm, log_dir, use_relu, run_name, momentum, run_type):
    """
    Create a new speech model

    Args:
      input_loader: the object that provides input tensors
      input_size: the number of values per time step
      num_classes: the number of output classes (vocabulary_size + 1 for blank label)
      learning_rate: the inital learning rate
      learning_rate_decay_factor: the factor to multiple the learning rate with when it should be decreased
      max_gradient_norm: the maximum gradient norm to apply, otherwise clipping is applied
      log_dir: the directory to log to for use of tensorboard
      use_relu: if True, use relu instead of tanh
      run_name: the name of this run
      momentum: the momentum parameter
      run_type: "train", "dev" or "test"
    """
    self.input_size = input_size
    self.convolution_count = 0

    self.activation_fnc = tf.nn.relu if use_relu else tf.nn.tanh

    # inputs is of dimension [batch_size, max_time, input_size]
    self.inputs, self.sequence_lengths, self.labels = input_loader.get_inputs()

    # Define non-trainables
    self.global_step = tf.Variable(0, trainable=False)
    self.learning_rate = tf.Variable(float(learning_rate), trainable=False, dtype=tf.float32, name='learning_rate')
    self.learning_rate_decay_op = self.learning_rate.assign(
                                   tf.mul(self.learning_rate, learning_rate_decay_factor, name='learning_rate_decay'))

    # Variable summaries
    tf.scalar_summary('learning_rate', self.learning_rate)

    self.logits = self._create_network(num_classes)

    # Generate summary image for logits [batch_size=batch_size, height=num_classes, width=max_time / 2, channels=1]
    tf.image_summary('logits', tf.expand_dims(tf.transpose(self.logits, (1, 2, 0)), 3))
    tf.histogram_summary('logits', self.logits)

    # Define loss and optimizer
    with tf.name_scope('training'):
      self.cost = tf.nn.ctc_loss(self.logits, self.labels, self.sequence_lengths // 2)
      self.avg_loss = tf.reduce_mean(self.cost, name='average_loss')
      tf.scalar_summary('loss', self.avg_loss)
      optimizer = tf.train.MomentumOptimizer(self.learning_rate, momentum, name='optimizer')
      gvs = optimizer.compute_gradients(self.avg_loss)
      gradients, trainables = zip(*gvs)
      clipped_gradients, norm = tf.clip_by_global_norm(gradients, max_gradient_norm, name='clip_gradients')
      self.update = optimizer.apply_gradients(zip(clipped_gradients, trainables),
                                              global_step=self.global_step, name='apply_gradients')

    # Decoding
    with tf.name_scope('decoding'):
      # TODO use beam search here later
      self.decoded, self.log_probabilities = tf.nn.ctc_greedy_decoder(self.logits, self.sequence_lengths // 2)

    # Initializing the variables
    self.init = tf.initialize_all_variables()

    # Create saver
    self.saver = tf.train.Saver(tf.all_variables())

    # Create summary writers
    self.merged_summaries = tf.merge_all_summaries()
    if run_name:
      run_name += '_'
    self.summary_writer = tf.train.SummaryWriter('{}/{}{}'.format(log_dir, run_name, run_type))

  def _convolution(self, value, filter_width, stride, input_channels, out_channels, apply_non_linearity=True):
    """
    Apply a convolutional layer

    Args:
      value: the input tensor to apply the convolution on
      filter_width: the width of the filter (kernel)
      stride: the striding of the filter (kernel)
      input_channels: the number if input channels
      out_channels: the number of output channels
      apply_non_linearity: whether to apply a non linearity

    Returns:
      the output after convolution, added biases and possible non linearity applied

    """

    layer_id = self.convolution_count
    self.convolution_count += 1

    with tf.name_scope('convolution_layer_{}'.format(layer_id)) as layer:
      # Filter and bias
      initial_filter = tf.truncated_normal([filter_width, input_channels, out_channels], stddev=0.01)
      filters = tf.Variable(initial_filter, name='filters')
      bias = tf.Variable(tf.constant(0.0, shape=[out_channels]), name='bias')

      # Apply convolution
      convolution_out = tf.nn.conv1d(value, filters, stride, 'SAME', use_cudnn_on_gpu=True, name='convolution')

      # Create summary
      with tf.name_scope('summaries'):
        # add depth of 1 (=grayscale) leading to shape [filter_width, input_channels, 1, out_channels]
        kernel_with_depth = tf.expand_dims(filters, 2)

        # to tf.image_summary format [batch_size=out_channels, height=filter_width, width=input_channels, channels=1]
        kernel_transposed = tf.transpose(kernel_with_depth, [3, 0, 1, 2])

        # this will display random 3 filters from all the output channels
        tf.image_summary(layer + 'filters', kernel_transposed, max_images=3)
        tf.histogram_summary(layer + 'filters', filters)

        tf.image_summary(layer + 'bias', tf.reshape(bias, [1, 1, out_channels, 1]))
        tf.histogram_summary(layer + 'bias', bias)

      # Add bias
      convolution_out = tf.nn.bias_add(convolution_out, bias)

      if apply_non_linearity:
        # Add non-linearity
        activations = self.activation_fnc(convolution_out, name='activation')
        tf.histogram_summary(layer + 'activation', activations)
        return activations, out_channels
      else:
        return convolution_out, out_channels

  def init_session(self, sess, init_variables=True):
    """
    Initialize a new session for the model.

    Args:
      sess: session to initalize
      init_variables: whether to initialize all variables
<commit_msg>Replaced deprecated methods with new variants<commit_after># Copyright 2016 <NAME>. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================

import tensorflow as tf
import abc


class SpeechModel:
  def __init__(self, input_loader, input_size, num_classes, learning_rate, learning_rate_decay_factor,
               max_gradient_norm, log_dir, use_relu, run_name, momentum, run_type):
    """
    Create a new speech model

    Args:
      input_loader: the object that provides input tensors
      input_size: the number of values per time step
      num_classes: the number of output classes (vocabulary_size + 1 for blank label)
      learning_rate: the inital learning rate
      learning_rate_decay_factor: the factor to multiple the learning rate with when it should be decreased
      max_gradient_norm: the maximum gradient norm to apply, otherwise clipping is applied
      log_dir: the directory to log to for use of tensorboard
      use_relu: if True, use relu instead of tanh
      run_name: the name of this run
      momentum: the momentum parameter
      run_type: "train", "dev" or "test"
    """
    self.input_size = input_size
    self.convolution_count = 0

    self.activation_fnc = tf.nn.relu if use_relu else tf.nn.tanh

    # inputs is of dimension [batch_size, max_time, input_size]
    self.inputs, self.sequence_lengths, self.labels = input_loader.get_inputs()

    # Define non-trainables
    self.global_step = tf.Variable(0, trainable=False)
    self.learning_rate = tf.Variable(float(learning_rate), trainable=False, dtype=tf.float32, name='learning_rate')
    self.learning_rate_decay_op = self.learning_rate.assign(learning_rate_decay_factor * self.learning_rate)

    # Variable summaries
    tf.summary.scalar('learning_rate', self.learning_rate)

    self.logits = self._create_network(num_classes)

    # Generate summary image for logits [batch_size=batch_size, height=num_classes, width=max_time / 2, channels=1]
    tf.summary.image('logits', tf.expand_dims(tf.transpose(self.logits, (1, 2, 0)), 3))
    tf.summary.histogram('logits', self.logits)

    # Define loss and optimizer
    with tf.name_scope('training'):
      self.cost = tf.nn.ctc_loss(self.logits, self.labels, self.sequence_lengths // 2)
      self.avg_loss = tf.reduce_mean(self.cost, name='average_loss')
      tf.summary.scalar('loss', self.avg_loss)
      optimizer = tf.train.MomentumOptimizer(self.learning_rate, momentum, name='optimizer')
      gvs = optimizer.compute_gradients(self.avg_loss)
      gradients, trainables = zip(*gvs)
      clipped_gradients, norm = tf.clip_by_global_norm(gradients, max_gradient_norm, name='clip_gradients')
      self.update = optimizer.apply_gradients(zip(clipped_gradients, trainables),
                                              global_step=self.global_step, name='apply_gradients')

    # Decoding
    with tf.name_scope('decoding'):
      # TODO use beam search here later
      self.decoded, self.log_probabilities = tf.nn.ctc_greedy_decoder(self.logits, self.sequence_lengths // 2)

    # Initializing the variables
    self.init = tf.global_variables_initializer()

    # Create saver
    self.saver = tf.train.Saver(tf.global_variables())

    # Create summary writers
    self.merged_summaries = tf.summary.merge_all()
    if run_name:
      run_name += '_'
    self.summary_writer = tf.summary.FileWriter('{}/{}{}'.format(log_dir, run_name, run_type))

  def _convolution(self, value, filter_width, stride, input_channels, out_channels, apply_non_linearity=True):
    """
    Apply a convolutional layer

    Args:
      value: the input tensor to apply the convolution on
      filter_width: the width of the filter (kernel)
      stride: the striding of the filter (kernel)
      input_channels: the number if input channels
      out_channels: the number of output channels
      apply_non_linearity: whether to apply a non linearity

    Returns:
      the output after convolution, added biases and possible non linearity applied

    """

    layer_id = self.convolution_count
    self.convolution_count += 1

    with tf.name_scope('convolution_layer_{}'.format(layer_id)) as layer:
      # Filter and bias
      initial_filter = tf.truncated_normal([filter_width, input_channels, out_channels], stddev=0.01)
      filters = tf.Variable(initial_filter, name='filters')
      bias = tf.Variable(tf.constant(0.0, shape=[out_channels]), name='bias')

      # Apply convolution
      convolution_out = tf.nn.conv1d(value, filters, stride, 'SAME', use_cudnn_on_gpu=True, name='convolution')

      # Create summary
      with tf.name_scope('summaries'):
        # add depth of 1 (=grayscale) leading to shape [filter_width, input_channels, 1, out_channels]
        kernel_with_depth = tf.expand_dims(filters, 2)

        # to tf.image_summary format [batch_size=out_channels, height=filter_width, width=input_channels, channels=1]
        kernel_transposed = tf.transpose(kernel_with_depth, [3, 0, 1, 2])

        # this will display random 3 filters from all the output channels
        tf.summary.image(layer + 'filters', kernel_transposed, max_outputs=3)
        tf.summary.histogram(layer + 'filters', filters)

        tf.summary.image(layer + 'bias', tf.reshape(bias, [1, 1, out_channels, 1]))
        tf.summary.histogram(layer + 'bias', bias)

      # Add bias
      convolution_out = tf.nn.bias_add(convolution_out, bias)

      if apply_non_linearity:
        # Add non-linearity
        activations = self.activation_fnc(convolution_out, name='activation')
        tf.summary.histogram(layer + 'activation', activations)
        return activations, out_channels
      else:
        return convolution_out, out_channels

  def init_session(self, sess, init_variables=True):
    """
    Initialize a new session for the model.

    Args:
      sess: session to initalize
      init_variables: whether to initialize all variables

{'fraction_non_alphanumeric': 0.10066031941031942, 'fraction_numerical': 0.006296068796068796, 'mean_word_length': 3.6701326640372893, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 5, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '714588', 'n_tokens_mistral': 3932, 'n_tokens_neox': 3765, 'n_words': 1071}