<commit_before>import logging
import time
import numpy as np
import tensorflow as tf
from tensorflow.models.rnn import rnn

# Disable Tensorflow logging messages.
logging.getLogger('tensorflow').setLevel(logging.WARNING)

class CharRNN(object):
  """Character RNN model."""
  
  def __init__(self, is_training, batch_size, num_unrollings, vocab_size, 
               hidden_size, max_grad_norm, embedding_size, num_layers,
               learning_rate, model, dropout=0.0, input_dropout=0.0, use_batch=True):
    self.batch_size = batch_size
    self.num_unrollings = num_unrollings
    if not use_batch:
      self.batch_size = 1
      self.num_unrollings = 1
    self.hidden_size = hidden_size
    self.vocab_size = vocab_size
    self.max_grad_norm = max_grad_norm
    self.num_layers = num_layers
    self.embedding_size = embedding_size
    self.model = model
    self.dropout = dropout
    self.input_dropout = input_dropout
    if embedding_size <= 0:
      self.input_size = vocab_size
      # Don't do dropout on one hot representation.
      self.input_dropout = 0.0
    else:
      self.input_size = embedding_size
    self.model_size = (embedding_size * vocab_size + # embedding parameters
                       # lstm parameters
                       4 * hidden_size * (hidden_size + self.input_size + 1) +
                       # softmax parameters
                       vocab_size * (hidden_size + 1) +
                       # multilayer lstm parameters for extra layers.
                       (num_layers - 1) * 4 * hidden_size *
                       (hidden_size + hidden_size + 1))
    # self.decay_rate = decay_rate

    # Placeholder to feed in input and targets/labels data.
    self.input_data = tf.placeholder(tf.int64,
                                     [self.batch_size, self.num_unrollings],
                                     name='inputs')
    self.targets = tf.placeholder(tf.int64,
                                  [self.batch_size, self.num_unrollings],
                                  name='targets')

    if self.model == 'rnn':
      cell_fn = tf.nn.rnn_cell.BasicRNNCell
    elif self.model == 'lstm':
      cell_fn = tf.nn.rnn_cell.BasicLSTMCell
    elif self.model == 'gru':
      cell_fn = tf.nn.rnn_cell.GRUCell

    params = {'input_size': self.input_size}
    if self.model == 'lstm':
      # add bias to forget gate in lstm.
      params['forget_bias'] = 0.0
    # Create multilayer cell.
    cell = cell_fn(self.hidden_size,
                   **params)

    cells = [cell]
    params['input_size'] = self.hidden_size
    # more explicit way to create cells for MultiRNNCell than
    # [higher_layer_cell] * (self.num_layers - 1)
    for i in range(self.num_layers-1):
      higher_layer_cell = cell_fn(self.hidden_size,
                                  **params)
      cells.append(higher_layer_cell)

    if is_training and self.dropout > 0:
      cells = [tf.nn.rnn_cell.DropoutWrapper(cell,
                                             output_keep_prob=1.0-self.dropout)
               for cell in cells]
        
    multi_cell = tf.nn.rnn_cell.MultiRNNCell(cells)

    with tf.name_scope('initial_state'):
      # zero_state is used to compute the intial state for cell.
      self.zero_state = multi_cell.zero_state(self.batch_size, tf.float32)
      # Placeholder to feed in initial state.
      self.initial_state = tf.placeholder(tf.float32,
                                          [self.batch_size, multi_cell.state_size],
                                          'initial_state')

    # Embeddings layers.
    with tf.name_scope('embedding_layer'):
      if embedding_size > 0:
        self.embedding = tf.get_variable("embedding",
                                         [self.vocab_size, self.embedding_size])
      else:
        self.embedding = tf.constant(np.eye(self.vocab_size), dtype=tf.float32)

      inputs = tf.nn.embedding_lookup(self.embedding, self.input_data)
      if is_training and self.input_dropout > 0:
        inputs = tf.nn.dropout(inputs, 1 - self.input_dropout)

    with tf.name_scope('slice_inputs'):
      # Slice inputs into a list of shape [batch_size, 1] data colums.
      sliced_inputs = [tf.squeeze(input_, [1])
                       for input_ in tf.split(1, self.num_unrollings, inputs)]
      
    # Copy cell to do unrolling and collect outputs.
    outputs, final_state = rnn.rnn(multi_cell, sliced_inputs, initial_state=self.initial_state)
    self.final_state = final_state

    with tf.name_scope('flatten_ouputs'):
      # Flatten the outputs into one dimension.
      flat_outputs = tf.reshape(tf.concat(1, outputs), [-1, hidden_size])

    with tf.name_scope('flatten_targets'):
      # Flatten the targets too.
      flat_targets = tf.reshape(tf.concat(1, self.targets), [-1])
    
    # Create softmax parameters, weights and bias.
    with tf.variable_scope('softmax') as sm_vs:<commit_msg>Changed to use tf.nn.rnn instead of tensorflow.models.rnn.rnn. And removed input_size argument that is deprecated.<commit_after>import logging
import time
import numpy as np
import tensorflow as tf

# Disable Tensorflow logging messages.
logging.getLogger('tensorflow').setLevel(logging.WARNING)

class CharRNN(object):
  """Character RNN model."""
  
  def __init__(self, is_training, batch_size, num_unrollings, vocab_size, 
               hidden_size, max_grad_norm, embedding_size, num_layers,
               learning_rate, model, dropout=0.0, input_dropout=0.0, use_batch=True):
    self.batch_size = batch_size
    self.num_unrollings = num_unrollings
    if not use_batch:
      self.batch_size = 1
      self.num_unrollings = 1
    self.hidden_size = hidden_size
    self.vocab_size = vocab_size
    self.max_grad_norm = max_grad_norm
    self.num_layers = num_layers
    self.embedding_size = embedding_size
    self.model = model
    self.dropout = dropout
    self.input_dropout = input_dropout
    if embedding_size <= 0:
      self.input_size = vocab_size
      # Don't do dropout on one hot representation.
      self.input_dropout = 0.0
    else:
      self.input_size = embedding_size
    self.model_size = (embedding_size * vocab_size + # embedding parameters
                       # lstm parameters
                       4 * hidden_size * (hidden_size + self.input_size + 1) +
                       # softmax parameters
                       vocab_size * (hidden_size + 1) +
                       # multilayer lstm parameters for extra layers.
                       (num_layers - 1) * 4 * hidden_size *
                       (hidden_size + hidden_size + 1))
    # self.decay_rate = decay_rate

    # Placeholder to feed in input and targets/labels data.
    self.input_data = tf.placeholder(tf.int64,
                                     [self.batch_size, self.num_unrollings],
                                     name='inputs')
    self.targets = tf.placeholder(tf.int64,
                                  [self.batch_size, self.num_unrollings],
                                  name='targets')

    if self.model == 'rnn':
      cell_fn = tf.nn.rnn_cell.BasicRNNCell
    elif self.model == 'lstm':
      cell_fn = tf.nn.rnn_cell.BasicLSTMCell
    elif self.model == 'gru':
      cell_fn = tf.nn.rnn_cell.GRUCell

    # params = {'input_size': self.input_size}
    params = {'state_is_tuple': False}
    if self.model == 'lstm':
      # add bias to forget gate in lstm.
      params['forget_bias'] = 0.0
    # Create multilayer cell.
    cell = cell_fn(self.hidden_size,
                   **params)

    cells = [cell]
    # params['input_size'] = self.hidden_size
    # more explicit way to create cells for MultiRNNCell than
    # [higher_layer_cell] * (self.num_layers - 1)
    for i in range(self.num_layers-1):
      higher_layer_cell = cell_fn(self.hidden_size,
                                  **params)
      cells.append(higher_layer_cell)

    if is_training and self.dropout > 0:
      cells = [tf.nn.rnn_cell.DropoutWrapper(cell,
                                             output_keep_prob=1.0-self.dropout)
               for cell in cells]
        
    multi_cell = tf.nn.rnn_cell.MultiRNNCell(cells)

    with tf.name_scope('initial_state'):
      # zero_state is used to compute the intial state for cell.
      self.zero_state = multi_cell.zero_state(self.batch_size, tf.float32)
      # Placeholder to feed in initial state.
      self.initial_state = tf.placeholder(tf.float32,
                                          [self.batch_size, multi_cell.state_size],
                                          'initial_state')

    # Embeddings layers.
    with tf.name_scope('embedding_layer'):
      if embedding_size > 0:
        self.embedding = tf.get_variable("embedding",
                                         [self.vocab_size, self.embedding_size])
      else:
        self.embedding = tf.constant(np.eye(self.vocab_size), dtype=tf.float32)

      inputs = tf.nn.embedding_lookup(self.embedding, self.input_data)
      if is_training and self.input_dropout > 0:
        inputs = tf.nn.dropout(inputs, 1 - self.input_dropout)

    with tf.name_scope('slice_inputs'):
      # Slice inputs into a list of shape [batch_size, 1] data colums.
      sliced_inputs = [tf.squeeze(input_, [1])
                       for input_ in tf.split(1, self.num_unrollings, inputs)]
      
    # Copy cell to do unrolling and collect outputs.
    outputs, final_state = tf.nn.rnn(multi_cell, sliced_inputs,
                                     initial_state=self.initial_state)
    self.final_state = final_state

    with tf.name_scope('flatten_ouputs'):
      # Flatten the outputs into one dimension.
      flat_outputs = tf.reshape(tf.concat(1, outputs), [-1, hidden_size])

    with tf.name_scope('flatten_targets'):
      # Flatten the targets too.
      flat_targets = tf.reshape(tf.concat(1, self.targets), [-1])
    
    # Create softmax parameters, weights and bias.
    with tf.variable_scope('softmax') as sm_vs:
{'fraction_non_alphanumeric': 0.0918295739348371, 'fraction_numerical': 0.008421052631578947, 'mean_word_length': 2.1519747235387046, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 27, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1307543', 'n_tokens_mistral': 3116, 'n_tokens_neox': 2993, 'n_words': 675}