<commit_before># -*- coding: utf-8 -*-
import tensorflow as tf
import numpy as np
from custom_lstm import CustomBasicLSTMCell

# Actor-Critic LSTM Network (Policy network and Value network)

class GameACLSTMNetwork(object):
  def __init__(self,
               action_size,
               thread_index, # -1 for global
               device="/cpu:0" ):

    self._device = device
    
    with tf.device(self._device):
      self._action_size = action_size
      
      self.W_conv1 = self._conv_weight_variable([8, 8, 4, 16])  # stride=4
      self.b_conv1 = self._conv_bias_variable([16], 8, 8, 4)

      self.W_conv2 = self._conv_weight_variable([4, 4, 16, 32]) # stride=2
      self.b_conv2 = self._conv_bias_variable([32], 4, 4, 16)

      self.W_fc1 = self._fc_weight_variable([2592, 256])
      self.b_fc1 = self._fc_bias_variable([256], 2592)

      # lstm
      self.lstm = CustomBasicLSTMCell(256)

      # weight for policy output layer
      self.W_fc2 = self._fc_weight_variable([256, action_size])
      self.b_fc2 = self._fc_bias_variable([action_size], 256)

      # weight for value output layer
      self.W_fc3 = self._fc_weight_variable([256, 1])
      self.b_fc3 = self._fc_bias_variable([1], 256)

      # state (input)
      self.s = tf.placeholder("float", [1, 84, 84, 4])
    
      h_conv1 = tf.nn.relu(self._conv2d(self.s, self.W_conv1, 4) + self.b_conv1)
      h_conv2 = tf.nn.relu(self._conv2d(h_conv1, self.W_conv2, 2) + self.b_conv2)

      h_conv2_flat = tf.reshape(h_conv2, [1, 2592])
      h_fc1 = tf.nn.relu(tf.matmul(h_conv2_flat, self.W_fc1) + self.b_fc1)

      # TODO: now num_steps for LSTM is only one
      self.initial_lstm_state = tf.zeros([1, self.lstm.state_size])
      scope = "net_" + str(thread_index)
      lstm_output, self.lstm_state = self.lstm(h_fc1, self.initial_lstm_state, scope)

      # policy (output)
      self.pi = tf.nn.softmax(tf.matmul(lstm_output, self.W_fc2) + self.b_fc2)
      # value (output)
      self.v = tf.matmul(lstm_output, self.W_fc3) + self.b_fc3

      self.reset_state()

  def prepare_loss(self, entropy_beta):
    with tf.device(self._device):
      # taken action (input for policy)
      self.a = tf.placeholder("float", [1, self._action_size])
    
      # temporary difference (R-V) (input for policy)
      self.td = tf.placeholder("float", [1])
      # policy entropy
      entropy = -tf.reduce_sum(self.pi * tf.log(self.pi))

      # policy loss (output)  (add minus, because this is for gradient ascent)
      policy_loss = -( tf.reduce_sum( tf.mul( tf.log(self.pi), self.a ) ) * self.td +
                             entropy * entropy_beta )

      # R (input for value)
      self.r = tf.placeholder("float", [1])
      # value loss (output)
      # (Learning rate for Critic is half of Actor's, so multiply by 0.5)
      value_loss = 0.5 * tf.nn.l2_loss(self.r - self.v)

      # gradienet of policy and value are summed up
      self.total_loss = policy_loss + value_loss

  def reset_state(self):
    self.lstm_state_out = np.zeros([1, self.lstm.state_size])

  def run_policy(self, sess, s_t):
    pi_out, self.lstm_state_out = sess.run( [self.pi, self.lstm_state],
                                            feed_dict = {self.s : [s_t],
                                                         self.initial_lstm_state :
                                                         self.lstm_state_out } )
    return pi_out[0]
  
  def run_value(self, sess, s_t):
    prev_lstm_state_out = self.lstm_state_out
    v_out, _ = sess.run( [self.v, self.lstm_state],
                         feed_dict = {self.s : [s_t],
                                      self.initial_lstm_state :
                                      self.lstm_state_out } )
    # roll back lstm state
    self.lstm_state_out = prev_lstm_state_out
    return v_out[0][0] # output is scalar

  def run_policy_and_value(self, sess, s_t):
    pi_out, v_out, self.lstm_state_out = sess.run( [self.pi, self.v, self.lstm_state],
                                                   feed_dict = {self.s : [s_t],
                                                                self.initial_lstm_state :
                                                                self.lstm_state_out } )
    return (pi_out[0], v_out[0][0])

  def get_vars(self):
    return [self.W_conv1, self.b_conv1,
            self.W_conv2, self.b_conv2,
            self.W_fc1, self.b_fc1,
            self.lstm.matrix, self.lstm.bias,
            self.W_fc2, self.b_fc2,
            self.W_fc3, self.b_fc3]

  def sync_from(self, src_netowrk, name=None):
    src_vars = src_netowrk.get_vars()
    dst_vars = self.get_vars()

    sync_ops = []

    with tf.device(self._device):
      with tf.op_scope([], name, "GameACNetwork") as name:
        for(src_var, dst_var) in zip(src_vars, dst_vars):
          sync_op = tf.assign(dst_var, src_var)
          sync_ops.append(sync_op)

        return tf.group(*sync_ops, name=name)

  # weight initialization based on muupan's code
  # https://github.com/muupan/async-rl/blob/master/a3c_ale.py
  def _fc_weight_variable(self, shape):
    input_channels = shape[0]
    d = 1.0 / np.sqrt(input_channels)
    initial = tf.random_uniform(shape, minval=-d, maxval=d)
    return tf.Variable(initial)

  def _fc_bias_variable(self, shape, input_channels):
    d = 1.0 / np.sqrt(input_channels)
    initial = tf.random_uniform(shape, minval=-d, maxval=d)
    return tf.Variable(initial)  

  def _conv_weight_variable(self, shape):
    w = shape[0]
    h = shape[1]
    input_channels = shape[2]
    d = 1.0 / np.sqrt(input_channels * w * h)
    initial = tf.random_uniform(shape, minval=-d, maxval=d)
    return tf.Variable(initial)

  def _conv_bias_variable(self, shape, w, h, input_channels):
    d = 1.0 / np.sqrt(input_channels * w * h)
    initial = tf.random_uniform(shape, minval=-d, maxval=d)
    return tf.Variable(initial)

  def _conv2d(self, x, W, stride):
    return tf.nn.conv2d(x, W, strides = [1, stride, stride, 1], padding = "VALID")
<commit_msg>implementing lstm with batch<commit_after># -*- coding: utf-8 -*-
import tensorflow as tf
import numpy as np
from custom_lstm import CustomBasicLSTMCell

# Actor-Critic LSTM Network (Policy network and Value network)

class GameACLSTMNetwork(object):
  def __init__(self,
               action_size,
               thread_index, # -1 for global
               device="/cpu:0" ):

    self._device = device
    
    with tf.device(self._device):
      self._action_size = action_size
      
      self.W_conv1 = self._conv_weight_variable([8, 8, 4, 16])  # stride=4
      self.b_conv1 = self._conv_bias_variable([16], 8, 8, 4)

      self.W_conv2 = self._conv_weight_variable([4, 4, 16, 32]) # stride=2
      self.b_conv2 = self._conv_bias_variable([32], 4, 4, 16)

      self.W_fc1 = self._fc_weight_variable([2592, 256])
      self.b_fc1 = self._fc_bias_variable([256], 2592)

      # lstm
      self.lstm = CustomBasicLSTMCell(256)

      # weight for policy output layer
      self.W_fc2 = self._fc_weight_variable([256, action_size])
      self.b_fc2 = self._fc_bias_variable([action_size], 256)

      # weight for value output layer
      self.W_fc3 = self._fc_weight_variable([256, 1])
      self.b_fc3 = self._fc_bias_variable([1], 256)

      # state (input)
      self.s = tf.placeholder("float", [None, 84, 84, 4])
    
      h_conv1 = tf.nn.relu(self._conv2d(self.s, self.W_conv1, 4) + self.b_conv1)
      h_conv2 = tf.nn.relu(self._conv2d(h_conv1, self.W_conv2, 2) + self.b_conv2)

      h_conv2_flat = tf.reshape(h_conv2, [-1, 2592])
      h_fc1 = tf.nn.relu(tf.matmul(h_conv2_flat, self.W_fc1) + self.b_fc1)

      # TODO:
      self.initial_lstm_state = tf.zeros([1, self.lstm.state_size])
      scope = "net_" + str(thread_index)
      lstm_output, self.lstm_state = self.lstm(h_fc1, self.initial_lstm_state, scope)

      # policy (output)
      self.pi = tf.nn.softmax(tf.matmul(lstm_output, self.W_fc2) + self.b_fc2)
      # value (output)
      v_ = tf.matmul(lstm_output, self.W_fc3) + self.b_fc3
      self.v = tf.reshape( v_, [-1] )

      self.reset_state()

  def prepare_loss(self, entropy_beta):
    with tf.device(self._device):
      # taken action (input for policy)
      self.a = tf.placeholder("float", [None, self._action_size])
    
      # temporary difference (R-V) (input for policy)
      self.td = tf.placeholder("float", [None])
      # policy entropy
      entropy = -tf.reduce_sum(self.pi * tf.log(self.pi))

      # policy loss (output)  (add minus, because this is for gradient ascent)
      policy_loss = -( tf.reduce_sum( tf.mul( tf.log(self.pi), self.a ) ) * self.td +
                             entropy * entropy_beta )

      # R (input for value)
      self.r = tf.placeholder("float", [None])
      # value loss (output)
      # (Learning rate for Critic is half of Actor's, so multiply by 0.5)
      value_loss = 0.5 * tf.nn.l2_loss(self.r - self.v)

      # gradienet of policy and value are summed up
      self.total_loss = policy_loss + value_loss

  def reset_state(self):
    self.lstm_state_out = np.zeros([1, self.lstm.state_size])

  def run_policy_and_value(self, sess, s_t):
    pi_out, v_out, self.lstm_state_out = sess.run( [self.pi, self.v, self.lstm_state],
                                                   feed_dict = {self.s : [s_t],
                                                                self.initial_lstm_state :
                                                                self.lstm_state_out } )
    return (pi_out[0], v_out[0])

  def run_policy(self, sess, s_t):
    pi_out, self.lstm_state_out = sess.run( [self.pi, self.lstm_state],
                                            feed_dict = {self.s : [s_t],
                                                         self.initial_lstm_state :
                                                         self.lstm_state_out } )
    return pi_out[0]
  
  def run_value(self, sess, s_t):
    prev_lstm_state_out = self.lstm_state_out
    v_out, _ = sess.run( [self.v, self.lstm_state],
                         feed_dict = {self.s : [s_t],
                                      self.initial_lstm_state :
                                      self.lstm_state_out } )
    # roll back lstm state
    self.lstm_state_out = prev_lstm_state_out
    return v_out[0]

  def get_vars(self):
    return [self.W_conv1, self.b_conv1,
            self.W_conv2, self.b_conv2,
            self.W_fc1, self.b_fc1,
            self.lstm.matrix, self.lstm.bias,
            self.W_fc2, self.b_fc2,
            self.W_fc3, self.b_fc3]

  def sync_from(self, src_netowrk, name=None):
    src_vars = src_netowrk.get_vars()
    dst_vars = self.get_vars()

    sync_ops = []

    with tf.device(self._device):
      with tf.op_scope([], name, "GameACNetwork") as name:
        for(src_var, dst_var) in zip(src_vars, dst_vars):
          sync_op = tf.assign(dst_var, src_var)
          sync_ops.append(sync_op)

        return tf.group(*sync_ops, name=name)

  # weight initialization based on muupan's code
  # https://github.com/muupan/async-rl/blob/master/a3c_ale.py
  def _fc_weight_variable(self, shape):
    input_channels = shape[0]
    d = 1.0 / np.sqrt(input_channels)
    initial = tf.random_uniform(shape, minval=-d, maxval=d)
    return tf.Variable(initial)

  def _fc_bias_variable(self, shape, input_channels):
    d = 1.0 / np.sqrt(input_channels)
    initial = tf.random_uniform(shape, minval=-d, maxval=d)
    return tf.Variable(initial)  

  def _conv_weight_variable(self, shape):
    w = shape[0]
    h = shape[1]
    input_channels = shape[2]
    d = 1.0 / np.sqrt(input_channels * w * h)
    initial = tf.random_uniform(shape, minval=-d, maxval=d)
    return tf.Variable(initial)

  def _conv_bias_variable(self, shape, w, h, input_channels):
    d = 1.0 / np.sqrt(input_channels * w * h)
    initial = tf.random_uniform(shape, minval=-d, maxval=d)
    return tf.Variable(initial)

  def _conv2d(self, x, W, stride):
    return tf.nn.conv2d(x, W, strides = [1, stride, stride, 1], padding = "VALID")

{'fraction_non_alphanumeric': 0.12002652519893899, 'fraction_numerical': 0.02345822281167109, 'mean_word_length': 2.2537756202804746, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 28, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '735528', 'n_tokens_mistral': 4413, 'n_tokens_neox': 4197, 'n_words': 857}