<jupyter_start><jupyter_text><jupyter_code>!apt-get install -y xvfb python-opengl > /dev/null 2>&1
!pip install gym pyvirtualdisplay > /dev/null 2>&1
! pip install pyglet==1.3.2
import gym
import numpy as np
import matplotlib.pyplot as plt
from IPython import display as ipythondisplay
import time
from pyvirtualdisplay import Display
display = Display(visible=0, size=(400, 300))
display.start()
import tensorflow as tf
import tensorflow.contrib.slim as slim
import numpy as np
import gym
import matplotlib.pyplot as plt
%matplotlib inline

try:
  xrange = xrange
except:
  xrange = range 
env = gym.make('CartPole-v0')

gamma = 0.99

def discount_rewards(r):
    """ take 1D float array of rewards and compute discounted reward """
    discounted_r = np.zeros_like(r)
    running_add = 0
    for t in reversed(xrange(0, r.size)):
        running_add = running_add * gamma + r[t]
        discounted_r[t] = running_add
    return discounted_r
class agent():
  def __init__(self, lr, s_size, a_size, h_size):
    
    # forward propagation    
    self.state_in = tf.placeholder(shape = [None, s_size], dtype = tf.float32)
    hidden = slim.fully_connected(self.state_in, h_size, biases_initializer = None, activation_fn = tf.nn.relu)
    self.output = slim.fully_connected(hidden, a_size, biases_initializer = None, activation_fn = tf.nn.softmax)
    self.chosen_action = tf.argmax(self.output, 1)
    
    # Backpropagation
    
    self.reward_holder = tf.placeholder(shape = [None], dtype = tf.float32)
    self.action_holder = tf.placeholder(shape = [None], dtype = tf.int32)
    
    self.indexes = tf.range(0, tf.shape(self.output)[0]) * tf.shape(self.output)[1] + self.action_holder
    
    self.responsible_outputs = tf.gather(tf.reshape(self.output, [-1]), self.indexes)
    self.loss = -tf.reduce_mean(tf.log(self.responsible_outputs)*self.reward_holder)
    
    tvars = tf.trainable_variables()
    self.gradient_holders = []
    
    for idx, var in enumerate(tvars):
      placeholder = tf.placeholder(tf.float32, name=str(idx)+'_holder')
      self.gradient_holders.append(placeholder)
      
    self.gradients = tf.gradients(self.loss, tvars)
    
    optimizer = tf.train.AdamOptimizer(learning_rate = lr)
    self.update_batch = optimizer.apply_gradients(zip(self.gradient_holders,tvars))  # also instead tf.gradient we can use computgradients in the adam optimizer
tf.reset_default_graph()

my_agent = agent(lr=1e-2,s_size=4,a_size=2,h_size=8)

total_episodes = 5000
max_ep = 999
update_frequency = 5


init = tf.global_variables_initializer()
with tf.Session() as sess:

  sess.run(init)
  i = 0
  total_reward = []
  total_steps = []
  grad_buffer = sess.run(tf.trainable_variables())
  
  for ix, grad in enumerate(grad_buffer):
    grad_buffer[ix] = grad*0
    
    
  while i < total_episodes:
    s = env.reset()
    running_reward = 0
    ep_history = []
    
    for j in range(max_ep):
      a_dist = sess.run(my_agent.output, feed_dict={my_agent.state_in:[s]})
      a = np.random.choice(a_dist[0], p=a_dist[0]) # probabilistically taking an action
      a = np.argmax(a_dist == a)
      
      s1, r, d, _ = env.step(a)
      ep_history.append([s,a,r,s1])
      s = s1
      
      running_reward += r
      
      if d==True:
        ep_history = np.array(ep_history)
        ep_history[:,2] = discount_rewards(ep_history[:,2])
        
        feed_dict = {my_agent.reward_holder: ep_history[:,2],
                     my_agent.action_holder: ep_history[:,1],
                     my_agent.state_in:np.vstack(ep_history[:,0])}
        
        grads = sess.run(my_agent.gradients, feed_dict = feed_dict)
        
        for idx,grad in enumerate(grads):
                    grad_buffer[idx] += grad
            
        if (i % update_frequency == 0) and (i != 0):
          feed_dict = dictionary = dict(zip(my_agent.gradient_holders, grad_buffer))
          _ = sess.run(my_agent.update_batch, feed_dict = feed_dict)
          for ix, grad in enumerate(grad_buffer):
            grad_buffer[ix] = grad * 0
            
          
        total_reward.append(running_reward)
        total_steps.append(j)
        break
          
         
    if i % 100 == 0:
      print (np.mean(total_reward[-100:]))
    
    i += 1
    
  
  s = env.reset()
  #plt.imshow(prev_screen)

  for i in range(2000):
  
    a_dist = sess.run(my_agent.output, feed_dict={my_agent.state_in:[s]})
    a = np.random.choice(a_dist[0], p=a_dist[0]) # probabilistically taking an action
    a = np.argmax(a_dist == a)
    obs, reward, done, info = env.step(a)
    screen = env.render(mode='rgb_array')
  
      
    plt.imshow(screen)
    ipythondisplay.clear_output(wait=True)
    ipythondisplay.display(plt.gcf())

    time.sleep(1)
    if done:
      break
    
    ipythondisplay.clear_output(wait=True)
    env.close()<jupyter_output>53.0
26.5
36.6
49.28
53.64
77.27
98.14
128.33
164.43
179.74
186.25
180.1
172.85
174.81
186.09
186.24
173.0
174.74
190.59
198.62
{'fraction_non_alphanumeric': 0.11836816720257234, 'fraction_numerical': 0.033360128617363344, 'mean_word_length': 2.9065934065934065, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 8, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '193404', 'license': 'MIT', 'n_tokens_mistral': 1894, 'n_tokens_neox': 1774, 'n_words': 361, 'path': 'Deep_RL_Agent_Cartpole.ipynb', 'repo_name': 'moustafa-7/RL_Course_Projects', 'size': '4976'}