<jupyter_start><jupyter_text>CartPole Problem<jupyter_code>#### Installing stable_baselines for tensorflow 2.x. Its still experimental
!git clone https://github.com/Stable-Baselines-Team/stable-baselines-tf2.git
!pip install ./stable-baselines-tf2<jupyter_output>Processing ./stable-baselines-tf2
Requirement already satisfied: gym[atari,classic_control]>=0.10.9 in /usr/local/lib/python3.7/dist-packages (from stable-baselines==3.0.0a0) (0.17.3)
Requirement already satisfied: scipy in /usr/local/lib/python3.7/dist-packages (from stable-baselines==3.0.0a0) (1.4.1)
Requirement already satisfied: joblib in /usr/local/lib/python3.7/dist-packages (from stable-baselines==3.0.0a0) (1.0.1)
Requirement already satisfied: cloudpickle>=0.5.5 in /usr/local/lib/python3.7/dist-packages (from stable-baselines==3.0.0a0) (1.3.0)
Requirement already satisfied: opencv-python in /usr/local/lib/python3.7/dist-packages (from stable-baselines==3.0.0a0) (4.1.2.30)
Requirement already satisfied: numpy in /usr/local/lib/python3.7/dist-packages (from stable-baselines==3.0.0a0) (1.19.5)
Requirement already satisfied: pandas in /usr/local/lib/python3.7/dist-packages (from stable-baselines==3.0.0a0) (1.1.5)
Requirement already satisfied: matplotlib in /usr/local/lib/python3.7[...]<jupyter_text>Solution<jupyter_code>import gym
import numpy as np
import tensorflow as tf
import matplotlib.pyplot as plt
import tensorflow_probability as tfp
from stable_baselines.common.vec_env import SubprocVecEnv  ### Subprocess environment

class A2C_with_multi_env:

  def __init__(self,env,env_name,num_of_env):
    self.state_dimension=env.observation_space.shape   ### Input state dimension
    self.no_of_action=env.action_space.n               ### No of actions
    self.no_of_env=num_of_env
    self.Actor=None      ### the learner
    self.Critic=None     ### Critic
    self.env=SubprocVecEnv([self.create_env(env_name,i) for i in range(self.no_of_env)])
    self.opt=tf.keras.optimizers.RMSprop(lr=0.0003)
    self.steps_in_epi=256    ### Fixed number of steps in a episode
    self.epochs=5000         ### Number of epochs

  def create_env(self,env_id,rank,seed=101):
    def get_env():         #### The seed lets generate same sequence, and starting state is same and
    ### every time performance is same
      env=gym.make(env_id)
      env.seed(seed+rank)
      return env
    return get_env
  ### Creates the multiple environments

  def get_actor(self):

    input_layer=tf.keras.layers.Input(self.state_dimension)  ### Takes the state for which we want to 
    ### predict the probability distribution of the actions.

    layer_1=tf.keras.layers.Dense(128,activation="relu")(input_layer)
    layer_2=tf.keras.layers.Dense(128,activation="relu")(layer_1)
    layer_3=tf.keras.layers.Dense(128,activation="relu")(layer_2)
    
    output_layer=tf.keras.layers.Dense(self.no_of_action, activation="softmax")(layer_3)

    ### Predicts the peobability of all the actions on the state s, so the number of nodes in
    ### the final layer of model is equal to the number of actions 
    ### and we generate a probabilistic distribution so softmax is used as the activation function.
    
    model=tf.keras.Model(inputs=[input_layer],outputs=[output_layer])
    
    return model
  
  def get_critic(self):

    input_layer=tf.keras.layers.Input(self.state_dimension)  ### Takes the state for which we want to 
    ### predict the estimate value function V(s)

    layer_1=tf.keras.layers.Dense(128,activation="relu")(input_layer)
    layer_2=tf.keras.layers.Dense(128,activation="relu")(layer_1)
    layer_3=tf.keras.layers.Dense(128,activation="relu")(layer_2)
    
    output_layer=tf.keras.layers.Dense(1)(layer_3)  ### Predicts the Value function for that state.

    model=tf.keras.Model(inputs=[input_layer],outputs=[output_layer])
    
    return model

  def loss_actor(self,S,A,Adv):

    log_pi_A_given_S=self.log_prob(S,A) 
    actor_loss= -1*tf.reduce_mean(tf.multiply(log_pi_A_given_S,Adv))
    return actor_loss
    ### Loss for actor L= - mean(log(P(a|s : w)).A(s,a))

  def loss_critic(self,S,Targets):
    return 0.5*tf.reduce_mean(tf.math.squared_difference(Targets,self.Critic(S)))
    ### Loss for critic L = mean(|R(lambda) - V(s)|^2)
  
  def entropy(self,S):
    logits=self.Actor(S)
    dist=tfp.distributions.Categorical(logits=logits)
    return tf.reduce_mean(dist.entropy())
    ### Entropy induces exploration.
  
  def action(self,s):

    out=self.Actor(s)   
    action_prob=tfp.distributions.Categorical(probs=out)
    action= action_prob.sample()
    ### sampling an action from the obtained probability distributions for all the action
    return action.numpy()  ### Action returned as 1D tensor-> converting to scalar

  def log_prob(self,s,a):

    out=self.Actor(s)
    ### again to be used in batch
    action_prob=tfp.distributions.Categorical(probs=out)
    ### takes in the states and the actions and returns the corresponding log probability 
    ### of the occurence of the taken action a on the state s
    ### log(P[a|s : w]) is obtained.
    
    return action_prob.log_prob(a)
  
  def get_episodes(self):
    ### To sample raw walks in the environment
    states=[]
    actions=[]
    rewards=[]
    values=[]
    next_values=[]
    not_done=[]
    

    curr_state=self.env.reset()
    for _ in range(self.steps_in_epi):
      ### Recording fixed number of steps.
      action=self.action(curr_state)
      next_state,reward,done,_=self.env.step(action)
      ### Obtaining next step.

      value=self.Critic(curr_state)
      value=value.numpy().flatten()
      next_value=self.Critic(next_state)
      next_value=next_value.numpy().flatten()

      states.append(curr_state)
      actions.append(action)
      rewards.append(reward)
      values.append(value)
      next_values.append(next_value)
      not_done.append(1-done)

      #### actions is in the form [[1,2,3,4],[1,2,3,4]..........]
      ### One for each environment on each step.

      ### Each of the variables are in the same form.

      ### Returns a stacked array of size equal to no. of environments.

      ### Automatically resets environment when done.
      
      ### logging the essential required values for loss estimation
  
      curr_state=next_state ### Updating.

    return np.array(states),np.array(actions),np.array(rewards),np.array(values),np.array(next_values),np.array(not_done)  ### All logs returned

  def get_gae(self,next_values,values,rewards,not_dones,gamma=0.99,lam=0.95):
    
    gae=0
    returns=[]
    ### Calculating GAE according to formulation.

    #### This setting is necessary, as we need element wise multiplication
    ### So that we can multiply the values for the correct environment.
    ### If we flatten before this, the returns for one environment will mix with other and 
    ### so will give a wrong estimate.
    for step in reversed(range(len(rewards))):
      TD_error_delta = rewards[step] + gamma * next_values[step] * not_dones[step] - values[step]
      gae = TD_error_delta + gamma * lam * not_dones[step] * gae
      ret = gae + values[step]
      returns.insert(0,ret)
    
    return np.array(returns)
  
  def test_play(self):
    ### Testing results for current weights.
    overall=0
    for _ in range(5):
      curr_states=self.env.reset()
      curr_state=curr_states[0]
      total_reward=0
      done=[False]*self.no_of_env
      while not done[0]:
        a=self.action(curr_state.reshape(1,-1))
        acts=[a[0]]
        for z in range(self.no_of_env-1):
          acts.append(0)
        next_state,reward,done,_=(self.env.step(acts))
        total_reward+=reward[0]
        curr_state=next_state[0]
      overall+=total_reward
    return overall/5
  
  def train(self):
    ### Training
    self.Actor=self.get_actor()
    self.Critic=self.get_critic()

    for i in range(self.epochs):

      ### For each epoch
      s,a,R,v,Nv,Nd=self.get_episodes()
      
      r=self.get_gae(Nv,v,R,Nd)

      s=s.reshape((len(s)*self.no_of_env,self.state_dimension[0]))
      a=a.reshape((len(a)*self.no_of_env,))
      v=v.reshape((len(v)*self.no_of_env,))
      r=r.reshape((len(r)*self.no_of_env,))

      ### Before training we flatten the arrays. (256,4) -> (1024,)

      #print(np.array(s).shape)
      #### Obtaining values
      adv= r - v
      #### Calculating advantages

      adv = (adv - adv.mean())/adv.std()
      ### Normalizing the advantages

      with tf.GradientTape() as t:
        act_loss=self.loss_actor(s,a,adv)
        crit_loss=self.loss_critic(s,r)
        loss=act_loss+0.25*crit_loss-0.01*self.entropy(s)
      ### Calculating loss
      grads=t.gradient(loss,self.Actor.trainable_variables+self.Critic.trainable_variables)
      self.opt.apply_gradients(zip(grads,self.Actor.trainable_variables+self.Critic.trainable_variables))
      #### Applying gradients.
    
      if i%10==0:
        score=self.test_play()
        print(f"On Iteration {i} scores: {score}")
env=gym.make('CartPole-v0')
env._max_episode_steps=1000
agent=A2C_with_multi_env(env,'CartPole-v0',4)
agent.train()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.12785087719298247, 'fraction_numerical': 0.0206140350877193, 'mean_word_length': 3.594962216624685, 'pattern_counts': {'":': 0, '<': 8, '<?xml version=': 0, '>': 12, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 1, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '435966', 'license': 'MIT', 'n_tokens_mistral': 3234, 'n_tokens_neox': 3104, 'n_words': 638, 'path': 'Actor Critic/A2C__with_multi_env.ipynb', 'repo_name': 'abr-98/Reinforcement-Learning', 'size': '9120'}