title actor critic method apoorv nandanhttps twitter comnandanapoorv converted to keras 3 by muhammad anas razahttps anasrz com date created 20200513 last modified 20230719 description implement actor critic method in cartpole environment accelerator none introduction this script shows an implementation of actor critic method on cartpolev0 environment actor critic method as an agent takes actions and moves through an environment it learns to map the observed state of the environment to two possible outputs 1 recommended action a probability value for each action in the action space the part of the agent responsible for this output is called the actor 2 estimated rewards in the future sum of all rewards it expects to receive in the future the part of the agent responsible for this output is the critic agent and critic learn to perform their tasks such that the recommended actions from the actor maximize the rewards cartpolev1 a pole is attached to a cart placed on a frictionless track the agent has to apply force to move the cart it is rewarded for every time step the pole remains upright the agent therefore must learn to keep the pole from falling over references cartpolehttp www derongliu orgadpadpcdrombarto1983 pdf actor critic methodhttps hal inria frhal00840470document setup configuration parameters for the whole setup implement actor critic network this network learns two functions 1 actor this takes as input the state of our environment and returns a probability value for each action in its action space 2 critic this takes as input the state of our environment and returns an estimate of total rewards in the future in our implementation they share the initial layer train env render adding this line would show the attempts of the agent in a pop up window predict action probabilities and estimated future rewards from environment state sample action from action probability distribution apply the sampled action in our environment update running reward to check condition for solving calculate expected value from rewards at each timestep what was the total reward received after that timestep rewards in the past are discounted by multiplying them with gamma these are the labels for our critic normalize calculating loss values to update our network at this point in history the critic estimated that we would get a total reward value in the future we took an action with log probability of logprob and ended up recieving a total reward ret the actor must be updated so that it predicts an action that leads to high rewards compared to critic s estimate with high probability the critic must be updated so that it predicts a better estimate of the future rewards backpropagation clear the loss and reward history log details visualizations in early stages of training imgurhttps i imgur com5gcs5kh gif in later stages of training imgurhttps i imgur com5ziizud gif introduction this script shows an implementation of actor critic method on cartpole v0 environment actor critic method as an agent takes actions and moves through an environment it learns to map the observed state of the environment to two possible outputs 1 recommended action a probability value for each action in the action space the part of the agent responsible for this output is called the actor 2 estimated rewards in the future sum of all rewards it expects to receive in the future the part of the agent responsible for this output is the critic agent and critic learn to perform their tasks such that the recommended actions from the actor maximize the rewards cartpole v1 a pole is attached to a cart placed on a frictionless track the agent has to apply force to move the cart it is rewarded for every time step the pole remains upright the agent therefore must learn to keep the pole from falling over references cartpole http www derongliu org adp adp cdrom barto1983 pdf actor critic method https hal inria fr hal 00840470 document setup configuration parameters for the whole setup discount factor for past rewards create the environment smallest number such that 1 0 eps 1 0 implement actor critic network this network learns two functions 1 actor this takes as input the state of our environment and returns a probability value for each action in its action space 2 critic this takes as input the state of our environment and returns an estimate of total rewards in the future in our implementation they share the initial layer train run until solved env render adding this line would show the attempts of the agent in a pop up window predict action probabilities and estimated future rewards from environment state sample action from action probability distribution apply the sampled action in our environment update running reward to check condition for solving calculate expected value from rewards at each timestep what was the total reward received after that timestep rewards in the past are discounted by multiplying them with gamma these are the labels for our critic normalize calculating loss values to update our network at this point in history the critic estimated that we would get a total reward value in the future we took an action with log probability of log_prob and ended up recieving a total reward ret the actor must be updated so that it predicts an action that leads to high rewards compared to critic s estimate with high probability actor loss the critic must be updated so that it predicts a better estimate of the future rewards backpropagation clear the loss and reward history log details condition to consider the task solved visualizations in early stages of training imgur https i imgur com 5gcs5kh gif in later stages of training imgur https i imgur com 5ziizud gif
import os

os.environ["KERAS_BACKEND"] = "tensorflow"

import keras
from keras import layers

import gym
import numpy as np
import tensorflow as tf



seed = 42
gamma = 0.99  
max_steps_per_episode = 10000
env = gym.make("CartPole-v1", new_step_api=True)  
env.reset(seed=seed)
eps = np.finfo(
    np.float32
).eps.item()  



num_inputs = 4
num_actions = 2
num_hidden = 128

inputs = layers.Input(shape=(num_inputs,))
common = layers.Dense(num_hidden, activation="relu")(inputs)
action = layers.Dense(num_actions, activation="softmax")(common)
critic = layers.Dense(1)(common)

model = keras.Model(inputs=inputs, outputs=[action, critic])



optimizer = keras.optimizers.Adam(learning_rate=0.01)
huber_loss = keras.losses.Huber()
action_probs_history = []
critic_value_history = []
rewards_history = []
running_reward = 0
episode_count = 0

while True:  
    state = env.reset()
    episode_reward = 0
    with tf.GradientTape() as tape:
        for timestep in range(1, max_steps_per_episode):
            
            

            state = tf.convert_to_tensor(state)
            state = tf.expand_dims(state, 0)

            
            
            action_probs, critic_value = model(state)
            critic_value_history.append(critic_value[0, 0])

            
            action = np.random.choice(num_actions, p=np.squeeze(action_probs))
            action_probs_history.append(tf.math.log(action_probs[0, action]))

            
            state, reward, done, _, _ = env.step(action)
            rewards_history.append(reward)
            episode_reward += reward

            if done:
                break

        
        running_reward = 0.05 * episode_reward + (1 - 0.05) * running_reward

        
        
        
        
        returns = []
        discounted_sum = 0
        for r in rewards_history[::-1]:
            discounted_sum = r + gamma * discounted_sum
            returns.insert(0, discounted_sum)

        
        returns = np.array(returns)
        returns = (returns - np.mean(returns)) / (np.std(returns) + eps)
        returns = returns.tolist()

        
        history = zip(action_probs_history, critic_value_history, returns)
        actor_losses = []
        critic_losses = []
        for log_prob, value, ret in history:
            
            
            
            
            
            diff = ret - value
            actor_losses.append(-log_prob * diff)  

            
            
            critic_losses.append(
                huber_loss(tf.expand_dims(value, 0), tf.expand_dims(ret, 0))
            )

        
        loss_value = sum(actor_losses) + sum(critic_losses)
        grads = tape.gradient(loss_value, model.trainable_variables)
        optimizer.apply_gradients(zip(grads, model.trainable_variables))

        
        action_probs_history.clear()
        critic_value_history.clear()
        rewards_history.clear()

    
    episode_count += 1
    if episode_count % 10 == 0:
        template = "running reward: {:.2f} at episode {}"
        print(template.format(running_reward, episode_count))

    if running_reward > 195:  
        print("Solved at episode {}!".format(episode_count))
        break