<jupyter_start><jupyter_text>The “Critic” estimates the value function. This could be the action-value (the Q value) or state-value (the V value).The “Actor” updates the policy distribution in the direction suggested by the Critic (such as with policy gradients).<jupyter_code>class ActorCriticNetwork(nn.Module):
  def __init__(self,alpha,input_dims,fc1_dims,fc2_dims,n_actions):
    super(ActorCriticNetwork,self).__init__()
    self.alpha = alpha
    self.input_dims = input_dims
    self.fc1_dims = fc1_dims
    self.fc2_dims = fc2_dims
    self.n_actions = n_actions
    self.fc1 = nn.Linear(*self.input_dims,self.fc1_dims)
    self.fc2 = nn.Linear(self.fc1_dims,self.fc2_dims)

    self.pi = nn.Linear(self.fc2_dims,n_actions)
    self.v = nn.Linear(self.fc2_dims,1)

    self.optimizer = optim.Adam(self.parameters(),lr=self.alpha)
    self.device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
    self.to(self.device)

  def forward(self, observation):
    state = torch.Tensor(observation).to(self.device)
    x = F.relu(self.fc1(state))
    x = F.relu(self.fc2(x))
    pi = self.pi(x)
    v = self.v(x)

    return pi,v<jupyter_output><empty_output><jupyter_text>Here the agent chooses an action on the basis of the evaluation that the network makes when presented the current states of the environment.Thr Critic evaluates the actor's choice of action by evaluating the value function.<jupyter_code>class NewAgent(object):
  def __init__(self,alpha,input_dims,gamma=0.99,layer1_size=256,layer2_size=256,n_actions=4):
    self.gamma = gamma
    self.actor_critic = ActorCriticNetwork(alpha,input_dims,layer1_size,layer2_size,n_actions=n_actions)
    self.log_probs = None

  def choose_action(self,observation):
    policy,_ = self.actor_critic.forward(observation)
    policy = F.softmax(policy)
    action_probs = torch.distributions.Categorical(policy)
    action = action_probs.sample()
    self.log_probs = action_probs.log_prob(action)

    return action.item()

  def learn(self, state, reward, state_, done):
    self.actor_critic.optimizer.zero_grad()

    _,critic_value = self.actor_critic.forward(state)
    _,critic_value_ = self.actor_critic.forward(state_)

    reward = torch.tensor(reward, dtype=torch.float).to(self.actor_critic.device)
    delta = reward + self.gamma*critic_value_*(1-int(done)) - critic_value

    actor_loss = -self.log_probs*delta #gradient ascent so it is -ve
    critic_loss = delta**2

    (actor_loss+critic_loss).backward()
    self.actor_critic.optimizer.step()

def plot(values,moving_avg_period):
  plt.figure(2)
  plt.clf()
  plt.title('Training..')
  plt.xlabel('Episode')
  plt.ylabel('Duration')
  plt.plot(values)
  moving_avg = get_moving_average(moving_avg_period,values)
  plt.plot(moving_avg)
  plt.pause(0.001)
  print("Episode",len(values),"\n",moving_avg_period,"episode moving avg:",moving_avg[-1])
  if is_ipython: ipythondisplay.clear_output(wait=True)

def get_moving_average(period,values):
  values = torch.tensor(values,dtype=torch.float)
  if len(values)>=period:
    moving_avg = values.unfold(dimension=0, size=period,step=1).mean(dim=1).flatten(start_dim=0)
    moving_avg = torch.cat((torch.zeros(period-1),moving_avg))
    return moving_avg.numpy()
  else:
    moving_avg = torch.zeros(len(values))
    return moving_avg.numpy()
agent = NewAgent(alpha=0.00001, input_dims=[8], gamma=0.99, n_actions=4,layer1_size=2048,layer2_size=512)

env = wrap_env(gym.make('LunarLander-v2').unwrapped)
score_history = []
num_episodes = 2000

for i in range(num_episodes):
  done = False
  observation = env.reset()
  score = 0
  while not done:
    action = agent.choose_action(observation)
    observation_,reward,done,info = env.step(action)
    agent.learn(observation,reward,observation_,done)
    observation = observation_
    score+=reward

  score_history.append(score)
  # print("episode",i,"score %.2f" %score)
  plot(score_history,100)

env.close()
show_video()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1238716148445336, 'fraction_numerical': 0.0183049147442327, 'mean_word_length': 4.962630792227205, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 10, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '504950', 'license': 'MIT', 'n_tokens_mistral': 1476, 'n_tokens_neox': 1436, 'n_words': 244, 'path': 'ActorCriticLunarLander.ipynb', 'repo_name': 'AdityaKapoor74/Reinforcement-Learning-Practice', 'size': '3988'}