def learn(self, env_name, failure_observations, subpolicy):
    '\n\t\t\tTrain the actor and critic networks. Here is where the main PPO algorithm resides.\n\n\t\t\tParameters:\n\t\t\t\ttotal_timesteps - the total number of timesteps to train for\n\n\t\t\tReturn:\n\t\t\t\tNone\n\t\t'
    print(f'Learning... Running {self.max_timesteps_per_episode} timesteps per episode, ', end='')
    print(f'{self.timesteps_per_batch} timesteps per batch for a total of {self.training_step} iterations')
    t_so_far = 0
    i_so_far = 0
    while (i_so_far < self.training_step):
        (batch_obs, batch_acts, batch_log_probs, batch_rtgs, batch_lens) = self.rollout(subpolicy, failure_observations)
        t_so_far += np.sum(batch_lens)
        i_so_far += 1
        self.logger['t_so_far'] = t_so_far
        self.logger['i_so_far'] = i_so_far
        (V, _) = self.evaluate(batch_obs, batch_acts)
        A_k = (batch_rtgs - V.detach())
        A_k = ((A_k - A_k.mean()) / (A_k.std() + 1e-10))
        for _ in range(self.n_updates_per_iteration):
            (V, curr_log_probs) = self.evaluate(batch_obs, batch_acts)
            ratios = torch.exp((curr_log_probs - batch_log_probs))
            surr1 = (ratios * A_k)
            surr2 = (torch.clamp(ratios, (1 - self.clip), (1 + self.clip)) * A_k)
            actor_loss = (- torch.min(surr1, surr2)).mean()
            critic_loss = nn.MSELoss()(V, batch_rtgs)
            self.actor_optim.zero_grad()
            actor_loss.backward(retain_graph=True)
            self.actor_optim.step()
            self.critic_optim.zero_grad()
            critic_loss.backward()
            self.critic_optim.step()
            self.logger['actor_losses'].append(actor_loss.detach())
            self.logger['actor_network'] = self.actor
        self._log_summary()
        if ((i_so_far % self.save_freq) == 0):
            if subpolicy:
                torch.save(self.actor.state_dict(), (('./ppo_actor_subpolicy' + env_name) + '.pth'))
                torch.save(self.critic.state_dict(), (('./ppo_critic_subpolicy' + env_name) + '.pth'))
            else:
                torch.save(self.actor.state_dict(), (('./ppo_actor' + env_name) + '.pth'))
                torch.save(self.critic.state_dict(), (('./ppo_critic' + env_name) + '.pth'))
Train the actor and critic networks. Here is where the main PPO algorithm resides.

Parameters:
        total_timesteps - the total number of timesteps to train for

Return:
        None
def learn(self, env_name, failure_observations, subpolicy):
    '\n\t\t\tTrain the actor and critic networks. Here is where the main PPO algorithm resides.\n\n\t\t\tParameters:\n\t\t\t\ttotal_timesteps - the total number of timesteps to train for\n\n\t\t\tReturn:\n\t\t\t\tNone\n\t\t'
    print(f'Learning... Running {self.max_timesteps_per_episode} timesteps per episode, ', end=)
    print(f'{self.timesteps_per_batch} timesteps per batch for a total of {self.training_step} iterations')
    t_so_far = 0
    i_so_far = 0
    while (i_so_far < self.training_step):
        (batch_obs, batch_acts, batch_log_probs, batch_rtgs, batch_lens) = self.rollout(subpolicy, failure_observations)
        t_so_far += np.sum(batch_lens)
        i_so_far += 1
        self.logger['t_so_far'] = t_so_far
        self.logger['i_so_far'] = i_so_far
        (V, _) = self.evaluate(batch_obs, batch_acts)
        A_k = (batch_rtgs - V.detach())
        A_k = ((A_k - A_k.mean()) / (A_k.std() + 1e-10))
        for _ in range(self.n_updates_per_iteration):
            (V, curr_log_probs) = self.evaluate(batch_obs, batch_acts)
            ratios = torch.exp((curr_log_probs - batch_log_probs))
            surr1 = (ratios * A_k)
            surr2 = (torch.clamp(ratios, (1 - self.clip), (1 + self.clip)) * A_k)
            actor_loss = (- torch.min(surr1, surr2)).mean()
            critic_loss = nn.MSELoss()(V, batch_rtgs)
            self.actor_optim.zero_grad()
            actor_loss.backward(retain_graph=True)
            self.actor_optim.step()
            self.critic_optim.zero_grad()
            critic_loss.backward()
            self.critic_optim.step()
            self.logger['actor_losses'].append(actor_loss.detach())
            self.logger['actor_network'] = self.actor
        self._log_summary()
        if ((i_so_far % self.save_freq) == 0):
            if subpolicy:
                torch.save(self.actor.state_dict(), (('./ppo_actor_subpolicy' + env_name) + '.pth'))
                torch.save(self.critic.state_dict(), (('./ppo_critic_subpolicy' + env_name) + '.pth'))
            else:
                torch.save(self.actor.state_dict(), (('./ppo_actor' + env_name) + '.pth'))
                torch.save(self.critic.state_dict(), (('./ppo_critic' + env_name) + '.pth'))