def get_loss(self, trajectories):
    reward = trajectories['_reward']
    mask = trajectories.mask()
    reward = (reward * mask)
    max_length = trajectories.lengths.max().item()
    action_probabilities = []
    for t in range(max_length):
        proba = self.learning_model(trajectories['frame'][:, t])
        action_probabilities.append(proba.unsqueeze(1))
    action_probabilities = torch.cat(action_probabilities, dim=1)
    critic = []
    for t in range(max_length):
        b = self.critic_model(trajectories['frame'][:, t])
        critic.append(b.unsqueeze(1))
    critic = torch.cat((critic + [b.unsqueeze(1)]), dim=1).squeeze((- 1))
    idx = torch.arange(trajectories.n_elems())
    last_critic = self.critic_model(trajectories['_frame'][(idx, (trajectories.lengths - 1))]).squeeze((- 1))
    critic[(idx, trajectories.lengths)] = last_critic
    target = (reward + ((self.config['discount_factor'] * (1 - trajectories['_done'].float())) * critic[:, 1:].detach()))
    td = (critic[:, :(- 1)] - target)
    critic_loss = (td ** 2)
    critic_loss = ((critic_loss * mask).sum(1) / mask.sum(1))
    avg_critic_loss = critic_loss.mean()
    action_distribution = torch.distributions.Categorical(action_probabilities)
    log_proba = action_distribution.log_prob(trajectories['action'])
    a2c_loss = ((- log_proba) * td.detach())
    a2c_loss = ((a2c_loss * mask).sum(1) / mask.sum(1))
    avg_a2c_loss = a2c_loss.mean()
    entropy = action_distribution.entropy()
    entropy = ((entropy * mask).sum(1) / mask.sum(1))
    avg_entropy = entropy.mean()
    return DictTensor({'critic_loss': avg_critic_loss, 'a2c_loss': avg_a2c_loss, 'entropy_loss': avg_entropy})
def get_loss(self, trajectories):
    reward = trajectories['_reward']
    mask = trajectories.mask()
    reward = (reward * mask)
    max_length = trajectories.lengths.max().item()
    action_probabilities = []
    for t in range(max_length):
        proba = self.learning_model(trajectories['frame'][:, t])
        action_probabilities.append(proba.unsqueeze(1))
    action_probabilities = torch.cat(action_probabilities, dim=1)
    critic = []
    for t in range(max_length):
        b = self.critic_model(trajectories['frame'][:, t])
        critic.append(b.unsqueeze(1))
    critic = torch.cat((critic + [b.unsqueeze(1)]), dim=1).squeeze((- 1))
    idx = torch.arange(trajectories.n_elems())
    last_critic = self.critic_model(trajectories['_frame'][(idx, (trajectories.lengths - 1))]).squeeze((- 1))
    critic[(idx, trajectories.lengths)] = last_critic
    target = (reward + ((self.config['discount_factor'] * (1 - trajectories['_done'].float())) * critic[:, 1:].detach()))
    td = (critic[:, :(- 1)] - target)
    critic_loss = (td ** 2)
    critic_loss = ((critic_loss * mask).sum(1) / mask.sum(1))
    avg_critic_loss = critic_loss.mean()
    action_distribution = torch.distributions.Categorical(action_probabilities)
    log_proba = action_distribution.log_prob(trajectories['action'])
    a2c_loss = ((- log_proba) * td.detach())
    a2c_loss = ((a2c_loss * mask).sum(1) / mask.sum(1))
    avg_a2c_loss = a2c_loss.mean()
    entropy = action_distribution.entropy()
    entropy = ((entropy * mask).sum(1) / mask.sum(1))
    avg_entropy = entropy.mean()
    return DictTensor({'critic_loss': avg_critic_loss, 'a2c_loss': avg_a2c_loss, 'entropy_loss': avg_entropy})