parameters: (trajectories, gamma, lam)
docstring: Add generalized advantage estimator.
      https://arxiv.org/pdf/1506.02438.pdf
  
      Args:
          trajectories: as returned by run_policy(), must include 'values'
              key from add_value().
          gamma: reward discount
          lam: lambda (see paper).
              lam=0 : use TD residuals
              lam=1 : A =  Sum Discounted Rewards - V_hat(s)
  
      Returns:
          None (mutates trajectories dictionary to add 'advantages')
docstring_summary: Add generalized advantage estimator.
      https://arxiv.org/pdf/1506.02438.pdf
docstring_tokens: Add generalized advantage estimator . https : // arxiv . org / pdf / 1506 . 02438 . pdf
function: def add_gae(trajectories, gamma, lam):
      """ Add generalized advantage estimator.
      https://arxiv.org/pdf/1506.02438.pdf
  
      Args:
          trajectories: as returned by run_policy(), must include 'values'
              key from add_value().
          gamma: reward discount
          lam: lambda (see paper).
              lam=0 : use TD residuals
              lam=1 : A =  Sum Discounted Rewards - V_hat(s)
  
      Returns:
          None (mutates trajectories dictionary to add 'advantages')
      """
      for trajectory in trajectories:
          if gamma < 0.999:  # don't scale for gamma ~= 1
              rewards = trajectory['rewards'] * (1 - gamma)
          else:
              rewards = trajectory['rewards']
          values = trajectory['values']
          # temporal differences
          # values[1:] deletes the first element (Vo) and attachs a 0 at the end (the future state value function at the end of the trajectory is 0) 
          # r - Vs + gamma*Vst+1
          tds = rewards - values + np.append(values[1:] * gamma, 0)
          advantages = discount(tds, gamma * lam)
          trajectory['advantages'] = advantages
function_tokens: def add_gae ( trajectories , gamma , lam ) : for trajectory in trajectories : if gamma < 0.999 : # don't scale for gamma ~= 1 rewards = trajectory [ 'rewards' ] * ( 1 - gamma ) else : rewards = trajectory [ 'rewards' ] values = trajectory [ 'values' ] # temporal differences # values[1:] deletes the first element (Vo) and attachs a 0 at the end (the future state value function at the end of the trajectory is 0)  # r - Vs + gamma*Vst+1 tds = rewards - values + np . append ( values [ 1 : ] * gamma , 0 ) advantages = discount ( tds , gamma * lam ) trajectory [ 'advantages' ] = advantages