def build_learner(agent, env_outputs, agent_outputs, env_id):
    'Builds the learner loop.\n\n  Args:\n    agent: A snt.RNNCore module outputting `AgentOutput` named tuples, with an\n      `unroll` call for computing the outputs for a whole trajectory.\n    agent_state: The initial agent state for each sequence in the batch.\n    env_outputs: A `StepOutput` namedtuple where each field is of shape\n      [T+1, ...].\n    agent_outputs: An `AgentOutput` namedtuple where each field is of shape\n      [T+1, ...].\n\n  Returns:\n    A tuple of (done, infos, and environment frames) where\n    the environment frames tensor causes an update.\n  '
    learner_outputs = agent.unroll(agent_outputs.action, env_outputs, env_id)
    bootstrap_value = learner_outputs.un_normalized_vf[(- 1)]
    agent_outputs = nest.map_structure((lambda t: t[1:]), agent_outputs)
    (rewards, infos, done, _) = nest.map_structure((lambda t: t[1:]), env_outputs)
    learner_outputs = nest.map_structure((lambda t: t[:(- 1)]), learner_outputs)
    if (FLAGS.reward_clipping == 'abs_one'):
        clipped_rewards = tf.clip_by_value(rewards, (- 1), 1)
    elif (FLAGS.reward_clipping == 'soft_asymmetric'):
        squeezed = tf.tanh((rewards / 5.0))
        clipped_rewards = (tf.where((rewards < 0), (0.3 * squeezed), squeezed) * 5.0)
    discounts = (tf.to_float((~ done)) * FLAGS.discounting)
    game_specific_mean = tf.gather(agent._mean, env_id)
    game_specific_std = tf.gather(agent._std, env_id)
    with tf.device('/cpu'):
        vtrace_returns = vtrace.from_logits(behaviour_policy_logits=agent_outputs.policy_logits, target_policy_logits=learner_outputs.policy_logits, actions=agent_outputs.action, discounts=discounts, rewards=clipped_rewards, un_normalized_values=learner_outputs.un_normalized_vf, normalized_values=learner_outputs.normalized_vf, mean=game_specific_mean, std=game_specific_std, bootstrap_value=bootstrap_value)
    normalized_vtrace = ((vtrace_returns.vs - game_specific_mean) / game_specific_std)
    normalized_vtrace = nest.map_structure(tf.stop_gradient, normalized_vtrace)
    total_loss = compute_policy_gradient_loss(learner_outputs.policy_logits, agent_outputs.action, vtrace_returns.pg_advantages)
    baseline_loss = compute_baseline_loss((normalized_vtrace - learner_outputs.normalized_vf))
    total_loss += (FLAGS.baseline_cost * baseline_loss)
    total_loss += (FLAGS.entropy_cost * compute_entropy_loss(learner_outputs.policy_logits))
    num_env_frames = tf.train.get_global_step()
    learning_rate = tf.train.polynomial_decay(FLAGS.learning_rate, num_env_frames, FLAGS.total_environment_frames, 0)
    optimizer = tf.train.RMSPropOptimizer(learning_rate, FLAGS.decay, FLAGS.momentum, FLAGS.epsilon)
    if (FLAGS.gradient_clipping > 0.0):
        variables = tf.trainable_variables()
        gradients = tf.gradients(total_loss, variables)
        (gradients, _) = tf.clip_by_global_norm(gradients, FLAGS.gradient_clipping)
        train_op = optimizer.apply_gradients(zip(gradients, variables))
    else:
        train_op = optimizer.minimize(total_loss)
    with tf.control_dependencies([train_op]):
        num_env_frames_and_train = num_env_frames.assign_add((FLAGS.batch_size * FLAGS.unroll_length))
    tf.summary.scalar('learning_rate', learning_rate)
    tf.summary.scalar('total_loss', total_loss)
    tf.summary.histogram('action', agent_outputs.action)
    with tf.device('/cpu'):
        (mean, mean_squared) = agent.update_moments(vtrace_returns.vs, env_id)
    return ((done, infos, num_env_frames_and_train) + (mean, mean_squared))
Builds the learner loop.

Args:
  agent: A snt.RNNCore module outputting `AgentOutput` named tuples, with an
    `unroll` call for computing the outputs for a whole trajectory.
  agent_state: The initial agent state for each sequence in the batch.
  env_outputs: A `StepOutput` namedtuple where each field is of shape
    [T+1, ...].
  agent_outputs: An `AgentOutput` namedtuple where each field is of shape
    [T+1, ...].

Returns:
  A tuple of (done, infos, and environment frames) where
  the environment frames tensor causes an update.
def build_learner(agent, env_outputs, agent_outputs, env_id):
    'Builds the learner loop.\n\n  Args:\n    agent: A snt.RNNCore module outputting `AgentOutput` named tuples, with an\n      `unroll` call for computing the outputs for a whole trajectory.\n    agent_state: The initial agent state for each sequence in the batch.\n    env_outputs: A `StepOutput` namedtuple where each field is of shape\n      [T+1, ...].\n    agent_outputs: An `AgentOutput` namedtuple where each field is of shape\n      [T+1, ...].\n\n  Returns:\n    A tuple of (done, infos, and environment frames) where\n    the environment frames tensor causes an update.\n  '
    learner_outputs = agent.unroll(agent_outputs.action, env_outputs, env_id)
    bootstrap_value = learner_outputs.un_normalized_vf[(- 1)]
    agent_outputs = nest.map_structure((lambda t: t[1:]), agent_outputs)
    (rewards, infos, done, _) = nest.map_structure((lambda t: t[1:]), env_outputs)
    learner_outputs = nest.map_structure((lambda t: t[:(- 1)]), learner_outputs)
    if (FLAGS.reward_clipping == 'abs_one'):
        clipped_rewards = tf.clip_by_value(rewards, (- 1), 1)
    elif (FLAGS.reward_clipping == 'soft_asymmetric'):
        squeezed = tf.tanh((rewards / 5.0))
        clipped_rewards = (tf.where((rewards < 0), (0.3 * squeezed), squeezed) * 5.0)
    discounts = (tf.to_float((~ done)) * FLAGS.discounting)
    game_specific_mean = tf.gather(agent._mean, env_id)
    game_specific_std = tf.gather(agent._std, env_id)
    with tf.device('/cpu'):
        vtrace_returns = vtrace.from_logits(behaviour_policy_logits=agent_outputs.policy_logits, target_policy_logits=learner_outputs.policy_logits, actions=agent_outputs.action, discounts=discounts, rewards=clipped_rewards, un_normalized_values=learner_outputs.un_normalized_vf, normalized_values=learner_outputs.normalized_vf, mean=game_specific_mean, std=game_specific_std, bootstrap_value=bootstrap_value)
    normalized_vtrace = ((vtrace_returns.vs - game_specific_mean) / game_specific_std)
    normalized_vtrace = nest.map_structure(tf.stop_gradient, normalized_vtrace)
    total_loss = compute_policy_gradient_loss(learner_outputs.policy_logits, agent_outputs.action, vtrace_returns.pg_advantages)
    baseline_loss = compute_baseline_loss((normalized_vtrace - learner_outputs.normalized_vf))
    total_loss += (FLAGS.baseline_cost * baseline_loss)
    total_loss += (FLAGS.entropy_cost * compute_entropy_loss(learner_outputs.policy_logits))
    num_env_frames = tf.train.get_global_step()
    learning_rate = tf.train.polynomial_decay(FLAGS.learning_rate, num_env_frames, FLAGS.total_environment_frames, 0)
    optimizer = tf.train.RMSPropOptimizer(learning_rate, FLAGS.decay, FLAGS.momentum, FLAGS.epsilon)
    if (FLAGS.gradient_clipping > 0.0):
        variables = tf.trainable_variables()
        gradients = tf.gradients(total_loss, variables)
        (gradients, _) = tf.clip_by_global_norm(gradients, FLAGS.gradient_clipping)
        train_op = optimizer.apply_gradients(zip(gradients, variables))
    else:
        train_op = optimizer.minimize(total_loss)
    with tf.control_dependencies([train_op]):
        num_env_frames_and_train = num_env_frames.assign_add((FLAGS.batch_size * FLAGS.unroll_length))
    tf.summary.scalar('learning_rate', learning_rate)
    tf.summary.scalar('total_loss', total_loss)
    tf.summary.histogram('action', agent_outputs.action)
    with tf.device('/cpu'):
        (mean, mean_squared) = agent.update_moments(vtrace_returns.vs, env_id)
    return ((done, infos, num_env_frames_and_train) + (mean, mean_squared))