parameters: (self, policy_values, q_values, indices)
return_statement: return nash_conv, q_value_loss, policy_loss
docstring: Returns the exploitability descent loss given a policy for a subset.
docstring_summary: Returns the exploitability descent loss given a policy for a subset.
docstring_tokens: Returns the exploitability descent loss given a policy for a subset .
function: def minibatch_loss(self, policy_values, q_values, indices):
      """Returns the exploitability descent loss given a policy for a subset."""
  
      evaluate_policy = _create_policy_evaluator(self.tabular_policy,
                                                 self.q_value_calculator)
      nash_conv, real_q_values, cf_reach_probabilities = tf.py_func(
          evaluate_policy, [policy_values], [tf.float64, tf.float64, tf.float64])
      baseline = tf.reduce_sum(policy_values * q_values, axis=-1, keepdims=True)
      advantage = q_values - baseline
  
      # We now select a minibatch from the data to propagate our loss on.
      policy_values = tf.gather(policy_values, indices)
      advantage = tf.gather(advantage, indices)
      cf_reach_probabilities = tf.gather(cf_reach_probabilities, indices)
  
      # The rest is the same as before.
      loss_per_state = -tf.reduce_sum(
          policy_values * tf.stop_gradient(advantage), axis=-1)
      q_value_loss = tf.reduce_mean((q_values - real_q_values)**2, axis=1)
      q_value_loss = tf.gather(q_value_loss, indices)
      q_value_loss = tf.reduce_sum(q_value_loss * cf_reach_probabilities)
      policy_loss = tf.reduce_sum(loss_per_state * cf_reach_probabilities)
      return nash_conv, q_value_loss, policy_loss
function_tokens: def minibatch_loss ( self , policy_values , q_values , indices ) : evaluate_policy = _create_policy_evaluator ( self . tabular_policy , self . q_value_calculator ) nash_conv , real_q_values , cf_reach_probabilities = tf . py_func ( evaluate_policy , [ policy_values ] , [ tf . float64 , tf . float64 , tf . float64 ] ) baseline = tf . reduce_sum ( policy_values * q_values , axis = - 1 , keepdims = True ) advantage = q_values - baseline # We now select a minibatch from the data to propagate our loss on. policy_values = tf . gather ( policy_values , indices ) advantage = tf . gather ( advantage , indices ) cf_reach_probabilities = tf . gather ( cf_reach_probabilities , indices ) # The rest is the same as before. loss_per_state = - tf . reduce_sum ( policy_values * tf . stop_gradient ( advantage ) , axis = - 1 ) q_value_loss = tf . reduce_mean ( ( q_values - real_q_values ) ** 2 , axis = 1 ) q_value_loss = tf . gather ( q_value_loss , indices ) q_value_loss = tf . reduce_sum ( q_value_loss * cf_reach_probabilities ) policy_loss = tf . reduce_sum ( loss_per_state * cf_reach_probabilities ) return nash_conv , q_value_loss , policy_loss