parameters: (self,
                 session,
                 game,
                 policy_network_layers=(256, 256),
                 advantage_network_layers=(128, 128),
                 num_iterations: int = 100,
                 num_traversals: int = 20,
                 learning_rate: float = 1e-4,
                 batch_size_advantage=None,
                 batch_size_strategy=None,
                 memory_capacity: int = int(1e6),
                 policy_network_train_steps: int = 1,
                 advantage_network_train_steps: int = 1,
                 reinitialize_advantage_networks: bool = True)
docstring: Initialize the Deep CFR algorithm.
  
      Args:
        session: (tf.Session) TensorFlow session.
        game: Open Spiel game.
        policy_network_layers: (list[int]) Layer sizes of strategy net MLP.
        advantage_network_layers: (list[int]) Layer sizes of advantage net MLP.
        num_iterations: Number of iterations.
        num_traversals: Number of traversals per iteration.
        learning_rate: Learning rate.
        batch_size_advantage: (int or None) Batch size to sample from advantage
          memories.
        batch_size_strategy: (int or None) Batch size to sample from strategy
          memories.
        memory_capacity: Number of samples that can be stored in memory.
        policy_network_train_steps: Number of policy network training steps (per
          iteration).
        advantage_network_train_steps: Number of advantage network training steps
          (per iteration).
        reinitialize_advantage_networks: Whether to re-initialize the
          advantage network before training on each iteration.
docstring_summary: Initialize the Deep CFR algorithm.
docstring_tokens: Initialize the Deep CFR algorithm .
function: def __init__(self,
                 session,
                 game,
                 policy_network_layers=(256, 256),
                 advantage_network_layers=(128, 128),
                 num_iterations: int = 100,
                 num_traversals: int = 20,
                 learning_rate: float = 1e-4,
                 batch_size_advantage=None,
                 batch_size_strategy=None,
                 memory_capacity: int = int(1e6),
                 policy_network_train_steps: int = 1,
                 advantage_network_train_steps: int = 1,
                 reinitialize_advantage_networks: bool = True):
      """Initialize the Deep CFR algorithm.
  
      Args:
        session: (tf.Session) TensorFlow session.
        game: Open Spiel game.
        policy_network_layers: (list[int]) Layer sizes of strategy net MLP.
        advantage_network_layers: (list[int]) Layer sizes of advantage net MLP.
        num_iterations: Number of iterations.
        num_traversals: Number of traversals per iteration.
        learning_rate: Learning rate.
        batch_size_advantage: (int or None) Batch size to sample from advantage
          memories.
        batch_size_strategy: (int or None) Batch size to sample from strategy
          memories.
        memory_capacity: Number of samples that can be stored in memory.
        policy_network_train_steps: Number of policy network training steps (per
          iteration).
        advantage_network_train_steps: Number of advantage network training steps
          (per iteration).
        reinitialize_advantage_networks: Whether to re-initialize the
          advantage network before training on each iteration.
      """
      all_players = list(range(game.num_players()))
      super(DeepCFRSolver, self).__init__(game, all_players)
      self._game = game
      if game.get_type().dynamics == pyspiel.GameType.Dynamics.SIMULTANEOUS:
        # `_traverse_game_tree` does not take into account this option.
        raise ValueError("Simulatenous games are not supported.")
      self._session = session
      self._batch_size_advantage = batch_size_advantage
      self._batch_size_strategy = batch_size_strategy
      self._policy_network_train_steps = policy_network_train_steps
      self._advantage_network_train_steps = advantage_network_train_steps
      self._num_players = game.num_players()
      self._root_node = self._game.new_initial_state()
      # TODO(author6) Allow embedding size (and network) to be specified.
      self._embedding_size = len(self._root_node.information_state_tensor(0))
      self._num_iterations = num_iterations
      self._num_traversals = num_traversals
      self._reinitialize_advantage_networks = reinitialize_advantage_networks
      self._num_actions = game.num_distinct_actions()
      self._iteration = 1
      self._environment_steps = 0
  
      # Create required TensorFlow placeholders to perform the Q-network updates.
      self._info_state_ph = tf.placeholder(
          shape=[None, self._embedding_size],
          dtype=tf.float32,
          name="info_state_ph")
      self._info_state_action_ph = tf.placeholder(
          shape=[None, self._embedding_size + 1],
          dtype=tf.float32,
          name="info_state_action_ph")
      self._action_probs_ph = tf.placeholder(
          shape=[None, self._num_actions],
          dtype=tf.float32,
          name="action_probs_ph")
      self._iter_ph = tf.placeholder(
          shape=[None, 1], dtype=tf.float32, name="iter_ph")
      self._advantage_ph = []
      for p in range(self._num_players):
        self._advantage_ph.append(
            tf.placeholder(
                shape=[None, self._num_actions],
                dtype=tf.float32,
                name="advantage_ph_" + str(p)))
  
      # Define strategy network, loss & memory.
      self._strategy_memories = ReservoirBuffer(memory_capacity)
      self._policy_network = simple_nets.MLP(self._embedding_size,
                                             list(policy_network_layers),
                                             self._num_actions)
      action_logits = self._policy_network(self._info_state_ph)
      # Illegal actions are handled in the traversal code where expected payoff
      # and sampled regret is computed from the advantage networks.
      self._action_probs = tf.nn.softmax(action_logits)
      self._loss_policy = tf.reduce_mean(
          tf.losses.mean_squared_error(
              labels=tf.math.sqrt(self._iter_ph) * self._action_probs_ph,
              predictions=tf.math.sqrt(self._iter_ph) * self._action_probs))
      self._optimizer_policy = tf.train.AdamOptimizer(learning_rate=learning_rate)
      self._learn_step_policy = self._optimizer_policy.minimize(self._loss_policy)
  
      # Define advantage network, loss & memory. (One per player)
      self._advantage_memories = [
          ReservoirBuffer(memory_capacity) for _ in range(self._num_players)
      ]
      self._advantage_networks = [
          simple_nets.MLP(self._embedding_size, list(advantage_network_layers),
                          self._num_actions) for _ in range(self._num_players)
      ]
      self._advantage_outputs = [
          self._advantage_networks[i](self._info_state_ph)
          for i in range(self._num_players)
      ]
      self._loss_advantages = []
      self._optimizer_advantages = []
      self._learn_step_advantages = []
      for p in range(self._num_players):
        self._loss_advantages.append(
            tf.reduce_mean(
                tf.losses.mean_squared_error(
                    labels=tf.math.sqrt(self._iter_ph) * self._advantage_ph[p],
                    predictions=tf.math.sqrt(self._iter_ph) *
                    self._advantage_outputs[p])))
        self._optimizer_advantages.append(
            tf.train.AdamOptimizer(learning_rate=learning_rate))
        self._learn_step_advantages.append(self._optimizer_advantages[p].minimize(
            self._loss_advantages[p]))
function_tokens: def __init__ ( self , session , game , policy_network_layers = ( 256 , 256 ) , advantage_network_layers = ( 128 , 128 ) , num_iterations : int = 100 , num_traversals : int = 20 , learning_rate : float = 1e-4 , batch_size_advantage = None , batch_size_strategy = None , memory_capacity : int = int ( 1e6 ) , policy_network_train_steps : int = 1 , advantage_network_train_steps : int = 1 , reinitialize_advantage_networks : bool = True ) : all_players = list ( range ( game . num_players ( ) ) ) super ( DeepCFRSolver , self ) . __init__ ( game , all_players ) self . _game = game if game . get_type ( ) . dynamics == pyspiel . GameType . Dynamics . SIMULTANEOUS : # `_traverse_game_tree` does not take into account this option. raise ValueError ( "Simulatenous games are not supported." ) self . _session = session self . _batch_size_advantage = batch_size_advantage self . _batch_size_strategy = batch_size_strategy self . _policy_network_train_steps = policy_network_train_steps self . _advantage_network_train_steps = advantage_network_train_steps self . _num_players = game . num_players ( ) self . _root_node = self . _game . new_initial_state ( ) # TODO(author6) Allow embedding size (and network) to be specified. self . _embedding_size = len ( self . _root_node . information_state_tensor ( 0 ) ) self . _num_iterations = num_iterations self . _num_traversals = num_traversals self . _reinitialize_advantage_networks = reinitialize_advantage_networks self . _num_actions = game . num_distinct_actions ( ) self . _iteration = 1 self . _environment_steps = 0 # Create required TensorFlow placeholders to perform the Q-network updates. self . _info_state_ph = tf . placeholder ( shape = [ None , self . _embedding_size ] , dtype = tf . float32 , name = "info_state_ph" ) self . _info_state_action_ph = tf . placeholder ( shape = [ None , self . _embedding_size + 1 ] , dtype = tf . float32 , name = "info_state_action_ph" ) self . _action_probs_ph = tf . placeholder ( shape = [ None , self . _num_actions ] , dtype = tf . float32 , name = "action_probs_ph" ) self . _iter_ph = tf . placeholder ( shape = [ None , 1 ] , dtype = tf . float32 , name = "iter_ph" ) self . _advantage_ph = [ ] for p in range ( self . _num_players ) : self . _advantage_ph . append ( tf . placeholder ( shape = [ None , self . _num_actions ] , dtype = tf . float32 , name = "advantage_ph_" + str ( p ) ) ) # Define strategy network, loss & memory. self . _strategy_memories = ReservoirBuffer ( memory_capacity ) self . _policy_network = simple_nets . MLP ( self . _embedding_size , list ( policy_network_layers ) , self . _num_actions ) action_logits = self . _policy_network ( self . _info_state_ph ) # Illegal actions are handled in the traversal code where expected payoff # and sampled regret is computed from the advantage networks. self . _action_probs = tf . nn . softmax ( action_logits ) self . _loss_policy = tf . reduce_mean ( tf . losses . mean_squared_error ( labels = tf . math . sqrt ( self . _iter_ph ) * self . _action_probs_ph , predictions = tf . math . sqrt ( self . _iter_ph ) * self . _action_probs ) ) self . _optimizer_policy = tf . train . AdamOptimizer ( learning_rate = learning_rate ) self . _learn_step_policy = self . _optimizer_policy . minimize ( self . _loss_policy ) # Define advantage network, loss & memory. (One per player) self . _advantage_memories = [ ReservoirBuffer ( memory_capacity ) for _ in range ( self . _num_players ) ] self . _advantage_networks = [ simple_nets . MLP ( self . _embedding_size , list ( advantage_network_layers ) , self . _num_actions ) for _ in range ( self . _num_players ) ] self . _advantage_outputs = [ self . _advantage_networks [ i ] ( self . _info_state_ph ) for i in range ( self . _num_players ) ] self . _loss_advantages = [ ] self . _optimizer_advantages = [ ] self . _learn_step_advantages = [ ] for p in range ( self . _num_players ) : self . _loss_advantages . append ( tf . reduce_mean ( tf . losses . mean_squared_error ( labels = tf . math . sqrt ( self . _iter_ph ) * self . _advantage_ph [ p ] , predictions = tf . math . sqrt ( self . _iter_ph ) * self . _advantage_outputs [ p ] ) ) ) self . _optimizer_advantages . append ( tf . train . AdamOptimizer ( learning_rate = learning_rate ) ) self . _learn_step_advantages . append ( self . _optimizer_advantages [ p ] . minimize ( self . _loss_advantages [ p ] ) )