parameters: (num_players, env, payoffs, centralized)
return_statement: return rewards, opts, converge_point, percent_opt
docstring: Run the experiments.
docstring_summary: Run the experiments.
docstring_tokens: Run the experiments .
function: def run_experiment(num_players, env, payoffs, centralized):
    """Run the experiments."""
    num_states = FLAGS.num_states
    num_messages = FLAGS.num_messages
    num_actions = env.action_spec()["num_actions"]
  
    # Results to store
    num_runs = FLAGS.num_runs
    training_episodes = FLAGS.num_episodes
    log_interval = FLAGS.log_interval
    rewards = np.zeros((num_runs, training_episodes // log_interval))
    opts = np.zeros((num_runs, training_episodes // log_interval))
    converge_point = np.zeros((num_states, num_states))
    percent_opt = 0
  
    # Repeat the experiment num_runs times
    for i in range(num_runs):
      eps_schedule = rl_tools.LinearSchedule(
          FLAGS.eps_init, FLAGS.eps_final, FLAGS.eps_decay_steps *
          2)  # *2 since there are 2 agent steps per episode
  
      agents = [
          # pylint: disable=g-complex-comprehension
          tabular_qlearner.QLearner(
              player_id=idx,
              num_actions=num_actions,
              step_size=FLAGS.step_size,
              epsilon_schedule=eps_schedule,
              centralized=centralized) for idx in range(num_players)
      ]
  
      # 1. Train the agents
      for cur_episode in range(training_episodes):
        time_step = env.reset()
        # Find cur_state for logging. See lewis_signaling.cc for info_state
        # details.
        cur_state = time_step.observations["info_state"][0][3:].index(1)
        while not time_step.last():
          player_id = time_step.observations["current_player"]
          agent_output = agents[player_id].step(time_step)
          time_step = env.step([agent_output.action])
  
        # Episode is over, step all agents with final info state.
        for agent in agents:
          agent.step(time_step)
  
        # Store rewards
        reward = time_step.rewards[0]
        max_reward = payoffs[cur_state].max()
        cur_idx = (i, cur_episode // log_interval)
        rewards[cur_idx] += reward / log_interval
        opts[cur_idx] += np.isclose(reward, max_reward) / log_interval
  
      base_info_state0 = [1.0, 0.0, 0.0] + [0.0] * num_states
      base_info_state1 = [0.0, 1.0, 0.0] + [0.0] * num_states
      if centralized:
        base_info_state0 = [base_info_state0, base_info_state0.copy()]
        base_info_state1 = [base_info_state1, base_info_state1.copy()]
  
      for s in range(num_states):
        info_state0 = copy.deepcopy(base_info_state0)
        if centralized:
          info_state0[0][3 + s] = 1.0
        else:
          info_state0[3 + s] = 1.0
        # pylint: disable=protected-access
        m, _ = agents[0]._epsilon_greedy(
            str(info_state0), np.arange(num_messages), 0)
        info_state1 = copy.deepcopy(base_info_state1)
        if centralized:
          info_state1[0][3 + s] = 1.0
          info_state1[1][3 + m] = 1.0
        else:
          info_state1[3 + m] = 1.0
        a, _ = agents[1]._epsilon_greedy(
            str(info_state1), np.arange(num_states), 0)
        converge_point[s, a] += 1
        best_act = payoffs[s].argmax()
        percent_opt += int(a == best_act) / num_runs / num_states
    return rewards, opts, converge_point, percent_opt
function_tokens: def run_experiment ( num_players , env , payoffs , centralized ) : num_states = FLAGS . num_states num_messages = FLAGS . num_messages num_actions = env . action_spec ( ) [ "num_actions" ] # Results to store num_runs = FLAGS . num_runs training_episodes = FLAGS . num_episodes log_interval = FLAGS . log_interval rewards = np . zeros ( ( num_runs , training_episodes // log_interval ) ) opts = np . zeros ( ( num_runs , training_episodes // log_interval ) ) converge_point = np . zeros ( ( num_states , num_states ) ) percent_opt = 0 # Repeat the experiment num_runs times for i in range ( num_runs ) : eps_schedule = rl_tools . LinearSchedule ( FLAGS . eps_init , FLAGS . eps_final , FLAGS . eps_decay_steps * 2 ) # *2 since there are 2 agent steps per episode agents = [ # pylint: disable=g-complex-comprehension tabular_qlearner . QLearner ( player_id = idx , num_actions = num_actions , step_size = FLAGS . step_size , epsilon_schedule = eps_schedule , centralized = centralized ) for idx in range ( num_players ) ] # 1. Train the agents for cur_episode in range ( training_episodes ) : time_step = env . reset ( ) # Find cur_state for logging. See lewis_signaling.cc for info_state # details. cur_state = time_step . observations [ "info_state" ] [ 0 ] [ 3 : ] . index ( 1 ) while not time_step . last ( ) : player_id = time_step . observations [ "current_player" ] agent_output = agents [ player_id ] . step ( time_step ) time_step = env . step ( [ agent_output . action ] ) # Episode is over, step all agents with final info state. for agent in agents : agent . step ( time_step ) # Store rewards reward = time_step . rewards [ 0 ] max_reward = payoffs [ cur_state ] . max ( ) cur_idx = ( i , cur_episode // log_interval ) rewards [ cur_idx ] += reward / log_interval opts [ cur_idx ] += np . isclose ( reward , max_reward ) / log_interval base_info_state0 = [ 1.0 , 0.0 , 0.0 ] + [ 0.0 ] * num_states base_info_state1 = [ 0.0 , 1.0 , 0.0 ] + [ 0.0 ] * num_states if centralized : base_info_state0 = [ base_info_state0 , base_info_state0 . copy ( ) ] base_info_state1 = [ base_info_state1 , base_info_state1 . copy ( ) ] for s in range ( num_states ) : info_state0 = copy . deepcopy ( base_info_state0 ) if centralized : info_state0 [ 0 ] [ 3 + s ] = 1.0 else : info_state0 [ 3 + s ] = 1.0 # pylint: disable=protected-access m , _ = agents [ 0 ] . _epsilon_greedy ( str ( info_state0 ) , np . arange ( num_messages ) , 0 ) info_state1 = copy . deepcopy ( base_info_state1 ) if centralized : info_state1 [ 0 ] [ 3 + s ] = 1.0 info_state1 [ 1 ] [ 3 + m ] = 1.0 else : info_state1 [ 3 + m ] = 1.0 a , _ = agents [ 1 ] . _epsilon_greedy ( str ( info_state1 ) , np . arange ( num_states ) , 0 ) converge_point [ s , a ] += 1 best_act = payoffs [ s ] . argmax ( ) percent_opt += int ( a == best_act ) / num_runs / num_states return rewards , opts , converge_point , percent_opt