<jupyter_start><jupyter_text>OpenSpiel* This Colab gets you started the basics of OpenSpiel.* OpenSpiel is a framework for reinforcement learning in games. The code is hosted [on github](https://github.com/deepmind/open_spiel/).* There is an accompanying video tutorial that works through this colab. It will be linked here once it is live.* There is also an [OpenSpiel paper](https://arxiv.org/abs/1908.09453) with more detail.  Install The following command will install OpenSpiel via pip.Only the required dependencies are installed. You may need other dependencies if you use some of the algorithms. There is a [the complete list of packages and versions](https://github.com/deepmind/open_spiel/blob/master/open_spiel/scripts/python_extra_deps.sh) we install for the CI tests, which can be installed as necessary.<jupyter_code>!pip install --upgrade open_spiel<jupyter_output><empty_output><jupyter_text>Part 1. OpenSpiel API Basics.<jupyter_code># Importing pyspiel and showing the list of supported games.
import pyspiel
print(pyspiel.registered_names())
# Loading a game (with no/default parameters).
game = pyspiel.load_game("tic_tac_toe")
print(game)
# Some properties of the games.
print(game.num_players())
print(game.max_utility())
print(game.min_utility())
print(game.num_distinct_actions())
# Creating initial states.
state = game.new_initial_state()
print(state)
# Basic information about states.
print(state.current_player())
print(state.is_terminal())
print(state.returns())
print(state.legal_actions())
# Playing the game: applying actions.
state = game.new_initial_state()
state.apply_action(1)
print(state)
print(state.current_player())
state.apply_action(2)
state.apply_action(4)
state.apply_action(0)
state.apply_action(7)
print(state)
print(state.is_terminal())
print(state.player_return(0))   # win for x (player 0)
print(state.current_player())
# Different game: Breakthrough with default parameters (number of rows and columns are both 8)
game = pyspiel.load_game("breakthrough")
state = game.new_initial_state()
print(state)
# Parameterized games: loading a 6x6 Breakthrough.
game = pyspiel.load_game("breakthrough(rows=6,columns=6)")
state = game.new_initial_state()
print(state)
print(state.legal_actions())
print(game.num_distinct_actions())
for action in state.legal_actions():
  print(f"{action} {state.action_to_string(action)}")<jupyter_output><empty_output><jupyter_text>Part 2. Normal-form Games and Evolutionary Dynamics in OpenSpiel.<jupyter_code>import pyspiel
game = pyspiel.create_matrix_game([[1, -1], [-1, 1]], [[-1, 1], [1, -1]])
print(game)   # name not provided: uses a default
state = game.new_initial_state()
print(state)  # action names also not provided; defaults used
# Normal-form games are 1-step simultaneous-move games.
print(state.current_player())    # special player id 
print(state.legal_actions(0))    # query legal actions for each player
print(state.legal_actions(1))
print(state.is_terminal())

# Applying a joint action (one action per player)
state.apply_actions([0, 0])
print(state.is_terminal())
print(state.returns())
# Evolutionary dynamics in Rock, Paper, Scissors
from open_spiel.python.egt import dynamics
from open_spiel.python.egt.utils import game_payoffs_array
import numpy as np

game = pyspiel.load_matrix_game("matrix_rps")   # load the Rock, Paper, Scissors matrix game
payoff_matrix = game_payoffs_array(game)        # convert any normal-form game to a numpy payoff matrix

dyn = dynamics.SinglePopulationDynamics(payoff_matrix, dynamics.replicator)
x = np.array([0.2, 0.2, 0.6])                   # population heavily-weighted toward scissors
dyn(x)
# Choose a step size and apply the dynamic
alpha = 0.01
x += alpha * dyn(x)
print(x)
x += alpha * dyn(x)
print(x)
x += alpha * dyn(x)
x += alpha * dyn(x)
x += alpha * dyn(x)
x += alpha * dyn(x)
print(x)<jupyter_output><empty_output><jupyter_text>Part 3. Chance Nodes and Partially-Observable Games.<jupyter_code># Kuhn poker: simplified poker with a 3-card deck (https://en.wikipedia.org/wiki/Kuhn_poker)
import pyspiel
game = pyspiel.load_game("kuhn_poker")
print(game.num_distinct_actions())    # bet and fold

# Chance nodes.
state = game.new_initial_state()
print(state.current_player())     # special chance player id
print(state.is_chance_node())
print(state.chance_outcomes())    # distibution over outcomes as a list of (outcome, probability) pairs
# Applying chance node outcomes: same function as applying actions.
state.apply_action(0)              # let's choose the first card (jack)
print(state.is_chance_node())      # still at a chance node (player 2's card).
print(state.chance_outcomes())     # jack no longer a possible outcome
state.apply_action(1)              # second player gets the queen
print(state.current_player())      # no longer chance node, time to play!
# States vs. information states
print(state)                              # ground/world state (all information open)
print(state.legal_actions())
for action in state.legal_actions():
  print(state.action_to_string(action))
print(state.information_state_string())   # only current player's information!
# Take an action (pass / check), second player's turn.
# Information state tensor is vector of floats (often bits) representing the information state.
state.apply_action(0)
print(state.current_player())
print(state.information_state_string())   # now contains second player's card and the public action sequence
print(state.information_state_tensor())
# Leduc poker is a larger game (6 cards, two suits), 3 actions: fold, check/call, raise.
game = pyspiel.load_game("leduc_poker")
print(game.num_distinct_actions())
state = game.new_initial_state()
print(state)
state.apply_action(0)     # first player gets first jack 
state.apply_action(1)     # second player gets second jack
print(state.current_player())
print(state.information_state_string())
print(state.information_state_tensor())

# Let's check until the second round.
print(state.legal_actions_mask())    # Helper function for neural networks.
state.apply_action(1)     # check
state.apply_action(1)     # check
print(state)
print(state.chance_outcomes())   # public card (4 left in the deck)
state.apply_action(2)
print(state.information_state_string())   # player 0's turn again.<jupyter_output><empty_output><jupyter_text>Part 4. Basic RL: Self-play Q-Learning in Tic-Tac-Toe.<jupyter_code># Let's do independent Q-learning in Tic-Tac-Toe, and play it against random.
# RL is based on python/examples/independent_tabular_qlearning.py
from open_spiel.python import rl_environment
from open_spiel.python import rl_tools
from open_spiel.python.algorithms import tabular_qlearner

# Create the environment
env = rl_environment.Environment("tic_tac_toe")
num_players = env.num_players
num_actions = env.action_spec()["num_actions"]

# Create the agents
agents = [
    tabular_qlearner.QLearner(player_id=idx, num_actions=num_actions)
    for idx in range(num_players)
]
# Train the Q-learning agents in self-play.
for cur_episode in range(25000):
  if cur_episode % 1000 == 0:
    print(f"Episodes: {cur_episode}")
  time_step = env.reset()
  while not time_step.last():
    player_id = time_step.observations["current_player"]
    agent_output = agents[player_id].step(time_step)
    time_step = env.step([agent_output.action])
  # Episode is over, step all agents with final info state.
  for agent in agents:
    agent.step(time_step)
print("Done!")
# Evaluate the Q-learning agent against a random agent.
from open_spiel.python.algorithms import random_agent
eval_agents = [agents[0], random_agent.RandomAgent(1, num_actions, "Entropy Master 2000") ]

time_step = env.reset()
while not time_step.last():
  print("")
  print(env.get_state)
  player_id = time_step.observations["current_player"]
  # Note the evaluation flag. A Q-learner will set epsilon=0 here.
  agent_output = eval_agents[player_id].step(time_step, is_evaluation=True)
  print(f"Agent {player_id} chooses {env.get_state.action_to_string(agent_output.action)}")
  time_step = env.step([agent_output.action])

print("")
print(env.get_state)
print(time_step.rewards)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.10841006752608963, 'fraction_numerical': 0.009576427255985266, 'mean_word_length': 5.5168, 'pattern_counts': {'":': 0, '<': 21, '<?xml version=': 0, '>': 21, 'https://': 4, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '5', 'dir': 'jupyter-structured-clean-dedup', 'id': '275999', 'license': 'Apache-2.0', 'n_tokens_mistral': 2653, 'n_tokens_neox': 2561, 'n_words': 721, 'path': 'open_spiel/colabs/OpenSpielTutorial.ipynb', 'repo_name': 'fsx950223/open_spiel', 'size': '8145'}