<jupyter_start><jupyter_text>**[GVI-01]**Import modules.<jupyter_code>import numpy as np
import matplotlib.pyplot as plt
import matplotlib
matplotlib.rcParams['font.size'] = 12<jupyter_output><empty_output><jupyter_text>**[GVI-02]**Define the Gambler class to simulate the game.<jupyter_code>class Gambler:
  def __init__(self, goal=100, win_rate=0.4):
    self.goal = goal
    self.win_rate = win_rate
    self.states = range(goal+1)

    self.policy = {}
    for s in self.states:
      if s == 0 or s == self.goal:
        continue
      self.policy[s] = np.random.randint(1, s+1)

    self.q = {}
    self.cnt = {}
    for s in self.states:
      for a in range(1, s+1):
        self.q[(s, a)] = 0
        self.cnt[(s, a)] = 0

  def play(self, s, a):
    if s == 0 or s == self.goal:
      return 0, s       # Reward, Next state

    if np.random.random() < self.win_rate:  # Win
      s = min(self.goal, s + a)
      if s == self.goal:
        return 1, s    # Reward, Next state
      return 0, s      # Reward, Next state
    else:                                   # Lose
      s -= a
      if s == 0:
        return -1, s  # Reward, Next state 
      return 0, s     # Reward, Next state<jupyter_output><empty_output><jupyter_text>**[GVI-03]**Define a function to show the action policy and the value function.<jupyter_code>def show_result(gambler):
  fig = plt.figure(figsize=(14, 4))
  linex = range(1, gambler.goal)

  subplot = fig.add_subplot(1, 2, 1)
  liney = [gambler.policy[s] for s in linex]
  subplot.plot(linex, liney)
  subplot.set_ylim([0, 100])
  subplot.set_title('Policy')

  subplot = fig.add_subplot(1, 2, 2)
  liney = [gambler.q[(s, gambler.policy[s])] for s in linex]
  subplot.plot(linex, liney)
  subplot.set_ylim([-1, 1])
  subplot.set_title('Value')<jupyter_output><empty_output><jupyter_text>**[GVI-04]**Define a function to play the game for a single episode.<jupyter_code>def get_episode(gambler):
  episode = []
  s = np.random.randint(1, gambler.goal)
  initial = True
  while True:
    if initial:
      a = np.random.randint(1, s+1)
      initial = False
    else:
      a = gambler.policy[s]
    r, s_new = gambler.play(s, a)
    episode.append((s, a, r))
    if s_new == 0 or s_new == gambler.goal:
      break
    s = s_new

  return episode<jupyter_output><empty_output><jupyter_text>**[GVI-05]**Define a function to update the action policy for a specific state.<jupyter_code>def policy_update(gambler, s):
  q_max = -10**10
  a_best = None
  for a in range(1, s+1):
    if gambler.q[(s, a)] > q_max:
      q_max = gambler.q[(s, a)]
      a_best = a

  gambler.policy[s] = a_best<jupyter_output><empty_output><jupyter_text>**[GVI-06]**Define a function to apply the value iteration algorithm with the Monte Carlo simulation.<jupyter_code>def train(gambler, num):
  c = 0
  while c < num:
    episode = get_episode(gambler)
    episode.reverse()
    total_r = 0
    for (s, a, r) in episode:
      total_r += r
      gambler.cnt[(s, a)] += 1
      gambler.q[(s, a)] += (total_r - gambler.q[(s, a)]) / gambler.cnt[(s, a)]
      policy_update(gambler, s)
    c += len(episode)<jupyter_output><empty_output><jupyter_text>**[GVI-07]**Create a Gambler instance.<jupyter_code>gambler = Gambler()<jupyter_output><empty_output><jupyter_text>**[GVI-08]**Apply the value iteration algorithm with 50,000,000 plays.<jupyter_code>%%time
train(gambler, num=50000000)<jupyter_output>CPU times: user 10min 36s, sys: 95.1 ms, total: 10min 36s
Wall time: 10min 40s<jupyter_text>**[GVI-09]**Show the resulting action policy and value function.<jupyter_code>show_result(gambler)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.14116675839295542, 'fraction_numerical': 0.0275178866263071, 'mean_word_length': 3.33253873659118, 'pattern_counts': {'":': 0, '<': 38, '<?xml version=': 0, '>': 37, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 1, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '9', 'dir': 'jupyter-structured-clean-dedup', 'id': '537774', 'license': 'Apache-2.0', 'n_tokens_mistral': 1487, 'n_tokens_neox': 1400, 'n_words': 331, 'path': "Chapter04/03_Gambler's_Problem_Value_Iteration.ipynb", 'repo_name': 'shozossk/colab_rlbook', 'size': '3634'}