<jupyter_start><jupyter_text><jupyter_code>import numpy as np
import matplotlib.pyplot as plt
%matplotlib inline
fig = plt.figure(figsize=(5,5))
ax = plt.gca()

# draw red wall
plt.plot([1, 1], [0, 1], color='red', linewidth=2)
plt.plot([1, 2], [2, 2], color='red', linewidth=2)
plt.plot([2, 2], [2, 1], color='red', linewidth=2)
plt.plot([2, 3], [1, 1], color='red', linewidth=2)

# 状態を示す文字S0~S8
plt.text(0.5, 2.5, 'S0', size=14, ha='center')
plt.text(1.5, 2.5, 'S1', size=14, ha='center')
plt.text(2.5, 2.5, 'S2', size=14, ha='center')
plt.text(0.5, 1.5, 'S3', size=14, ha='center')
plt.text(1.5, 1.5, 'S4', size=14, ha='center')
plt.text(2.5, 1.5, 'S5', size=14, ha='center')
plt.text(0.5, 0.5, 'S6', size=14, ha='center')
plt.text(1.5, 0.5, 'S7', size=14, ha='center')
plt.text(2.5, 0.5, 'S8', size=14, ha='center')
plt.text(0.5, 2.3, 'START', ha='center')
plt.text(2.5, 0.3, 'GOAL', ha='center')

# 描画範囲の設定と目盛りを消す設定
ax.set_xlim(0,3)
ax.set_ylim(0,3)
plt.tick_params(axis='both', which='both', bottom='off', top='off',
               labelbottom='off', right='off', left='off', labelleft='off')

# draw green circle for S0
line, = ax.plot([0.5], [2.5], marker="o", color='g', markersize=60)
# 初期の方策を決定するパラメータtheta_0を設定

# 行は状態0~7, 列は移動方向で↑，→，↓，←を指す
theta_0 = np.array([[np.nan,      1,      1, np.nan], # S0
                    [np.nan,      1, np.nan,      1], # S1
                    [np.nan, np.nan,      1,      1], # S2
                    [     1,      1,      1, np.nan], # S3
                    [np.nan, np.nan,      1,      1], # S4
                    [     1, np.nan, np.nan, np.nan], # S5
                    [     1, np.nan, np.nan, np.nan], # S6
                    [     1,      1, np.nan, np.nan], # S7 ※S8はゴールなので方策なし
                    ])

[a, b] = theta_0.shape
Q = np.random.rand(a, b) * theta_0
Q
def simple_convert_into_pi_from_theta(theta):
    [m, n] = theta.shape
    pi = np.zeros((m, n))
    for i in range(0, m):
        pi[i, :] = theta[i, :] / np.nansum(theta[i, :])
        print(np.nansum(theta[i, :]))
    pi = np.nan_to_num(pi)

    return pi

pi_0 = simple_convert_into_pi_from_theta(theta_0)
# ε-greedy法

def get_action(s, Q, epsilon, pi_0):
    direction = ['up', 'right', 'down', 'left']

    # 行動決定
    if np.random.rand() < epsilon:
        # εの確率でランダムに動く
        next_direction = np.random.choice(direction, p=pi_0[s, :])
    else:
        # Qの最大値の行動を採用する
        next_direction = direction[np.nanargmax(Q[s, :])]

    # 行動をindexへ
    if next_direction == "up":
        action = 0
    elif next_direction == "right":
        action = 1
    elif next_direction == "down":
        action = 2
    elif next_direction == "left":
        action = 3

    return action

def get_s_next(s, a, Q, epsilon, pi_0):
    # Qとepsilonとpiはいるのか？
    direction = ['up', 'right', 'down', 'left']
    next_direction = direction[a]

    if next_direction == "up":
        s_next = s - 3
    elif next_direction == "right":
        s_next = s + 1
    elif next_direction == "down":
        s_next = s + 3
    elif next_direction == "left":
        s_next = s - 1

    return s_next
def Sarsa(s, a, r, s_next, a_next, Q, eta, gamma):
    if s_next == 8:
        Q[s, a] = Q[s, a] + eta * (r - Q[s, a])
    else:
        Q[s, a] = Q[s, a] + eta * (r + gamma * Q[s_next, a_next] - Q[s, a])

    return Q
def goal_maze_ret_s_a_Q(Q, epsilon, eta, gamma, pi):
    s = 0 # start
    a = a_next = get_action(s, Q, epsilon, pi) # 初期の行動
    s_a_history = [[0, np.nan]]

    while 1:
        a = a_next

        s_a_history[-1][1] = a

        s_next = get_s_next(s, a, Q, epsilon, pi)

        s_a_history.append([s_next, np.nan])

        if s_next == 8:
            r = 1
            a_next = np.nan
        else:
            r = 0
            a_next = get_action(s_next, Q, epsilon, pi)
        
        # 価値関数を更新
        Q = Sarsa(s, a, r, s_next, a_next, Q, eta, gamma)

        # 終了判定
        if s_next == 8:
            break
        else:
            s = s_next
    
    return [s_a_history, Q]
eta = 0.1     # 学習率
gamma = 0.9   # 時間割引率
epsilon = 0.5 # ε-greedy法のランダムを選ぶ比率
v = np.nanmax(Q, axis=1) # 状態ごとに価値の最大値を決める
is_continue = True
episode = 1

while is_continue:
    print("Episode :", episode)

    epsilon = epsilon / 2

    # Sarsaで迷路を解き、移動した履歴と更新したQを決める
    [s_a_history, Q] = goal_maze_ret_s_a_Q(Q, epsilon, eta, gamma, pi_0)

    # 状態価値の変化
    new_v = np.nanmax(Q, axis=1) # 状態ごとに価値の最大値を決める
    print(np.sum(np.abs(new_v - v))) # 状態価値の変化を出力
    v = new_v

    print("Steps :", len(s_a_history))

    episode = episode + 1
    if episode > 100:
        break
print("            up    ,    right  ,     down  ,     left  ")
Q
from matplotlib import animation
from IPython.display import HTML

def init():
    line.set_data([], [])
    return (line, )

def animate(i):
    state = s_a_history[i][0]
    x = (state % 3) + 0.5
    y = 2.5 - int(state / 3)
    line.set_data(x, y)
    return (line, )

anim = animation.FuncAnimation(fig, animate, init_func=init, frames=len(s_a_history), interval=200, repeat=False)
HTML(anim.to_jshtml())<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.171540726202159, 'fraction_numerical': 0.03631010794896958, 'mean_word_length': 2.309090909090909, 'pattern_counts': {'":': 8, '<': 6, '<?xml version=': 0, '>': 6, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 3, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '522045', 'license': 'MIT', 'n_tokens_mistral': 2364, 'n_tokens_neox': 2273, 'n_words': 476, 'path': 'Sarsa.ipynb', 'repo_name': 'Mikaner/reinforcement', 'size': '5095'}