<jupyter_start><jupyter_text>Reinforcement Learning Textbook Exercise 4.9<jupyter_code>import numpy as np
import matplotlib.pyplot as plt

p = 0.55
state_size = 100

# Initialize state values
states = [0 for i in range(state_size + 1)]
states[state_size] = 1

# initialize with random values
# for i in range(1, state_size):
#     states[i] = np.random.random()

def draw_state_values(states):
    plt.plot([i for i in range(state_size+1)], states)

def simulate(states):
    threshold = 0.1
    # while delta > threshold:
    for k in range(2000):
        for i in range(1, state_size):
            max_r = 0
            # actions
            for j in range(1, min(i + 1, state_size - i + 1)):
                # reward for each action
                r = p * states[i+j] + (1-p) * states[i-j]
                max_r = max(max_r, r)
            states[i] = max_r
        if k in [0, 1, 2, 3, 10, 32, 50]:
            draw_state_values(states)
    draw_state_values(states)
    plt.show()
    return states

def optimal_policy(states):
    policy = []
    for i in range(1, state_size):
        # get best policy
        rewards = dict()
        for j in range(1, min(i+1, state_size-i+1)):
            # rewards for each eaction
            rewards[j] = p * states[i+j] + (1-p) * states[i-j]
        best_move = max(rewards, key=lambda x: rewards[x])
        policy.append(best_move)
    plt.bar([i for i in range(1, state_size)], policy)
    plt.show()

simulate(states)
plt.show()
optimal_policy(states)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.12059973924380704, 'fraction_numerical': 0.026727509778357236, 'mean_word_length': 2.449438202247191, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 6, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '84451', 'license': 'MIT', 'n_tokens_mistral': 560, 'n_tokens_neox': 527, 'n_words': 149, 'path': 'Chapter 4/gamble.ipynb', 'repo_name': 'ZhenguoChen/Reinforcement-Learning-2nd-Edition-by-Sutton-Exercise-Solutions', 'size': '1534'}