<jupyter_start><jupyter_text>Dyna Q> Combination of Q-learning and Q-planning<jupyter_code>import gym
import numpy as np
import matplotlib.pyplot as plt
import random
from IPython.display import clear_output
from time import sleep
env_name = "Taxi-v3"
epsilon = 1
decay_rate = 0.001
min_epsilon = 0.01
max_episodes = 1500
print_interval = 100
test_episodes = 3
lr = 0.4
gamma = 0.99
model_loops = 3
env = gym.make(env_name)
n_states = env.observation_space.n
n_actions = env.action_space.n
print(f"Number of states: {n_states}\n"
      f"Number of actions: {n_actions}")
q_table = np.zeros((n_states, n_actions))
model = np.zeros((n_states, n_actions), dtype=tuple)
memory = []
def choose_action(state):
    global q_table
    if epsilon > np.random.uniform():
        action = env.action_space.sample()
    else:
        action = np.argmax(q_table[state, :])
    return action
def update_table(state, action, reward, done, next_state):
    global q_table
    q_table[state, action] += lr * (reward + gamma * np.max(q_table[next_state, :]) * (1 - done) - q_table[state, action])<jupyter_output><empty_output><jupyter_text>Pseudocode><jupyter_code>running_reward = []
for episode in range(1, 1 + max_episodes):
    state = env.reset()
    done = False  
    episode_reward = 0
    while not done:
        action = choose_action(state)
        next_state, reward, done, _ = env.step(action)
        update_table(state, action, reward, done, next_state)
        model[state, action] = (reward, next_state)
        memory.append((state, action, done))
        for _ in range(model_loops):
            s, a, d = random.choice(memory)
            r, s_ = model[s, a]
            update_table(s, a, r, d, s_)
        
        episode_reward += reward
        state = next_state         
    epsilon  = epsilon - decay_rate if epsilon - decay_rate > min_epsilon else min_epsilon
    
    if episode == 1:
        running_reward.append(episode_reward)
    else:
        running_reward.append(0.99 * running_reward[-1] + 0.01 * episode_reward)
    if episode % print_interval == 0:
        print(f"Ep:{episode}| "
              f"Ep_reward:{episode_reward}| "
              f"Running_reward:{running_reward[-1]:.3f}| "
             f"Epsilon:{epsilon:.3f}| ")
plt.figure()
plt.style.use("ggplot")
plt.plot(np.arange(max_episodes), running_reward)
plt.title("Running_reward")
for episode in range(1, 1 + test_episodes):
    state = env.reset()
    done = False  
    episode_reward = 0
    while not done:
        action = choose_action(state)
        next_state, reward, done, _ = env.step(action)
        env.render()
        clear_output(wait=True)
        sleep(0.3)
        episode_reward += reward
        state = next_state

    print(f"Ep:{episode}| "
          f"Ep_reward:{episode_reward}| ")
    env.close()<jupyter_output>Ep:3| Ep_reward:8|
{'fraction_non_alphanumeric': 0.11932418162618796, 'fraction_numerical': 0.016191481872580078, 'mean_word_length': 3.077474892395983, 'pattern_counts': {'":': 0, '<': 8, '<?xml version=': 0, '>': 12, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '256295', 'license': 'MIT', 'n_tokens_mistral': 1033, 'n_tokens_neox': 992, 'n_words': 216, 'path': 'Dyna_Q/Taxi_env.ipynb', 'repo_name': 'alirezakazemipour/Q-Table-Numpy', 'size': '2841'}