<jupyter_start><jupyter_text>Comparison Sarsa and Q-Learning FrozenLake<jupyter_code>%load_ext autoreload
%autoreload 2
import sys
sys.path.append('../agents')
sys.path.append('../environments')
sys.path.append('../tools')

import gym
import numpy as np
import matplotlib.pyplot as plt
from pprint import pprint
import pandas as pd
from utils import *
from QLearning import QLearning
from SARSA import SARSA
from MonteCarlo import MonteCarloEV, MonteCarloFV

%matplotlib inline
# initialize env
env = Experiment(gym.make('FrozenLake-v0'))

tries = 10
episodes = 1000
results_QLearning = np.zeros((tries, episodes))
results_MonteCarlo = np.zeros((tries, episodes))<jupyter_output><empty_output><jupyter_text>Learning Rate study<jupyter_code>ts = []
lr = .5
episodes = int(1e3)
decay = 1-(1/episodes)*2
for _ in range(episodes):
    ts.append(lr)
    lr = lr*decay

plt.plot(ts)
ts = []
lr = .8
episodes = int(1e3)
decay = 1-(1/episodes)*3
for _ in range(episodes):
    ts.append(lr)
    lr = lr*decay

plt.plot(ts)<jupyter_output><empty_output><jupyter_text>Q-Learning<jupyter_code>epsilon_decay = 1-(1/episodes)*3
learning_decay = 1-(1/episodes)*2

tries =1
episodes = 100
results_QLearning = np.zeros((tries, episodes))
for t in range(tries):

    agent = QLearning(env.env, learning_rate =0.5, discount_factor=0.9, 
                      exploration_rate=0,
                      epsilon_decay_func = lambda x: x*epsilon_decay,
                      alpha_decay_func = lambda x: x*learning_decay,
                      qtable_default=1
                     )

    # fit and save results
    env.fit(agent, episodes)
    results_QLearning[t, :] = agent.rewards_per_episode<jupyter_output><empty_output><jupyter_text>Monte Carlo Control<jupyter_code>tries =3
episodes = 3000
results_MonteCarlo = np.zeros((tries, episodes))
epsilon_decay = 1-(1/episodes)*2

for t in range(tries):
    agent = MonteCarlo(env.env, discount_factor=0.9, 
                      exploration_rate=0.8,
                      epsilon_decay_func = lambda x: x*epsilon_decay,
                      qtable_default=1
                     )
    # fit and save results
    env.fit(agent, episodes)
    results_MonteCarlo[t, :] = agent.rewards_per_episode

# 3x10000 episodes, qtable default = 0
plot_rewards(np.mean(results_MonteCarlo, axis=0), smoothing=0.1, color='green')
# 3x3000 episodes, qtable default =1
plot_rewards(np.mean(results_MonteCarlo, axis=0), smoothing=0.1, color='green')<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.11285140562248996, 'fraction_numerical': 0.027309236947791166, 'mean_word_length': 3.6823308270676693, 'pattern_counts': {'":': 0, '<': 17, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '42506', 'license': 'MIT', 'n_tokens_mistral': 964, 'n_tokens_neox': 894, 'n_words': 178, 'path': 'notebooks/MonteCarloTDComparison-FrozenLake.ipynb', 'repo_name': 'lucasosouza/fastRL', 'size': '2490'}