<jupyter_start><jupyter_text>n-step method vs. sum of TD error method Exercise 7.2With an n-step method, the value estimates do change fromstep to step, so an algorithm that used the sum of TD errors (see previous exercise) in place of the error in (7.2) would actually be a slightly different algorithm. Would it be a better algorithm or a worse one? Devise and program a small experiment to answer this question empirically.<jupyter_code>from gym_minigrid.wrappers import FullyObsWrapper

import gym
import gym_minigrid
import math
import numpy as np
import matplotlib.pyplot as plt
import random
%matplotlib inline


'''
Windy Gridworld

Agent is trained using gym_minigrid
https://github.com/maximecb/gym-minigrid
FullyObsWrapper used to simplify observations

Custom environment "windy.py" adds king's moves & wind
'''

def get_position(obs):
    '''
    Get agent's position from state observations
    
    Parameters
    ----------
        obs : dict
            Observations
    Returns
    -------
        tuple
            Agent's position
    '''
    return (np.where(obs['image'][:,:,0] == 10)[0][0], 
            np.where(obs['image'][:,:,0] == 10)[1][0])

def choose_action(s, Q, epsilon):
    '''
    Chooses epsilon-greedy action
    
    Parameters
    ----------
        s : tuple
            Agent's position
        Q : ndarray
            Action-values
        epsilon : float
            Greediness parameter
    Returns
    -------
        int
            Action to be taken
    '''
    i = random.uniform(0, 1)
    if i < epsilon:
        return random.randint(0, Q.shape[-1] - 1)
    else:
        return np.argmax(Q[s])
    
def train_nstep_TD_sum(env, pi, alpha, gamma, n, max_episode):
    '''
    Estimate v using n-step TD error sum
    
    Parameters
    ----------       
        env : gym environment
            Environment to train in
        pi : func
            Policy to evaluate
        alpha : float
            Convergence parameter
        gamma : float
            Discount parameter
        n : int
            Puts the n in n-step
        max_episode : int
            maximum number of episodes
    Returns
    -------
        ndarray
            state-value function for pi
    '''
    vs = []
    v = np.zeros((10+2, 7+2)) # +2 to pad environment boundaries
    episode = 0
    
    while episode < max_episode:
        if episode % 100 == 0:
            print('episode: {}'.format(episode))
        obs = env.reset()
        done = False
        st = get_position(obs)
        s = [st]
        r = [0] # R_0 = 0
        tau = 0
        T = math.inf
        t = 0
        
        while tau != T - 1:
            if t < T:
                a = pi(st)
                obs, reward, done, info = env.step(a)
                st = get_position(obs)
                s.append(st)
                r.append(reward)
                if done:
                    T = t + 1
            tau = t - n + 1
            if tau >= 0:
                gammas = np.array([gamma ** (i - tau) for i in range(tau, min(tau + n - 1, T - 1) + 1)])
                deltas = np.array([r[i + 1] + gamma * v[s[i + 1]] - v[s[i]] for i in range(tau, min(tau + n - 1, T - 1) + 1)])
                error = np.sum(gammas * deltas)
                v[s[tau]] = v[s[tau]] + alpha * error
            t += 1
            
        env.close()
        episode += 1
        vs.append(v.copy())
        
    return vs

def train_nstep_TD(env, pi, alpha, gamma, n, max_episode):

    vs = []
    v = np.zeros((10+2, 7+2)) # +2 to pad environment boundaries
    episode = 0
    
    while episode < max_episode:
        if episode % 100 == 0:
            print('episode: {}'.format(episode))
        obs = env.reset()
        done = False
        st = get_position(obs)
        s = [st]
        r = [0] # R_0 = 0
        tau = 0
        T = math.inf
        t = 0
        
        while tau != T - 1:
            if t < T:
                a = pi(st)
                obs, reward, done, info = env.step(a)
                st = get_position(obs)
                s.append(st)
                r.append(reward)
                if done:
                    T = t + 1
            tau = t - n + 1
            if tau >= 0:
                gammas = np.array([gamma ** (i - tau - 1) for i in range(tau + 1, min(tau + n, T) + 1)])
                r_sub = np.array(r[tau + 1:len(gammas) + tau + 1])
                g = np.sum(r_sub * gammas)
                if tau + n < T:
                    g += gamma ** n * v[s[tau + n]]
                v[s[tau]] = v[s[tau]] + alpha * (g - v[s[tau]])
            t += 1

        env.close()
        episode += 1
        vs.append(v.copy())
    
    return vs

def run(env):
    alpha = 0.5
    gamma = 0.9
    n = 100
    max_episode = 100000
    
    def pi(st):
        # Use a stochastic policy
        return random.randint(0, 8)
    
    vs2 = train_nstep_TD_sum(env, pi, alpha, gamma, n, max_episode)
    vs1 = train_nstep_TD(env, pi, alpha, gamma, n, max_episode)
    
    return vs1, vs2

if __name__ == '__main__':
    # Ex 7.2
    env = gym.make('MiniGrid-Windy-10x7-v0')
    env = FullyObsWrapper(env)
    vs1, vs2 = run(env)
    vs1 = np.array(vs1)
    vs2 = np.array(vs2)

    with open('windy_values.npy', 'wb') as f:
        np.save(f, vs1)
        np.save(f, vs2)
        
    plt.figure()
    plt.imshow(vs1[-1,:,:].T)
    plt.figure()
    plt.imshow(vs1[-1,:,:].T)
    plt.figure()
    plt.plot(vs2[:,7,7])
    plt.plot(vs1[:,7,7])<jupyter_output>episode: 0
episode: 100
episode: 200
episode: 300
episode: 400
episode: 500
episode: 600
episode: 700
episode: 800
episode: 900
episode: 1000
episode: 1100
episode: 1200
episode: 1300
episode: 1400
episode: 1500
episode: 1600
episode: 1700
episode: 1800
episode: 1900
episode: 2000
episode: 2100
episode: 2200
episode: 2300
episode: 2400
episode: 2500
episode: 2600
episode: 2700
episode: 2800
episode: 2900
episode: 3000
episode: 3100
episode: 3200
episode: 3300
episode: 3400
episode: 3500
episode: 3600
episode: 3700
episode: 3800
episode: 3900
episode: 4000
episode: 4100
episode: 4200
episode: 4300
episode: 4400
episode: 4500
episode: 4600
episode: 4700
episode: 4800
episode: 4900
episode: 5000
episode: 5100
episode: 5200
episode: 5300
episode: 5400
episode: 5500
episode: 5600
episode: 5700
episode: 5800
episode: 5900
episode: 6000
episode: 6100
episode: 6200
episode: 6300
episode: 6400
episode: 6500
episode: 6600
episode: 6700
episode: 6800
episode: 6900
episode: 7000
episode: 7100
epis[...]
{'fraction_non_alphanumeric': 0.11636139756810836, 'fraction_numerical': 0.060797291057411114, 'mean_word_length': 1.7545570156846122, 'pattern_counts': {'":': 0, '<': 10, '<?xml version=': 0, '>': 6, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '629872', 'license': 'MIT', 'n_tokens_mistral': 2627, 'n_tokens_neox': 2194, 'n_words': 668, 'path': 'Chapter 7/ex_7_2/.ipynb_checkpoints/ex_7_2_gridworld-checkpoint.ipynb', 'repo_name': 'JChunX/rl-book-ans', 'size': '6497'}