<jupyter_start><jupyter_text>SARSAIn this notebook, we will implement SARSA algorithm to play Hill Climbing. Step 1: Import the libraries<jupyter_code>import time
import random
import gym
import numpy as np
import matplotlib.pyplot as plt
from collections import namedtuple, deque<jupyter_output><empty_output><jupyter_text>Step 2: Create our environment<jupyter_code>env = gym.make('MountainCar-v0')
env._max_episode_steps = 1000
env.seed(1)<jupyter_output>c:\users\deepanshu\.conda\envs\ai\lib\site-packages\gym\logger.py:30: UserWarning: [33mWARN: Box bound precision lowered by casting to float32[0m
  warnings.warn(colorize('%s: %s'%('WARN', msg % args), 'yellow'))<jupyter_text>Step 3: Viewing our EnviromentExecute the code cell below to play Cartpole with a random policy.<jupyter_code>print("The size of state is: ", env.observation_space.shape)
print("No. of Actions: ", env.action_space.n)
env.reset()
plt.figure()
plt.imshow(env.render(mode='rgb_array'))
env.close()
plt.title('Original Frame')
plt.show()
def random_play():
    env.reset()
    while True:
        env.render(mode='rgb_array')
        action = env.action_space.sample()
        state, reward, done, _ = env.step(action)
        if done:
            env.close()
            break
random_play()<jupyter_output><empty_output><jupyter_text>Step 4: Creating State Preprocess Function<jupyter_code>pos_space = np.linspace(-1.2, 0.6, 12)
vel_space = np.linspace(-0.07, 0.07, 20)

def get_state(state):
    pos, vel = state
    
    pos_bin = np.digitize(pos, pos_space)
    vel_bin = np.digitize(vel, vel_space)
    return (pos_bin, vel_bin)<jupyter_output><empty_output><jupyter_text>Step 6: Creating out agent<jupyter_code>GAMMA = 0.99            # discount factor
LR = .1                 # learning rate

class Agent():
    def __init__(self, p_size = 12, v_size = 20, action_size = 3):
        self.action_space = list(range(0, action_size))
        
        # Intilizing state
        self.states = []
        for p in range(p_size + 1):
            for v in range(v_size + 1):
                self.states.append((p, v))
        
        # Iitliazing Q
        self.Q = {}
        for s in self.states:
            for a in self.action_space:
                self.Q[s, a] = 0
        
                
    def act(self, state, eps=.0):
        """Returns actions for given state as per action value function."""
        state = get_state(state)
        values = np.array([self.Q[state, a] for a in self.action_space])
        action = np.argmax(values)
        g_action = self.eps_action(action, eps)
        return g_action
    
    def eps_action(self, action, eps):
        g_action = np.random.choice(self.action_space) if np.random.random() < eps else action 
        return g_action
    
    def step(self, s, a, ns, r):
        na = self.act(ns)
        s = get_state(s)
        ns = get_state(ns)
        self.Q[s, a] = self.Q[s, a] + LR * (r + (GAMMA * self.Q[ns, na]) - self.Q[s, a])
        return na<jupyter_output><empty_output><jupyter_text>Step 7: Watching untrained agent play<jupyter_code>agent = Agent(p_size=12, v_size=20, action_size= 3)

# watch an untrained agent
state = env.reset()
for j in range(200):
    action = agent.act(state, 0.5)
    env.render()
    next_state, reward, done, _ = env.step(action)
    state = next_state
    if done:
        break 
        
env.close()<jupyter_output><empty_output><jupyter_text>Step 8: Loading Agent<jupyter_code>start_epoch = 0
eps_start = 1.0
scores = []
scores_window = deque(maxlen=100)

# To Load checkpoint uncomment code
# checkpoint = np.load('mountain_climb_sarsa.npy', allow_pickle=True)
# agent.Q = checkpoint[()]['Q']
# scores = checkpoint[()]['scores']
# eps_start = checkpoint[()]['eps_start']
# start_epoch = len(scores)

# index = 1
# for i in reversed(scores):
#     scores_window.append(i)
#     if index == 100:
#         break
#     index += 1<jupyter_output><empty_output><jupyter_text>Step 9: Train the Agent with DQN<jupyter_code>def train(n_episodes=50000, eps_end=0.01):
    global eps_start
    eps = eps_start
    for i_episode in range(start_epoch+1, n_episodes+1):
        state = env.reset()
        score = 0
        action = agent.act(state, eps)
        while True:
            next_state, reward, done, _ = env.step(action)
            action = agent.step(state, action, next_state, reward)
            action = agent.eps_action(action, eps)
            state = next_state
            score += reward
            if done:
                break
        eps = eps - 2/n_episodes if eps > eps_end else eps_end # decrease epsilon
        scores_window.append(score)       # save most recent score
        scores.append(score)              # save most recent score
        eps_start = eps
        print('\rEpisode {}\tAverage Score: {:.2f}\tEpsilion: {:.2f}'.format(i_episode, np.mean(scores_window), eps), end="")
        if i_episode % 1000 == 0:
            print('\rEpisode {}\tAverage Score: {:.2f}\tEpsilion: {:.2f}'.format(i_episode, np.mean(scores_window), eps))
    
    return scores
scores = train()
# plot the scores
fig = plt.figure()
ax = fig.add_subplot(111)
plt.plot(np.arange(len(scores)), scores)
plt.ylabel('Score')
plt.xlabel('Episode #')
plt.show()
# watch an trained agent
state = env.reset()
score = 0
for j in range(200):
    env.render()
    action = agent.act(state)
    state, reward, done, _ = env.step(action)
    score += reward
    if done:
        print("Your total score is: ", score)
        break 
        
env.close()
data = {
    "Q": agent.Q,
    "scores": scores,
    "eps_start": eps_start
}
np.save("mountain_climb_sarsa.npy", data)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1271574498062698, 'fraction_numerical': 0.017259598450158505, 'mean_word_length': 2.9301038062283737, 'pattern_counts': {'":': 3, '<': 33, '<?xml version=': 0, '>': 33, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 3, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '8', 'dir': 'jupyter-structured-clean-dedup', 'id': '347658', 'license': 'MIT', 'n_tokens_mistral': 2079, 'n_tokens_neox': 1962, 'n_words': 465, 'path': 'notes/03_temporal_difference/mountain_climb_sarsa.ipynb', 'repo_name': 'deepanshut041/Reinforcement-Learning-Basic', 'size': '5678'}