<jupyter_start><jupyter_text>Accessing Problem<jupyter_code>!pip3 install pygame
import os
os.environ['SDL_VIDEODRIVER']='dummy'
import pygame
pygame.display.set_mode((640,480))
!unzip images.zip
!unzip modules.zip
from gridworld import GridWorld
import numpy as np
world=\
    """
    wwwwwwwwww
    wa       w
    w    wwwww
    wwww     w
    w        w
    w        w
    w        w
    w g  wwwww
    w        w
    wwwwwwwwww
    """

#### World Declaration
env=GridWorld(world)

env._max_epi_step=200     ## Number of maximum steps in an episode
import matplotlib.pyplot as plt
import cv2
img=cv2.imread("problem_window.png")
plt.figure(figsize = (10,10))
plt.imshow(img)
plt.show()<jupyter_output><empty_output><jupyter_text>Q Learning<jupyter_code>import numpy as np
import tqdm
def generate_sample(policy):

  episodes=[]
  curr_state=env.reset()  ### Returns a starting state.
  

  while True:
    action=policy[curr_state]
    next_state,reward,done,_=env.step(action)  ### Returns from environment on taking an action
    yield(curr_state,action,reward,next_state)  ### Throws this tuple, works like a generator

    if done== True:
      curr_state=env.reset()  ### Returns a starting state
    else:
      curr_state=next_state ### updating state

    
### As in case of Q-leaning episodic setting is not required, so we just need to throw the tuples to
## Update the value functions accordingly. The action is taken according to the policy.

def get_e(episode,steady_epsilon=0.01,steady_episode=1000):
  ### Obtaining the epsilon for the episode.
  ### The steady epsilon, we settle on is 0.01 after episode 1000
  ### Till 1000, epsilon decays linearly.

  if episode>steady_episode:  ##If we are above the steady episode, we return a steady epsilon
    return steady_epsilon
  else:
    slope=(steady_epsilon - 1)/(steady_episode - 0) 
    ### Line (1,0) to (steady_epsilon,steady_episode)

    ### slope*episode will give us the decrease in the value of epsilon
    ### To get the value we add 1 to the value so it is (1 - decrease), as epsilon starts from 1.
    return slope*episode + 1
def obtain_action_greed(Q_function, state, epsilon, actions):
  ### Obtaining policy based on the current Q function values, and epsilon value.
  ### This method is called E-greedy policy.

  ### "state" represents the state for which the policy is being updated

  optimal_action=np.argmax(Q_function[state])  ### finding the action for the state that
  ## provides the maximum Q-value

  if np.random.random()<=1-epsilon:  ### Random.random() generates number in (0,1) 
  ###probability 1/(b-a). where b and a are upper and lower limits
    chosen_action=optimal_action
    #### Choosing the best action according to policy. with probability (1-epsilon)
  else:
    chosen_action=np.random.choice(actions)
    ### else choosing random action with probability epsilon/|actions| for each.
  
  return chosen_action ### The action for the given state
def get_policy(gamma=0.99,alpha=0.01):
  Q=np.zeros((env.state_count,len(env.action_space)))

  ### initializing the Q function to record the details of the actions on a particular state.
  epsilon=1 ## Initially exploration control value 1: maximum
  policy=np.random.choice(env.action_space,size=env.state_count)
  ### Initialized a complete random policy, an action for each state.
  
  generator=generate_sample(policy) ### initializing a generator.
  for tuple_number in tqdm.tqdm(range(100000)):  
    ### We train on a batch of 100000 collected episodes.
    s,a,r,s_= next(generator)### Generating the next sample

    Q[s,a]=Q[s,a]+alpha*((r + gamma*np.max(Q[s_]))-Q[s,a]) ### Bootstrapping type update of Q function 
    ### using the max state-action value for the next state s'.
    
    epsilon=get_e(tuple_number) ### Getting the epsilon for the update.
    policy[s]=obtain_action_greed(Q,s,epsilon,env.action_space)
    ### Updating the policy for the state under consideration.
  return policy
policy=get_policy()
policy
env.setPolicy(policy)
img=cv2.imread("Route_q.png")
plt.figure(figsize = (10,10))
plt.imshow(img)
plt.show()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.10326481257557436, 'fraction_numerical': 0.017412333736396616, 'mean_word_length': 4.013333333333334, 'pattern_counts': {'":': 0, '<': 10, '<?xml version=': 0, '>': 10, 'https://': 0, 'lorem ipsum': 0, 'www.': 7, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 2, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '438877', 'license': 'MIT', 'n_tokens_mistral': 1377, 'n_tokens_neox': 1303, 'n_words': 420, 'path': 'Model Free Policy Control/Table_based_Q_learning.ipynb', 'repo_name': 'abr-98/Reinforcement-Learning', 'size': '4135'}