<jupyter_start><jupyter_text>Implementation of the value iteration algorithmThis algorithm finds the optimal value function and policy given aknown model of the environment.![img](./imgs/value-iteration.png)From: Sutton and Barto, 2018. Ch. 4.<jupyter_code># first, import necessary modules
import sys
import gym
import random
import numpy as np
import seaborn as sns
import matplotlib.pyplot as plt

# add your own path to the RL repo here
sys.path.append('/Users/wingillis/dev/reinforcement-learning')

from collections import defaultdict
from lib.envs.gridworld import GridworldEnv
from lib.plotting import plot_gridworld_value_function
sns.set_style('white')
# initialize the environment
shape = (5, 5)  # size of the gridworld
env = GridworldEnv(shape, n_goals=2)
env.seed(23)
random.seed(23)
def value_iteration(env, gamma=1.0, theta=1e-5):
  '''
  Arguments:
    env: open.ai environment
    gamma: discount factor
    theta: convergence threshold
  '''
  
  def one_step_lookahead(state, V):
    '''calculate value of taking each action from this state'''
    pass
      
  
  # initialize value function
  
  # loop until the value function has converged
  while True:
    delta = 0  # amt of the largest state-value update
    
    # go through each state, use a one-step lookahead to
    # find the best action, and update each state's value
      
    if delta < theta:  # stop updating once some level of precision has been reached
      break
      
  # define the policy as the best choice from all potential actions
  
  return policy, V
policy, V = value_iteration(env, gamma=0.75)<jupyter_output><empty_output><jupyter_text>This is what your policy and value function should look likewith `gamma = 0.75`<jupyter_code>policy
V.reshape(shape)
fig = plot_gridworld_value_function(V.reshape(shape))
fig.tight_layout()
print("grid policy (0=up, 1=right, 2=down, 3=left):")
print(np.argmax(policy, axis=1).reshape(shape))<jupyter_output>grid policy (0=up, 1=right, 2=down, 3=left):
[[0 3 3 3 2]
 [0 0 0 0 2]
 [0 0 0 1 2]
 [0 0 1 1 2]
 [0 1 1 1 0]]
{'fraction_non_alphanumeric': 0.09130645944633317, 'fraction_numerical': 0.027683341427877612, 'mean_word_length': 4.175879396984925, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 8, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '513611', 'license': 'MIT', 'n_tokens_mistral': 709, 'n_tokens_neox': 655, 'n_words': 237, 'path': 'reinforcement learning/01-DP-value-iteration.ipynb', 'repo_name': 'john-vastola/ML-from-scratch-seminar', 'size': '2059'}