<jupyter_start><jupyter_text>Policy iteration  Parameters<jupyter_code>gamma = 1 # discounting rate
rewardSize = -1
gridSize = 4
terminationStates = [[0,0], [gridSize-1, gridSize-1]]
actions = [[-1, 0], [1, 0], [0, 1], [0, -1]]
numIterations = 1000
terminationStates<jupyter_output><empty_output><jupyter_text>Utilities<jupyter_code>def actionRewardFunction(initialPosition, action):
    
    if initialPosition in terminationStates:
        return initialPosition, 0
    
    reward = rewardSize
    finalPosition = np.array(initialPosition) + np.array(action)
    if -1 in finalPosition or 4 in finalPosition: 
        finalPosition = initialPosition
        
    return finalPosition, reward<jupyter_output><empty_output><jupyter_text>Initialization<jupyter_code>valueMap = np.zeros((gridSize, gridSize))
states = [[i, j] for i in range(gridSize) for j in range(gridSize)]
# values of the value function at step 0
valueMap
states<jupyter_output><empty_output><jupyter_text>Policy evaluation<jupyter_code>deltas = []
for it in range(numIterations):
    copyValueMap = np.copy(valueMap)
    deltaState = []
    for state in states:
        weightedRewards = 0
        for action in actions:
            finalPosition, reward = actionRewardFunction(state, action)
            weightedRewards += (1/len(actions))*(reward+(gamma*valueMap[finalPosition[0], finalPosition[1]]))
        deltaState.append(np.abs(copyValueMap[state[0], state[1]]-weightedRewards))
        copyValueMap[state[0], state[1]] = weightedRewards
    deltas.append(deltaState)
    valueMap = copyValueMap
    if it in [0,1,2,9, 99, numIterations-1]:
        print("Iteration {}".format(it+1))
        print(valueMap)
        print("")
        
plt.figure(figsize=(20, 10))
plt.plot(deltas)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.11850195640022358, 'fraction_numerical': 0.024035774175517048, 'mean_word_length': 4.343283582089552, 'pattern_counts': {'":': 0, '<': 17, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '85985', 'license': 'MIT', 'n_tokens_mistral': 610, 'n_tokens_neox': 578, 'n_words': 122, 'path': 'content/homeworks/hw08/notebook/policy_iterator_RL_gridworld.ipynb', 'repo_name': 'gurdeep101/2020-CS109B', 'size': '1789'}