<jupyter_start><jupyter_text>Define the EnvironmentA server requires information from a sensor. The server would like the information to be fresh. However, there is a cost to querying information from the sensor.     The state at the server canbe either fresh or stale. The former indicates that the information at the server about the sensor is fresh and the latter indicates that it is stale.     At any time, the server may choose to query or remain silent.    MDP given in the diagrma<jupyter_code>class ServerProb():
    def __init__(self):
        '''
        Initialize the Gridword
        self.state (list) : How many non-termimal state does this GridWordHave
        self.statePlus : Non terminal + terminal state
        self.posibleStateAction : Possible action for each state
        self.dynamic : Dynamics of this GridWorld
        '''
        self.state = np.arange(1,3)
        self.statePlus = np.arange(1,3)
        self.posibleStateAction = self.defPossibleStateAction()
        self.dynamic = self.defineDynamic() 
        
    def defPossibleStateAction(self):
        #defining Possible action for this ServerProb
        posibleStateAction = dict()
        for s in self.state:
            posibleStateAction[s] = [1,2]  #query,silent
        return posibleStateAction
    
    def defineDynamic(self):
        #defining the dynamics of the Environmvent
        dynamic = dict()
        dynamic[(1,1)] = [(1,-4,0.9),(2,-4,.1)]
        dynamic[(1,2)] = [(1,4,.5),(2,4,.5)]
        dynamic[(2,1)] = [(1,-8,0.8),(2,-8,.2)]
        dynamic[(2,2)] = [(2,4,1)]
        return dynamic<jupyter_output><empty_output><jupyter_text>Value Iteration![image.png](attachment:image.png)<jupyter_code>def valuesIteration(env,v,gamma = 0.5,threshold=0.01):
    '''
    This is the implementation of Policy Iteration
    '''
    hist = []
    while True:
        delta = 0
        for s in env.state:
            vtemp = 0
            actionReward = []
            for action in env.posibleStateAction[s]:
                actTemp = 0
                for sDash,reward,prob in env.dynamic[(s,action)]:
                    #print(prob,reward, v[sDash-1] )
                    actTemp += prob * (reward + gamma * v[sDash-1])
                actionReward.append(actTemp)
            vtemp = max(actionReward)
            delta = max(delta,abs(vtemp-v[s-1]))
            v[s-1] = vtemp
        #print(v,delta)
        hist.append(cp.deepcopy(v))
        if (delta < threshold):
            break;
    return hist
def oneStepLookAhead(s,env,v,gamma = 1):
    '''
    This method is use to do a one step lookahead.
    It is used when when we when we were imporivng the policy Step 3
    '''
    actionReward = []
    for action in env.posibleStateAction[s]:
        act = 0
        for sDash,reward,prob in env.dynamic[(s,action)]:
            act += prob * (reward + gamma * v[sDash-1])
        actionReward.append(act)
    return actionReward
def findPolicy(env,v):
    newPolicy = dict()
    for s in env.state:
        #get the values from each of values function
        actionRewards = oneStepLookAhead(s,env,v)
        #placing 1 which has the hightest values
        maxReward = [1 if i == max(actionRewards) else 0 for i in actionRewards ]
        #finding the probability
        maxReward = maxReward/np.sum(maxReward)
        #making the new Policy
        newPolicy[s] = [(i+1,j) for i,j in enumerate(maxReward) ]
    return newPolicy
def printPolicy(policy):
    print("state \t [query,\t       silent]")
    for key in policy:
        print(key ,"\t", policy[key])
env = ServerProb() 
v = np.zeros(len(env.statePlus))
hist = valuesIteration(env,v)
policy = findPolicy(env,hist[-1])
print("Final Policy")
printPolicy(policy)
hist<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.10766374966852293, 'fraction_numerical': 0.015380535666931849, 'mean_word_length': 2.4990723562152133, 'pattern_counts': {'":': 0, '<': 10, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '627651', 'license': 'MIT', 'n_tokens_mistral': 1188, 'n_tokens_neox': 1143, 'n_words': 334, 'path': 'HW2/Code/Question13_ValuesIteration.ipynb', 'repo_name': 'ppthegp/Reinforcement-Learning-RL--M2020', 'size': '3771'}