<jupyter_start><jupyter_text>--- Helper functions Reward and reward per episode<jupyter_code>def var(N,episodes,epsilon,alpha,gamma):
    M = np.zeros((N,episodes))
    Mre=np.zeros((N,episodes))
    for n in range(N):
        _,R,_,_,_,Rf,_ = FL_play(episodes=episodes,epsilon=epsilon,alpha=alpha,gamma=gamma)
        M[n] = Rf
        Mre[n] = R
    return np.mean(M,axis=0),np.mean(Mre,axis=0)
def plot_average_rewards_per_episode(N,episodes,epsilon,alpha,gamma):
    M,Mre = var(N,episodes=episodes,epsilon=epsilon,alpha=alpha,gamma=gamma)
    Mr,Mrer = var(N,episodes=episodes,epsilon=1.0,alpha=alpha,gamma=gamma)
    plt.figure(figsize=(15,8))
    plt.plot(range(M.shape[0]),np.cumsum(M),label="$\epsilon$=0.1, std:{:.3f}, lr={:.1f}, $\gamma$={:.1f}".format(np.std(Mre),alpha,gamma))
    plt.plot(range(Mr.shape[0]),np.cumsum(Mr),label="$\epsilon$=1.0, std:{:.3f}, lr={:.1f}, $\gamma$={:.1f}".format(np.std(Mrer),alpha,gamma))
    plt.title('Learning curve - Max R: {:.0f}'.format(sum(Mre)),fontsize=16)
    plt.xlabel('Episodes',fontsize=16)
    plt.ylabel('Average Reward per episode',fontsize=16)
    plt.legend(loc=0,fontsize=16)
    plt.show()
def plot_average_rewards(N,episodes,epsilon,alpha,gamma):
    M,Mre = var(N,episodes=episodes,epsilon=epsilon,alpha=alpha,gamma=gamma)
    Mr,Mrer = var(N,episodes=episodes,epsilon=1.0,alpha=alpha,gamma=gamma)
    plt.figure(figsize=(15,8))
    plt.plot(range(M.shape[0]),np.cumsum(Mre),label="$\epsilon$=0.1, std:{:.3f}, lr={:.1f}, $\gamma$={:.1f}".format(np.std(Mre),alpha,gamma))
    plt.plot(range(Mr.shape[0]),np.cumsum(Mrer),label="$\epsilon$=1.0, std:{:.3f}, lr={:.1f}, $\gamma$={:.1f}".format(np.std(Mrer),alpha,gamma))
    plt.title('Learning Gain - Max R: {:.0f}'.format(sum(Mre)),fontsize=16)
    plt.xlabel('Episodes',fontsize=16)
    plt.ylabel('Average Reward',fontsize=16)
    plt.legend(loc=0,fontsize=16)
    plt.show()<jupyter_output><empty_output><jupyter_text>Q-Network<jupyter_code>def var_q(N,episodes,epsilon,alpha,gamma):
    M = np.zeros((env.observation_space.n, env.action_space.n))
    for n in range(N):
        Q,_,_,_,_,_,_ = FL_play(episodes=episodes,epsilon=epsilon,alpha=alpha,gamma=gamma)
        M += Q
    return M / N
def var_qnetwork(N,episodes,epsilon,alpha,gamma):
    Q = var_q(N,episodes=episodes,epsilon=epsilon,alpha=alpha,gamma=gamma)
    plt.figure(figsize=(20,5))
    sns.heatmap(Q.T,annot=Q.T,linewidths=.75,cmap="YlGnBu",square=True, fmt=".2f")
    plt.xlabel('States',fontsize=16)
    plt.ylabel('Actions',fontsize=16)
    plt.plot('Q-network value')
    plt.yticks((0,1,2,3),('Left','Down','Right','Up'),rotation=0,fontsize=16)
    plt.show()<jupyter_output><empty_output><jupyter_text>--- Gym - Frozen Lake<jupyter_code>env = gym.make('FrozenLake-v0')<jupyter_output><empty_output><jupyter_text>SARSA<jupyter_code>def policy(Q,state,epsilon):
    if np.random.rand() < epsilon:
        action = np.random.choice([0,1])
    else:
        action = np.argmax(Q[state,:])
    return action
def FL_play(episodes,epsilon,alpha,gamma):
    Q = np.random.rand(env.observation_space.n, env.action_space.n)*0.0001
    R = []
    Rf = []
    for i in range(episodes):
        state = env.reset()
        r = 0
        action = policy(Q,state,epsilon)
        while True:
            next_state,reward,done,_ = env.step(action)
            next_action = policy(Q,next_state,epsilon)
            r+=reward
            if done:
                Q[state,action] = (1-alpha)*Q[state,action] + alpha*reward
                break
            else:
                Q[state,action] = (1-alpha)*Q[state,action] + alpha*(reward +gamma*Q[next_state,next_action]) 

            state = next_state
            action = next_action
            
        R.append(r)
        Rf.append(r/(i+1))

    return Q,R,epsilon,alpha,gamma,Rf,L
%%time
plot_average_rewards(N=50,episodes=150,epsilon=0.1,alpha=0.9,gamma=0.9)
%%time
plot_average_rewards(N=50,episodes=1500,epsilon=0.1,alpha=0.9,gamma=0.9)
%%time
plot_average_rewards(N=5,episodes=10000,epsilon=0.1,alpha=0.9,gamma=0.1)
%%time
plot_average_rewards_per_episode(N=10,episodes=10000,epsilon=0.1,alpha=0.9,gamma=0.9)
%%time
plot_average_rewards_per_episode(N=10,episodes=10000,epsilon=0.05,alpha=0.9,gamma=0.9)
%%time
var_qnetwork(N=5,episodes=20000,epsilon=0.1,alpha=0.9,gamma=0.95)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1911561492399816, 'fraction_numerical': 0.036849378166743434, 'mean_word_length': 4.66232073011734, 'pattern_counts': {'":': 0, '<': 18, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '488586', 'license': 'MIT', 'n_tokens_mistral': 1860, 'n_tokens_neox': 1765, 'n_words': 174, 'path': 'TemporalDifference/SARSA/SARSA-FrozenLake.ipynb', 'repo_name': 'onimaru/Reinforcement_Learning', 'size': '4342'}