<jupyter_start><jupyter_text>Q Learning, TD(0) in R**Author:** <NAME>  Environment<jupyter_code>library(gym)
#install.packages("MDPtoolbox")
library(MDPtoolbox)
#remote_base "http://127.0.0.1:5000"
rl_env_setup_func <- function(remote_base, env_name) {
#start server in terminal with $python gym_http_server.py
    remote_base <<- "http://127.0.0.1:5000"
    client <<- create_GymClient(remote_base)
    #print(client)
    #create instance
    instance_id <<- env_create(client, env_name)
    #print(instance_id)
    #list the environments
    list_envs <<- env_list_all(client)
    #print(list_envs)
    #set up your agent
    action_space_info <- env_action_space_info(client, instance_id)
    #print(action_space_info)
    agent <<- random_discrete_agent(action_space_info)
    agent <<- random_discrete_agent(action_space_info[["n"]])
    }
q_learning_func <- function(S , r1, r2, p) {
    param <- mdp_example_forest(S = 3, r1 = 2, r2 = 0, p = 0.1)
    P <<- param$P
    R<<- param$R
    print(P)
    print(R)
    q_learn = mdp_Q_learning(param$P, param$R, 0.9)
    policy <<- q_learn$policy
    print(policy)
    Q <<- q_learn$Q
    print(Q)
}
td0_func <- function(P, R, discount_factor = 0.9, policy) {
    td_0 = mdp_eval_policy_TD_0(P, R, 0.9, policy)
    td_0_V <<- td_0$V
    print(td_0_V)
    mean_disc <<- td_0$mean_discrepancy
    print(mean_disc)
}
#run env_setup_func
rl_env_setup_func("http://127.0.0.1:5000", "Pendulum-v0")


rl_models <- list(
    q_learning_func(S = 3, r1 = 0, r2 = 1, p = 0.1),
    td0_func(P, R, 0.9, policy)
 )<jupyter_output>, , 1

     [,1] [,2] [,3]
[1,]  0.1  0.9  0.0
[2,]  0.1  0.0  0.9
[3,]  0.1  0.0  0.9

, , 2

     [,1] [,2] [,3]
[1,]    1    0    0
[2,]    1    0    0
[3,]    1    0    0

     R1 R2
[1,]  0  0
[2,]  0  1
[3,]  2  0
[1] 1 1 1
         [,1]      [,2]
[1,] 13.02927  8.221391
[2,] 14.64401  9.388445
[3,] 16.76295 11.494760
[1] 12.98714 14.61173 16.55159
  [1]          NA 0.141147853 0.113129801 0.078934025 0.058106908 0.047625758
  [7] 0.046367676 0.037507267 0.029640785 0.032382333 0.035765126 0.029625023
 [13] 0.025508884 0.022817425 0.018618499 0.020898929 0.022704944 0.021064318
 [19] 0.017647353 0.019070779 0.017218804 0.015438812 0.016975246 0.014726928
 [25] 0.014182654 0.016886715 0.014086794 0.013779498 0.014588998 0.009402746
 [31] 0.015363636 0.013146793 0.012085830 0.012530021 0.012207213 0.010452342
 [37] 0.010245388 0.010897563 0.010757305 0.008067171 0.010906684 0.011431592
 [43] 0.008291524 0.009729519 0.009407433 0.008739813 0.010427022 0.009484296
 [49] 0.008063075 0[...]
{'fraction_non_alphanumeric': 0.15130299494360172, 'fraction_numerical': 0.26721120186697783, 'mean_word_length': 3.1821138211382114, 'pattern_counts': {'":': 0, '<': 35, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 3, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '555843', 'license': 'MIT', 'n_tokens_mistral': 1590, 'n_tokens_neox': 1192, 'n_words': 237, 'path': 'Section 5/Q Learning, TD(0) in R.ipynb', 'repo_name': 'PacktPublishing/Practical-Reinforcement-Learning-Agents-and-Environments', 'size': '2582'}