<jupyter_start><jupyter_text>Generate Tuples<jupyter_code>env = SepsisEnv()
fg_tuples = env.generate_tuples(group="foreground", n_trajectories=10)
bg_tuples = env.generate_tuples(group="background", n_trajectories=10)
all_tuples = bg_tuples + fg_tuples
random.shuffle(all_tuples)
split = 0.8
train_tuples = all_tuples[:int(split*len(all_tuples))]
test_tuples = all_tuples[int(split*len(all_tuples)):]
sns.distplot([x[3] for x in all_tuples])<jupyter_output>/Users/aishwaryamandyam/anaconda3/envs/research/lib/python3.7/site-packages/seaborn/distributions.py:2557: FutureWarning: `distplot` is a deprecated function and will be removed in a future version. Please adapt your code to use either `displot` (a figure-level function with similar flexibility) or `histplot` (an axes-level function for histograms).
  warnings.warn(msg, FutureWarning)<jupyter_text>Train Agents<jupyter_code>lmm_agent = LMMFQIagent(train_tuples=train_tuples, test_tuples=test_tuples, estimator='linnet', gamma=0.5, state_dim=46, batch_size=len(train_tuples), iters=1000)
# The two policies are in lmm_agent.piE_foreground and lmm_agent.piE_background
Q_dist = lmm_agent.runFQI(repeats=1)
plt.plot(Q_dist, label= "LMMFQI")
plt.xlabel("Iteration")
plt.ylabel("Q Estimate")
plt.legend()
plt.show()
lmm_agent.Qtable
fqi_agent = FQIagent(train_tuples=train_tuples, test_tuples=test_tuples, state_dim=46, gamma=0.5, batch_size=len(train_tuples), iters=1000, estimator='gbm')
Q_dist = fqi_agent.runFQI(repeats=1)
plt.plot(Q_dist, label= "FQI")
plt.xlabel("Iteration")
plt.ylabel("Q Estimate")
plt.legend()
plt.show()<jupyter_output><empty_output><jupyter_text>Validation<jupyter_code>def validate_agent(sepsis_env, ds='foreground', plot='action'):
        algos = ['fqi', 'lmmfqi', 'oracle', 'random']
        val_rewards = {}
        alg_actions = {}
        for alg in algos:
            val_rewards[alg] = []
            alg_actions[alg] = []
        
        for alg in algos:
            print("Alg: ", alg)
            pend.reset()
            
            for i in range(100):
                s = pend._get_obs().reshape((1, -1))
#                 print(pend.state)
                if alg == 'fqi':
                    # FQI agent\n",
                    fqi_action = fqi_agent.piE.predict(s)
                    #fqi_action = np.rint(fqi_action)
                    alg_actions['fqi'].append(fqi_action[0])
                    ns, cost, _, _ = pend.step(fqi_action)
                    val_rewards['fqi'].append(cost/10)
                    s = ns
                elif alg == 'lmmfqi':
                    # LMMFQI agent
                    if ds == 'foreground':
                        group = [1]
                        lmmfqi_action = lmm_agent.piE_foreground.predict(s, group)
                    else:
                        group = [0]
                        lmmfqi_action = lmm_agent.piE_background.predict(s, group)
                    alg_actions['lmmfqi'].append(lmmfqi_action[0])
                    ns, cost, _, _ = pend.step(lmmfqi_action)
                    val_rewards['lmmfqi'].append(cost/10)
                elif alg == 'random':
                   # Random action
                    random_action = pend.action_space.sample()
                    random_action = np.rint(random_action)
                    alg_actions['random'].append(random_action[0])
                    ns, cost, _, _ = pend.step(random_action)
                    val_rewards['random'].append(cost)
                elif alg == 'oracle':
                    # Oracle
                    best_reward = None
                    best_ns = None
                    best_action = None
                    actions = [-2, -1, 0, 1, 2]
                    for j, a in enumerate(actions):
                        a = np.asarray([a])
                        ns, cost, _, _ = pend.step(a)
                        if best_reward is None or cost > best_reward:
                            best_reward = cost
                            best_ns = ns
                            best_action = a
                    alg_actions['oracle'].append(best_action[0])
                    val_rewards['oracle'].append(best_reward)
                    ns = best_ns
                else:
                    raise Exception("Invalid algorithm selected")

        #plt.title(\"Cumulative Reward for ds: \" + str(ds))\n",
        plt.xlabel("Step")
        
        x = [i for i in range(100)]
        rewards_fqi = util_fqi.cumulative_reward(val_rewards['fqi'])
        rewards_lmmfqi = util_fqi.cumulative_reward(val_rewards['lmmfqi'])
        rewards_oracle = util_fqi.cumulative_reward(val_rewards['oracle'])
        rewards_random = util_fqi.cumulative_reward(val_rewards['random'])
        if plot == 'reward':
            plt.plot(x, rewards_fqi, label="FQI", alpha=0.7)
            plt.plot(x, rewards_lmmfqi, label='LMMFQI', alpha=0.7)
#             plt.plot(x, rewards_oracle, label='Oracle')
#             plt.plot(x, rewards_random, label='Random', alpha=0.7)
            plt.ylabel("Cumulative Reward")
        else:
            plt.plot(x, alg_actions['fqi'], label="FQI", alpha=0.7)
            plt.plot(x, alg_actions['lmmfqi'], label='LMMFQI', alpha=0.7)
#             plt.plot(x, alg_actions['oracle'], label='Oracle')
#             plt.plot(x, alg_actions['random'], label='Random', alpha=0.7)
            plt.ylabel("Action")
        plt.legend()
        plt.show()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.12128984976181752, 'fraction_numerical': 0.012275558812751924, 'mean_word_length': 2.1017045454545453, 'pattern_counts': {'":': 0, '<': 12, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '385566', 'license': 'MIT', 'n_tokens_mistral': 1885, 'n_tokens_neox': 1733, 'n_words': 298, 'path': 'simulated_fqi/notebooks/sepsis_experiments.ipynb', 'repo_name': 'bee-hive/nested-policy-rl', 'size': '5458'}