<jupyter_start><jupyter_text>1-vs-(N-1) Flu ABM Env- 1-vs-(N-1) experiment- Basic AC learner- For comparison with MARL outcomes<jupyter_code>import itertools, importlib, sys, warnings, os

import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import seaborn as sns

# ML libs
import tensorflow as tf
print("Tensorflow version:", tf.__version__)
print("Pandas version:", pd.__version__)

log_path = './log/flu'
#tensorboard --logdir=flugame_worker_1:'./log/<<..>>'

## suppress annoy verbose tf msgs
warnings.filterwarnings("ignore")
os.environ['TF_CPP_MIN_LOG_LEVEL'] = '2' # '3' to block all including error msgs
tf.compat.v1.logging.set_verbosity(tf.compat.v1.logging.ERROR)
# plt.rcParams['figure.figsize'] = (18,7)
sys.path.append('./embodied_arch')

import embodied_AC as emg
import flumodel_python.flu_env as Fenv
from embodied_misc import ActionPolicyNetwork, ValueNetwork, SensoriumNetworkTemplate

importlib.reload(emg)<jupyter_output><empty_output><jupyter_text>Env Setup - Single Agent<jupyter_code>importlib.reload(Fenv);
# ?Fenv.Flu_env
exos = np.random.choice(9223)
exos
tf.reset_default_graph()
flu_env = Fenv.Flu_env(
    exo_idx=exos,
    model_path="./flumodel_python/"
)<jupyter_output><empty_output><jupyter_text>SARL Setup Demo<jupyter_code>actor = lambda s: ActionPolicyNetwork(s, hSeq=(8,), gamma_reg=1e-1)
value = lambda s: ValueNetwork(s, hSeq=(8,), gamma_reg=1.)
sensor = lambda st, out_dim: SensoriumNetworkTemplate(st, hSeq=(16,8,8), out_dim=out_dim, gamma_reg=5.)

num_episodes, max_len, n_epochs, evry = (100, 35, 751, 150)
# num_episodes, n_epochs, max_len = (30, 400, 25)
importlib.reload(emg);

flurf = emg.EmbodiedAgentAC(
    name="fluAC", env_=flu_env, 
    _every_=evry, max_episode_length=max_len, 
    latentDim=4, alpha_p=10., alpha_v=5., 
    actorNN=actor, sensorium=sensor, valueNN=value
)

print(flurf, flurf.s_size, flurf.a_size)
print(flu_env.state_space_size, flu_env.action_space_size)<jupyter_output>8 1<jupyter_text>Burn-in Flu ABM to Equilibrium<jupyter_code>burn_in = 451
flurf.env.soft_reset(DBM_reset=True)
flurf.env.infection_rate_history
for _ in range(burn_in):
    _ = flurf.env.stepAll()


plt.plot(pd.Series(flurf.env.infection_rate_history).rolling(window=50).mean().iloc[50-1:].values);

print(
    np.mean(np.array(flurf.env.infection_rate_history)[:flurf.max_episode_length]),
    np.mean(np.array(flurf.env.infection_rate_history)[-flurf.max_episode_length:])
)

## no equilibrium, just cycling...<jupyter_output>0.07598395316057682 0.05665959325289261<jupyter_text>Baseline<jupyter_code>sess = tf.InteractiveSession()
flurf.init_graph(sess) # note tboard log dir
print('Baselining untrained pnet...')
rwds0 = []
for k in range(num_episodes):
    flurf.play(sess, terminal_reward=0.);
    rwds0.append(flurf.last_total_return)
    print("\rEpisode {}/{}".format(k, num_episodes),end="")

# Compute average rewards
base_perf = 100.*np.mean(np.array(rwds0)/float(flurf.max_episode_length))

print("\nAgent is flu-free for an average of {}pct of seasons".format(1.*base_perf))<jupyter_output>Baselining untrained pnet...
Episode 99/100
Agent is flu-free for an average of 94.71428571428572pct of seasons<jupyter_text>Train<jupyter_code>obs = []
for ct in range(40):
    flurf.play(sess)
    tmp = flurf.pretrainV(sess)
    obs.append(tmp)
    print('\r\tIteration {}: Value loss({})'.format(ct, tmp), end="")
plt.plot(obs)
# Train pnet on cartpole episodes
print('Training...')
saver = tf.train.Saver(max_to_keep=1)
hist = flurf.work(sess, saver, num_epochs=n_epochs);
sns.violinplot(hist)<jupyter_output><empty_output><jupyter_text>Test<jupyter_code>print('Testing...')
rwds = []
for k in range(num_episodes):
    flurf.play(sess, terminal_reward=0.);
    rwds.append(flurf.last_total_return)
    print("\rEpisode {}/{}".format(k, num_episodes),end="")

# Compute average rewards
trained_perf = 100.*np.mean(np.array(rwds)/float(flurf.max_episode_length))
print("\nAgent is flu-free for an average of {}pct of seasons compared to baseline {}pct".format(
    trained_perf, base_perf) )<jupyter_output>Testing...
Episode 99/100
Agent is flu-free for an average of 97.08571428571429pct of seasons compared to baseline 94.71428571428572pct<jupyter_text>Evaluate<jupyter_code>fig, axs = plt.subplots(2, 1, sharex=True)
sns.violinplot(rwds0, ax = axs[0])
axs[0].set_title('Baseline Episode Rewards')
sns.violinplot(rwds, ax = axs[1])
axs[1].set_title('Trained Episode Rewards')
100.*(np.array(rwds)-np.array(rwds0))/float(flurf.max_episode_length)
cmp = sns.boxplot(100.*(np.array(rwds)-np.array(rwds0))/float(flurf.max_episode_length))
cmp.set_title('Distribution of Agent\'s Improvement in Seasonal Immunity');<jupyter_output><empty_output><jupyter_text>Baseline for Default Behavioral ModelUsing internal Env instead of new Flu Env<jupyter_code># import flumodel_python.flu_env_basic as FABM
# # ?Fenv.Flu_ABM
# importlib.reload(FABM);
# flu = FABM.Flu_ABM(model_path="./flumodel_python/")
# No Burn-in Flu ABM... No DBM reset.
# note change in comparison standard: against average background popn, not against itself under DBM
flu_hist = np.zeros(num_episodes)
for k in range(num_episodes):
    for _ in range(flurf.max_episode_length):
        _ = flurf.env.stepAll() 
    flu_hist[k] = np.mean(np.array(flurf.env.infection_rate_history)[-flurf.max_episode_length:])<jupyter_output><empty_output><jupyter_text>Evaluate RL Agent Against Default Behavioral Model<jupyter_code>rwds_dbm = 100.*(1.-flu_hist)
rwds_init = 100.*np.array(rwds0)/float(flurf.max_episode_length)
rwds_rl = 100.*np.array(rwds)/float(flurf.max_episode_length)
# print(rwds_dbm.shape, rwds_rl.shape)
print(np.mean(rwds_dbm), np.mean(rwds_init), np.mean(rwds_rl))
# print(np.mean(rwds_rl - rwds_dbm))

# plt.rcParams['figure.figsize'] = (8,3)
# cmp_dbm = sns.boxplot(rwds_rl-rwds_dbm)
# cmp_dbm.set_title(
#     'Average Difference in Agent Performance (Seasonal Non-Infection Pct):\nRL Agent over Default Behavioral Model ({:0.2f} pct)'.format(
#     np.mean(rwds_rl - rwds_dbm)));
plt.rcParams['figure.figsize'] = (9,7)
fig, axs = plt.subplots(2, 1, sharex=True)

fig.suptitle("Comparing Avg. Agent Performance (Seasonal Non-Infection Pct)", fontsize=16)

sns.boxplot(rwds_rl-rwds_dbm, ax = axs[0])
axs[0].set_title('Trained RL Agent vs. Default Behavioral Model (+{:0.2f} pct)'.format(
    np.mean(rwds_rl - rwds_dbm)));
# axs[0].set_xlabel("Percentage Improvment");
axs[0].tick_params(labelbottom=True)
# plt.setp(axs[0].get_xticklabels(), visible=True)

sns.boxplot(rwds_rl-rwds_init, ax = axs[1])
axs[1].set_title('RL Agent After vs. Before Training (+{:0.2f} pct)'.format(
    np.mean(rwds_rl - rwds_init)));
axs[1].set_xlabel("Percentage Improvment");<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.14427786848745733, 'fraction_numerical': 0.03191331453169066, 'mean_word_length': 6.191035218783351, 'pattern_counts': {'":': 0, '<': 39, '<?xml version=': 0, '>': 39, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 6, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '10', 'dir': 'jupyter-structured-clean-dedup', 'id': '373937', 'license': 'MIT', 'n_tokens_mistral': 2863, 'n_tokens_neox': 2684, 'n_words': 464, 'path': 'successful_runs/comparisons-to-default/flu-SARL-expt.ipynb', 'repo_name': 'oosoba/rl-policy-abms', 'size': '6737'}