<jupyter_start><jupyter_text>k-vs-(N-k) Flu ABM EnvKicking tires on multiplayer instance of Flu ABM with RL learners<jupyter_code>import itertools, importlib, sys, warnings
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import seaborn as sns

# ML libs
import tensorflow as tf
print("Tensorflow version:", tf.__version__)
# warnings.filterwarnings("ignore")
# %matplotlib inline
# plt.rcParams['figure.figsize'] = (18,7)

log_path = './log/flu'
#tensorboard --logdir=flugame_worker_1:'./log/train_rf_flugame_worker' 
sys.path.append('./embodied_arch')

import embodied as emg
import embodied_indep as emi 
import flumodel_python.flu_env as Fenv
from embodied_misc import ActionPolicyNetwork, SensoriumNetworkTemplate<jupyter_output>/miniconda3/lib/python3.6/site-packages/gym/envs/registration.py:14: PkgResourcesDeprecationWarning: Parameters to load are deprecated.  Call .resolve and .require separately.
  result = entry_point.load(False)
/Users/oosoba/Documents/RAND/Coding/multiagent-ABM/flumodel_python/flu_env_basic.py:43: FutureWarning: from_items is deprecated. Please use DataFrame.from_dict(dict(items), ...) instead. DataFrame.from_dict(OrderedDict(items)) may be used to preserve the key order.
  columns=['HCW', 'adaptive', 'vacc.attitude']
/Users/oosoba/Documents/RAND/Coding/multiagent-ABM/flumodel_python/flu_env.py:20: FutureWarning: from_items is deprecated. Please use DataFrame.from_dict(dict(items), ...) instead. DataFrame.from_dict(OrderedDict(items)) may be used to preserve the key order.
  columns=['HCW', 'adaptive', 'vacc.attitude']<jupyter_text>Env Setup<jupyter_code>exos = [1,2,3,10]  # (np.random.sample(33) < 0.3)

importlib.reload(Fenv);
importlib.reload(emg);
importlib.reload(emi);

# ?Fenv.Flu_env
tf.reset_default_graph()
flu_menv = Fenv.Flu_env(
    exo_idx=exos,
    model_path="./flumodel_python/"
)
print(flu_menv.actor_count)
print(flu_menv.state_space_size, flu_menv.action_space_size)<jupyter_output>4
12 1<jupyter_text>MARL Setup Demo<jupyter_code>actor = lambda s: ActionPolicyNetwork(s, hSeq=(8,), gamma_reg=1e-1)
sensor = lambda st, out_dim: SensoriumNetworkTemplate(st, hSeq=(16,8,), out_dim=out_dim, gamma_reg=5.)

# num_episodes, n_epochs, max_len = (5, 4, 5)
num_episodes, n_epochs, max_len = (10, 400, 25)
flumrf = emi.EmbodiedAgent_IRFB(
    name="flu_mRFB",
    env_=flu_menv,
    alpha_p=5e3, alpha_v=5e2,
    max_episode_length=max_len,
    latentDim=4, 
    actorNN=actor, sensorium=sensor
)
# ??flumrf.play
(flumrf.a_size, flumrf.env.action_space_size)
sess = tf.InteractiveSession()
flumrf.init_graph(sess) # note tboard log dir
saver = tf.train.Saver(max_to_keep=1)<jupyter_output>Tensorboard logs in:  ./log/train_flu_mRFB<jupyter_text>Baseline<jupyter_code>print('Baselining untrained pnet...')
rwds0 = []
for k in range(num_episodes):
    flumrf.play(sess, terminal_reward=0.);
    rwds0.append(flumrf.last_total_returns)
    if k%20 == 0: print("\rEpisode {}/{}".format(k, num_episodes),end="")
base_perf = np.mean(rwds0)
print("\nAgent is flu-free for an average of {}pct of seasons".format(100.*base_perf))

base_per_agent = np.mean(np.array(rwds0)/float(flumrf.max_episode_length), axis=0)
st = flumrf.env.reset()<jupyter_output><empty_output><jupyter_text>Train Agent Population<jupyter_code># ### Train Agents
print('Training...')
flumrf.work(sess, num_epochs=n_epochs, saver=saver)<jupyter_output>Training...
Starting agent flu_mRFB
Epoch no.: 0/400

	Stats @Step 0: 	(Min, Mean, Max)
Perf/Recent Rewards: (22.0, 23.5, 25.0)
Losses/Policy LLs: (-7.0447245, -1.245924, -0.0010796713)
Losses/Critic Scores: (0.41931838, 4.800447, 16.665392)
Losses/Policy Entropies: (0.01, 0.41457543, 0.69228905)
Saved Model
Epoch no.: 100/400

	Stats @Step 100: 	(Min, Mean, Max)
Perf/Recent Rewards: (22.0, 23.75, 25.0)
Losses/Policy LLs: (-4.2649093, -0.32924187, -0.0012875671)
Losses/Critic Scores: (10.490915, 11.349155, 12.360098)
Losses/Policy Entropies: (0.01, 0.29661378, 0.68900746)
WARNING:tensorflow:From /miniconda3/lib/python3.6/site-packages/tensorflow/python/training/saver.py:966: remove_checkpoint (from tensorflow.python.training.checkpoint_management) is deprecated and will be removed in a future version.
Instructions for updating:
Use standard file APIs to delete files with this prefix.
Saved Model
Epoch no.: 200/400

	Stats @Step 200: 	(Min, Mean, Max)
Perf/Recent Rewards: (22.0, 23.25, [...]<jupyter_text>Test<jupyter_code># Test pnet!
print('Testing...')
rwds = []
for k in range(num_episodes):
    flumrf.play(sess)
    rwds.append(flumrf.last_total_returns)
    if k%int(num_episodes/5) == 0: print("\rEpisode {}/{}".format(k, num_episodes),end="")
trained_perf = np.mean(np.array(rwds)/float(flumrf.max_episode_length))
print("\nAgent is flu-free for an average of {} pct compared to baseline of {} pct".format(
    100*trained_perf, 100*base_perf) )

trained_per_agent = np.mean(np.array(rwds)/float(flumrf.max_episode_length), axis=0)<jupyter_output>Testing...
Episode 8/10
Agent is flu-free for an average of 92.4 pct compared to baseline of 2260.0 pct<jupyter_text>Evaluate<jupyter_code>fig, axs = plt.subplots(2, 1, sharex=True)
sns.violinplot(rwds0, ax = axs[0])
axs[0].set_title('Baseline Episode Rewards')
sns.violinplot(rwds, ax = axs[1])
axs[1].set_title('Trained Episode Rewards')
base_per_agent-trained_per_agent
fig, axs = plt.subplots(2,1)
cmp = sns.boxplot(base_per_agent-trained_per_agent, ax = axs[0])
axs[0].set_title('Distribution: Success Rate Per-Agent Improvement');
axs[1].bar(x=range(len(exos)), height=100.*(base_per_agent-trained_per_agent))<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.13814616755793227, 'fraction_numerical': 0.05258467023172905, 'mean_word_length': 5.996259351620948, 'pattern_counts': {'":': 0, '<': 25, '<?xml version=': 0, '>': 24, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 5, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '7', 'dir': 'jupyter-structured-clean-dedup', 'id': '403642', 'license': 'MIT', 'n_tokens_mistral': 2417, 'n_tokens_neox': 2193, 'n_words': 427, 'path': 'successful_runs/flu-MARL-expt-[freeze-fail].ipynb', 'repo_name': 'oosoba/rl-policy-abms', 'size': '5610'}