<jupyter_start><jupyter_text>Preamble: Libs + signal def<jupyter_code>import itertools, importlib, sys, warnings, os

import numpy as np
import matplotlib.pyplot as plt
import seaborn as sns

# ML libs
import tensorflow as tf
from tensorflow.python import debug as tf_debug
print("Tensorflow version:", tf.__version__)

## suppress annoy verbose tf msgs
warnings.filterwarnings("ignore")
os.environ['TF_CPP_MIN_LOG_LEVEL'] = '2' # '3' to block all including error msgs
tf.compat.v1.logging.set_verbosity(tf.compat.v1.logging.ERROR)
sys.path.append('./embodied_arch')
import embodied_arch.embodied_central_Qcritic as emac
importlib.reload(emac)

from embodied_misc import ActionPolicyNetwork, ValueNetwork, SensoriumNetworkTemplate
from embodied_arch.embodied_central_Qcritic import QsaNetwork
sys.path.append('./minoritygame')
import minoritygame.minority_multienv as MGME
importlib.reload(MGME) 


%matplotlib inline
plt.style.use('ggplot')
plt.rcParams['figure.figsize'] = (7,7)

log_path = './log/mingame'<jupyter_output><empty_output><jupyter_text>Setup k-vs-(N-k) Mingame Env<jupyter_code>importlib.reload(MGME)

# n_agents = 61
# exos = (np.random.sample(n_agents) < 0.3)  # np.sum(exos)
# exos = [True, True, False, False, False, True]  # Majority subpop
# exos = [True, True, False, False, False, False]  # Minority subpop

exos = [True, True, False, False, False]  # Minority subpop
menv = MGME.MinorityGame_Multiagent_env(
    m=3, s=4,
    exo_actorsQ=exos
)

print(len(menv.actor_exoQ), menv.actor_count, menv.actor_index)
print(menv.state_space_size, menv.action_space_size)<jupyter_output>5 2 [0, 1]
3 1<jupyter_text>Setup MARL<jupyter_code># num_episodes, n_epochs, max_len = (100, 601, 251)
num_episodes, n_epochs, max_len = (100, 751, 150)
actor = lambda s: ActionPolicyNetwork(s, hSeq=(32,), gamma_reg=1e-1)
value = lambda s: ValueNetwork(s, hSeq=(16,8,), gamma_reg=1.)
sensor = lambda st, out_dim: SensoriumNetworkTemplate(
    st, hSeq=(16,8,), 
    out_dim=out_dim, gamma_reg=5.
)

qsa = lambda sts, ats: QsaNetwork(
    sts, ats, 
    hSeq=(16,16,8,), gamma_reg=2.
)
tf.reset_default_graph()
importlib.reload(emac)
empopn = emac.EmbodiedAgent_MAC(
    name="mgMAC_k-vs-N-k", env_=menv, 
    alpha_p=50., alpha_v=12., alpha_q=3.,
    actorNN=actor, valueNN=value,
    latentDim=12, sensorium=sensor, QNN=qsa,
    max_episode_length=max_len,
    _every_=150, CyclicSchedule=None #("log", 150)  #("exp", 150) #None #("log", 50)
)
# empopn = emac.EmbodiedAgent_MAC( 
#     name=agent_name,
#     env_=MinorityGame_Multiagent_env,
#     latentDim=_zdim_,
#     space_size=(_s_size_, _a_size_),
#     sensorium=SensoriumNetworkTemplate,
#     actorNN=ActionPolicyNetwork,
#     valueNN=ValueNetwork,
#     QNN=QsaNetwork,
#     alpha_p=5e-2, alpha_v=1e-1, alpha_q=1e-1,
#     _every_=_every_, recover=None,
#     max_episode_length=_max_len_
# )

# print(empopn, empopn.s_size, empopn.a_size)
sess = tf.InteractiveSession()
# sess = tf_debug.LocalCLIDebugWrapperSession(sess)

empopn.init_graph(sess)  # note tboard log dir
## Verify step + play set up
state = empopn.env.reset()
# print(state, empopn.act(state, sess))

empopn.play(sess)
np.sum(np.array(empopn.episode_buffer['rewards']), axis=0)
# np.array(empopn.episode_buffer['rewards'])
print(empopn.last_total_returns/max_len)
print(emac.summarize_np(empopn.last_total_returns))<jupyter_output>[0.4        0.29333333]
(44.0, 52.0, 60.0)<jupyter_text>Pre-test Agent<jupyter_code>print('Baselining untrained pnet...')
rwd_mg0 = []
for k in range(num_episodes):
    empopn.play(sess, terminal_reward=0.)
    rwd_mg0.append(empopn.last_total_returns)
    if k%int(num_episodes/5) == 0: print("\rEpisode {}/{}".format(k, num_episodes),end="")
base_perf_mg = 100.*np.mean(np.array(rwd_mg0)/float(empopn.max_episode_length), axis=0)
print("\nAgents win an average of {} pct".format(base_perf_mg))

base_per_agent = 100.*np.mean(np.array(rwd_mg0)/float(empopn.max_episode_length), axis=0)<jupyter_output>Baselining untrained pnet...
Episode 80/100
Agents win an average of [37.00666667 34.08      ] pct<jupyter_text>Train MARL Agents<jupyter_code># state = empopn.env.reset()
# empopn.play(sess)
# tst = empopn.train_eval_QC(sess)
# print(tst.shape, np.mean(tst, axis=0))
obs = []
for ct in range(300): #(1250):
    empopn.play(sess)
    tmp = empopn.train_eval_QC(sess)
    obs.append(np.mean(tmp, axis=0))
    print('\r\t Iteration {}: Q loss({})'.format(ct, np.mean(tmp, axis=0)), end="")
plt.plot(obs[1:]);
# ### Train Agent w/ Algo on Experience Tuples
# Train pnet on mingame episodes
print('Training...')
saver = tf.train.Saver(max_to_keep=0)
hist = empopn.work(sess, num_epochs=n_epochs, saver=saver)<jupyter_output>Training...
Starting agent mgMAC_k-vs-N-k
Epoch no.: 0/751

	Stats @Step 0: 	(Min, Mean, Max)
Perf/Recent Rewards: (49.0, 50.5, 52.0)
Losses/Policy LLs: (-2.1662548, -0.5944331, -0.12172259)
Losses/Policy Entropies: (0.35603827, 0.51974726, 0.679375)
Values/Critic Scores: (-0.2650252, -0.09834319, 0.06948831)
Values/Mean Q Scores: (7.5349374, 7.5867987, 7.6386604)
Saved Model
Epoch no.: 150/751

	Stats @Step 150: 	(Min, Mean, Max)
Perf/Recent Rewards: (70.0, 71.0, 72.0)
Losses/Policy LLs: (-2.8835905, -0.14212415, -0.00019472178)
Losses/Policy Entropies: (0.002, 0.110333495, 0.21562883)
Values/Critic Scores: (-0.77487415, 0.6203121, 2.014483)
Values/Mean Q Scores: (23.636042, 28.34919, 33.062336)
Saved Model
Epoch no.: 165/751Model issues @Step 165. Stats(['Perf/Recent Rewards', 'Losses/Policy LLs', 'Losses/Policy Entropies', 'Values/Critic Scores', 'Values/Mean Q Scores']): ( [(72.0, 73.0, 74.0), (-3.4621296, -0.06171968, -6.399641e-10), (0.002, nan, 0.13944833), (1.6636868, 3.668613,[...]<jupyter_text>Post-test Agent<jupyter_code># Test pnet!
print('Testing...')
rwd_mg = []
for k in range(num_episodes):
    empopn.play(sess, terminal_reward=0.)
    rwd_mg.append(empopn.last_total_returns)
    if k%int(num_episodes/5) == 0: print("\rEpisode {}/{}".format(k, num_episodes),end="")
trained_perf_mg = 100.*np.mean(np.array(rwd_mg)/float(empopn.max_episode_length), axis=0)
print("\nAgent wins an average of {} pct compared to baseline of {} pct".format(
    trained_perf_mg, base_perf_mg) )

trained_per_agent = 100.*np.mean(np.array(rwd_mg)/float(empopn.max_episode_length), axis=0)
fig, axs = plt.subplots(2, 1, sharex=True)
sns.violinplot(np.array(rwd_mg0)/empopn.max_episode_length - 0.5, ax = axs[0])
axs[0].set_title('Baseline Mean Success Rates')
sns.violinplot(np.array(rwd_mg)/empopn.max_episode_length - 0.5, ax = axs[1])
axs[1].set_title('Trained Mean Success Rate')

print("\nAgent wins an average of {} pct \ncompared to baseline of {} pct".format(
    trained_perf_mg, base_perf_mg) )
fig, axs = plt.subplots(2,1)
cmp = sns.boxplot(trained_per_agent-base_per_agent, ax = axs[0])
axs[0].set_title('Distribution: Success Rate Per-Agent Improvement');
axs[1].bar(x=range(sum(exos)), height=(trained_per_agent-base_per_agent))<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1543094794206224, 'fraction_numerical': 0.0681198910081744, 'mean_word_length': 5.439519852262235, 'pattern_counts': {'":': 0, '<': 22, '<?xml version=': 0, '>': 21, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 3, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '6', 'dir': 'jupyter-structured-clean-dedup', 'id': '368691', 'license': 'MIT', 'n_tokens_mistral': 3230, 'n_tokens_neox': 2894, 'n_words': 526, 'path': 'successful_runs/mingame-preJan/mingame-multi-env[MAC-30Dec19].ipynb', 'repo_name': 'oosoba/rl-policy-abms', 'size': '6973'}