# Path: agents/capacities.py
# def swish(x):
# def eps_greedy(inputs_t, q_preds_t, nb_actions, N0, min_eps, nb_state=None):
# def tabular_eps_greedy(inputs_t, q_preds_t, nb_states, nb_actions, N0, min_eps):
# def tabular_UCB(Qs_t, inputs_t):
# def eligibility_traces(Qs_t, states_t, actions_t, discount, lambda_value):
# def eligibility_dutch_traces(Qs_t, states_t, actions_t, lr, discount, lambda_value):
# def get_expected_rewards(episode_rewards, discount=.99):
# def tf_get_n_step_expected_rewards(episode_rewards_t, estimates_t, discount, n_step):
# def get_n_step_expected_rewards(episode_rewards, estimates, discount=.99, n_step=1):
# def get_n_step_expected_rewards_mat(episode_rewards, estimates, discount=.99, n_step=1):
# def get_lambda_expected_rewards(episode_rewards, estimates, discount=.99, lambda_value=.9):
# def get_mc_target(rewards_t, discount):
# def get_td_target(Qs_t, rewards_t, next_states_t, next_actions_t, discount):
# def get_td_n_target(Qs_t, rewards_t, next_state_t, next_action_t, discount, n_step):
# def get_q_learning_target(Qs_t, rewards_t, next_states_t, discount):
# def get_expected_sarsa_target(Qs_t, rewards_t, next_states_t, next_probs_t, discount):
# def get_sigma_target(Qs_t, sigma, rewards_t, next_states_t, next_actions_t, next_probs_t, discount):
# def tabular_learning(Qs_t, states_t, actions_t, targets):
# def tabular_learning_with_lr(init_lr, decay_steps, Qs_t, states_t, actions_t, targets):
# def counter(name):
# def fix_scope(from_scope):
# def build_batches(dtKeys, sequence_history, batch_size):
# def policy(network_params, inputs, summary_collections=None):
# def value_f(network_params, inputs, summary_collections=None):
# def predictive_model(network_params, m_inputs, dynamic_batch_size, m_initial_state=None, summary_collections=None):
# def projection(proj_params, inputs, summary_collections=None):
#         N = tf.Variable(1., trainable=False, dtype=tf.float32, name='N')
#         N = tf.Variable(tf.ones(shape=[nb_state]), name='N', trainable=False)
#     W1 = tf.get_variable('W1'
#         , shape=[ network_params['nb_inputs'], network_params['nb_units'] ]
#         , initializer=tf.random_normal_initializer(mean=network_params['initial_mean'], stddev=network_params['initial_stddev'])
#     )
#     W2 = tf.get_variable('W2'
#         , shape=[ network_params['nb_units'], network_params['nb_units'] ]
#         , initializer=tf.random_normal_initializer(mean=network_params['initial_mean'], stddev=network_params['initial_stddev'])
#     )
#     W3 = tf.get_variable('W3'
#         , shape=[ network_params['nb_units'], network_params['nb_outputs'] ]
#         , initializer=tf.random_normal_initializer(mean=network_params['initial_mean'], stddev=network_params['initial_stddev'])
#     )
#     W1 = tf.get_variable('W1'
#         , shape=[ network_params['nb_inputs'], network_params['nb_units'] ]
#         , initializer=w_init
#     )
#     W2 = tf.get_variable('W2'
#         , shape=[ network_params['nb_units'], network_params['nb_units'] ]
#         , initializer=w_init
#     )
#     W3 = tf.get_variable('W3'
#         , shape=[ network_params['nb_units'], network_params['nb_outputs'] ]
#         , initializer=w_init
#     )
#     W1 = tf.get_variable('W1'
#         , shape=[ proj_params['nb_units'], proj_params['nb_units'] ]
#         , initializer=w_init
#     )
#     W2 = tf.get_variable('W2'
#         , shape=[ proj_params['nb_units'], proj_params['nb_actions'] ]
#         , initializer=w_init
#     )
# 
# Path: agents/basic_agent.py
# class TabularBasicAgent(BasicAgent):
#     """
#     Agent implementing tabular Q-learning.
#     """
#     def __init__(self, config, env):
#         if 'debug' in config:
#             config.update(phis.getPhiConfig(config['env_name'], config['debug']))
#         else:
#             config.update(phis.getPhiConfig(config['env_name']))
#         super(TabularBasicAgent, self).__init__(config, env)


import numpy as np
import tensorflow as tf
from agents import TabularBasicAgent, capacities
            , 'min_eps': get_min_eps()
            , 'initial_q_value': get_initial_q_value()
        }
        random_config.update(fixed_params)

        return random_config

    def build_graph(self, graph):
        with graph.as_default():
            tf.set_random_seed(self.random_seed)

            self.inputs_plh = tf.placeholder(tf.int32, shape=[None], name="inputs_plh")
            
            q_scope = tf.VariableScope(reuse=False, name='QValues')
            with tf.variable_scope(q_scope):
                self.Qs = tf.get_variable('Qs'
                    , shape=[self.nb_state, self.action_space.n]
                    , initializer=tf.constant_initializer(self.initial_q_value)
                    , dtype=tf.float32
                )
                tf.summary.histogram('Qarray', self.Qs)
                self.q_preds_t = tf.gather(self.Qs, self.inputs_plh)

            policy_scope = tf.VariableScope(reuse=False, name='Policy')
            with tf.variable_scope(policy_scope):
                if 'UCB' in self.config and self.config['UCB']:
                    self.actions_t, self.probs_t = capacities.tabular_UCB(
                        self.Qs, self.inputs_plh
                    )    
                else:

# Path: agents/capacities.py
# def swish(x):
# def eps_greedy(inputs_t, q_preds_t, nb_actions, N0, min_eps, nb_state=None):
# def tabular_eps_greedy(inputs_t, q_preds_t, nb_states, nb_actions, N0, min_eps):
# def tabular_UCB(Qs_t, inputs_t):
# def eligibility_traces(Qs_t, states_t, actions_t, discount, lambda_value):
# def eligibility_dutch_traces(Qs_t, states_t, actions_t, lr, discount, lambda_value):
# def get_expected_rewards(episode_rewards, discount=.99):
# def tf_get_n_step_expected_rewards(episode_rewards_t, estimates_t, discount, n_step):
# def get_n_step_expected_rewards(episode_rewards, estimates, discount=.99, n_step=1):
# def get_n_step_expected_rewards_mat(episode_rewards, estimates, discount=.99, n_step=1):
# def get_lambda_expected_rewards(episode_rewards, estimates, discount=.99, lambda_value=.9):
# def get_mc_target(rewards_t, discount):
# def get_td_target(Qs_t, rewards_t, next_states_t, next_actions_t, discount):
# def get_td_n_target(Qs_t, rewards_t, next_state_t, next_action_t, discount, n_step):
# def get_q_learning_target(Qs_t, rewards_t, next_states_t, discount):
# def get_expected_sarsa_target(Qs_t, rewards_t, next_states_t, next_probs_t, discount):
# def get_sigma_target(Qs_t, sigma, rewards_t, next_states_t, next_actions_t, next_probs_t, discount):
# def tabular_learning(Qs_t, states_t, actions_t, targets):
# def tabular_learning_with_lr(init_lr, decay_steps, Qs_t, states_t, actions_t, targets):
# def counter(name):
# def fix_scope(from_scope):
# def build_batches(dtKeys, sequence_history, batch_size):
# def policy(network_params, inputs, summary_collections=None):
# def value_f(network_params, inputs, summary_collections=None):
# def predictive_model(network_params, m_inputs, dynamic_batch_size, m_initial_state=None, summary_collections=None):
# def projection(proj_params, inputs, summary_collections=None):
#         N = tf.Variable(1., trainable=False, dtype=tf.float32, name='N')
#         N = tf.Variable(tf.ones(shape=[nb_state]), name='N', trainable=False)
#     W1 = tf.get_variable('W1'
#         , shape=[ network_params['nb_inputs'], network_params['nb_units'] ]
#         , initializer=tf.random_normal_initializer(mean=network_params['initial_mean'], stddev=network_params['initial_stddev'])
#     )
#     W2 = tf.get_variable('W2'
#         , shape=[ network_params['nb_units'], network_params['nb_units'] ]
#         , initializer=tf.random_normal_initializer(mean=network_params['initial_mean'], stddev=network_params['initial_stddev'])
#     )
#     W3 = tf.get_variable('W3'
#         , shape=[ network_params['nb_units'], network_params['nb_outputs'] ]
#         , initializer=tf.random_normal_initializer(mean=network_params['initial_mean'], stddev=network_params['initial_stddev'])
#     )
#     W1 = tf.get_variable('W1'
#         , shape=[ network_params['nb_inputs'], network_params['nb_units'] ]
#         , initializer=w_init
#     )
#     W2 = tf.get_variable('W2'
#         , shape=[ network_params['nb_units'], network_params['nb_units'] ]
#         , initializer=w_init
#     )
#     W3 = tf.get_variable('W3'
#         , shape=[ network_params['nb_units'], network_params['nb_outputs'] ]
#         , initializer=w_init
#     )
#     W1 = tf.get_variable('W1'
#         , shape=[ proj_params['nb_units'], proj_params['nb_units'] ]
#         , initializer=w_init
#     )
#     W2 = tf.get_variable('W2'
#         , shape=[ proj_params['nb_units'], proj_params['nb_actions'] ]
#         , initializer=w_init
#     )
# 
# Path: agents/basic_agent.py
# class TabularBasicAgent(BasicAgent):
#     """
#     Agent implementing tabular Q-learning.
#     """
#     def __init__(self, config, env):
#         if 'debug' in config:
#             config.update(phis.getPhiConfig(config['env_name'], config['debug']))
#         else:
#             config.update(phis.getPhiConfig(config['env_name']))
#         super(TabularBasicAgent, self).__init__(config, env)


# Path: agents/tabular_q_agent.py
import numpy as np
import tensorflow as tf
from agents import TabularBasicAgent, capacities

            , 'min_eps': get_min_eps()
            , 'initial_q_value': get_initial_q_value()
        }
        random_config.update(fixed_params)

        return random_config

    def build_graph(self, graph):
        with graph.as_default():
            tf.set_random_seed(self.random_seed)

            self.inputs_plh = tf.placeholder(tf.int32, shape=[None], name="inputs_plh")
            
            q_scope = tf.VariableScope(reuse=False, name='QValues')
            with tf.variable_scope(q_scope):
                self.Qs = tf.get_variable('Qs'
                    , shape=[self.nb_state, self.action_space.n]
                    , initializer=tf.constant_initializer(self.initial_q_value)
                    , dtype=tf.float32
                )
                tf.summary.histogram('Qarray', self.Qs)
                self.q_preds_t = tf.gather(self.Qs, self.inputs_plh)

            policy_scope = tf.VariableScope(reuse=False, name='Policy')
            with tf.variable_scope(policy_scope):
                if 'UCB' in self.config and self.config['UCB']:
                    self.actions_t, self.probs_t = capacities.tabular_UCB(
                        self.Qs, self.inputs_plh
                    )    
                else:

                    self.actions_t, self.probs_t = capacities.tabular_eps_greedy(