# Path: agents/capacities.py
# def swish(x):
# def eps_greedy(inputs_t, q_preds_t, nb_actions, N0, min_eps, nb_state=None):
# def tabular_eps_greedy(inputs_t, q_preds_t, nb_states, nb_actions, N0, min_eps):
# def tabular_UCB(Qs_t, inputs_t):
# def eligibility_traces(Qs_t, states_t, actions_t, discount, lambda_value):
# def eligibility_dutch_traces(Qs_t, states_t, actions_t, lr, discount, lambda_value):
# def get_expected_rewards(episode_rewards, discount=.99):
# def tf_get_n_step_expected_rewards(episode_rewards_t, estimates_t, discount, n_step):
# def get_n_step_expected_rewards(episode_rewards, estimates, discount=.99, n_step=1):
# def get_n_step_expected_rewards_mat(episode_rewards, estimates, discount=.99, n_step=1):
# def get_lambda_expected_rewards(episode_rewards, estimates, discount=.99, lambda_value=.9):
# def get_mc_target(rewards_t, discount):
# def get_td_target(Qs_t, rewards_t, next_states_t, next_actions_t, discount):
# def get_td_n_target(Qs_t, rewards_t, next_state_t, next_action_t, discount, n_step):
# def get_q_learning_target(Qs_t, rewards_t, next_states_t, discount):
# def get_expected_sarsa_target(Qs_t, rewards_t, next_states_t, next_probs_t, discount):
# def get_sigma_target(Qs_t, sigma, rewards_t, next_states_t, next_actions_t, next_probs_t, discount):
# def tabular_learning(Qs_t, states_t, actions_t, targets):
# def tabular_learning_with_lr(init_lr, decay_steps, Qs_t, states_t, actions_t, targets):
# def counter(name):
# def fix_scope(from_scope):
# def build_batches(dtKeys, sequence_history, batch_size):
# def policy(network_params, inputs, summary_collections=None):
# def value_f(network_params, inputs, summary_collections=None):
# def predictive_model(network_params, m_inputs, dynamic_batch_size, m_initial_state=None, summary_collections=None):
# def projection(proj_params, inputs, summary_collections=None):
#         N = tf.Variable(1., trainable=False, dtype=tf.float32, name='N')
#         N = tf.Variable(tf.ones(shape=[nb_state]), name='N', trainable=False)
#     W1 = tf.get_variable('W1'
#         , shape=[ network_params['nb_inputs'], network_params['nb_units'] ]
#         , initializer=tf.random_normal_initializer(mean=network_params['initial_mean'], stddev=network_params['initial_stddev'])
#     )
#     W2 = tf.get_variable('W2'
#         , shape=[ network_params['nb_units'], network_params['nb_units'] ]
#         , initializer=tf.random_normal_initializer(mean=network_params['initial_mean'], stddev=network_params['initial_stddev'])
#     )
#     W3 = tf.get_variable('W3'
#         , shape=[ network_params['nb_units'], network_params['nb_outputs'] ]
#         , initializer=tf.random_normal_initializer(mean=network_params['initial_mean'], stddev=network_params['initial_stddev'])
#     )
#     W1 = tf.get_variable('W1'
#         , shape=[ network_params['nb_inputs'], network_params['nb_units'] ]
#         , initializer=w_init
#     )
#     W2 = tf.get_variable('W2'
#         , shape=[ network_params['nb_units'], network_params['nb_units'] ]
#         , initializer=w_init
#     )
#     W3 = tf.get_variable('W3'
#         , shape=[ network_params['nb_units'], network_params['nb_outputs'] ]
#         , initializer=w_init
#     )
#     W1 = tf.get_variable('W1'
#         , shape=[ proj_params['nb_units'], proj_params['nb_units'] ]
#         , initializer=w_init
#     )
#     W2 = tf.get_variable('W2'
#         , shape=[ proj_params['nb_units'], proj_params['nb_actions'] ]
#         , initializer=w_init
#     )


import os, sys, unittest, timeit
import numpy as np
import tensorflow as tf
from agents import capacities

dir = os.path.dirname(os.path.realpath(__file__))
sys.path.append(dir + '/../..')


class TestCapacities(unittest.TestCase):

    def test_tabular_eps_greedy(self):
        nb_states = 3
        nb_actions = 2

        with tf.Graph().as_default():
            tf.set_random_seed(1)

            inputs_t = tf.random_uniform(shape=[2], minval=0, maxval=3, dtype=tf.int32)
            # inputs_t = tf.Print(inputs_t, data=[inputs_t], message='inputs_t')

            Qs = tf.random_uniform(shape=[nb_states, nb_actions])
            # Qs = tf.Print(Qs, data=[Qs], message='Qs', summarize=12)

            q_preds = tf.gather(Qs, inputs_t)
            # q_preds = tf.Print(q_preds, data=[q_preds], message='q_preds', summarize=6)

            N0 = 100

# Path: agents/capacities.py
# def swish(x):
# def eps_greedy(inputs_t, q_preds_t, nb_actions, N0, min_eps, nb_state=None):
# def tabular_eps_greedy(inputs_t, q_preds_t, nb_states, nb_actions, N0, min_eps):
# def tabular_UCB(Qs_t, inputs_t):
# def eligibility_traces(Qs_t, states_t, actions_t, discount, lambda_value):
# def eligibility_dutch_traces(Qs_t, states_t, actions_t, lr, discount, lambda_value):
# def get_expected_rewards(episode_rewards, discount=.99):
# def tf_get_n_step_expected_rewards(episode_rewards_t, estimates_t, discount, n_step):
# def get_n_step_expected_rewards(episode_rewards, estimates, discount=.99, n_step=1):
# def get_n_step_expected_rewards_mat(episode_rewards, estimates, discount=.99, n_step=1):
# def get_lambda_expected_rewards(episode_rewards, estimates, discount=.99, lambda_value=.9):
# def get_mc_target(rewards_t, discount):
# def get_td_target(Qs_t, rewards_t, next_states_t, next_actions_t, discount):
# def get_td_n_target(Qs_t, rewards_t, next_state_t, next_action_t, discount, n_step):
# def get_q_learning_target(Qs_t, rewards_t, next_states_t, discount):
# def get_expected_sarsa_target(Qs_t, rewards_t, next_states_t, next_probs_t, discount):
# def get_sigma_target(Qs_t, sigma, rewards_t, next_states_t, next_actions_t, next_probs_t, discount):
# def tabular_learning(Qs_t, states_t, actions_t, targets):
# def tabular_learning_with_lr(init_lr, decay_steps, Qs_t, states_t, actions_t, targets):
# def counter(name):
# def fix_scope(from_scope):
# def build_batches(dtKeys, sequence_history, batch_size):
# def policy(network_params, inputs, summary_collections=None):
# def value_f(network_params, inputs, summary_collections=None):
# def predictive_model(network_params, m_inputs, dynamic_batch_size, m_initial_state=None, summary_collections=None):
# def projection(proj_params, inputs, summary_collections=None):
#         N = tf.Variable(1., trainable=False, dtype=tf.float32, name='N')
#         N = tf.Variable(tf.ones(shape=[nb_state]), name='N', trainable=False)
#     W1 = tf.get_variable('W1'
#         , shape=[ network_params['nb_inputs'], network_params['nb_units'] ]
#         , initializer=tf.random_normal_initializer(mean=network_params['initial_mean'], stddev=network_params['initial_stddev'])
#     )
#     W2 = tf.get_variable('W2'
#         , shape=[ network_params['nb_units'], network_params['nb_units'] ]
#         , initializer=tf.random_normal_initializer(mean=network_params['initial_mean'], stddev=network_params['initial_stddev'])
#     )
#     W3 = tf.get_variable('W3'
#         , shape=[ network_params['nb_units'], network_params['nb_outputs'] ]
#         , initializer=tf.random_normal_initializer(mean=network_params['initial_mean'], stddev=network_params['initial_stddev'])
#     )
#     W1 = tf.get_variable('W1'
#         , shape=[ network_params['nb_inputs'], network_params['nb_units'] ]
#         , initializer=w_init
#     )
#     W2 = tf.get_variable('W2'
#         , shape=[ network_params['nb_units'], network_params['nb_units'] ]
#         , initializer=w_init
#     )
#     W3 = tf.get_variable('W3'
#         , shape=[ network_params['nb_units'], network_params['nb_outputs'] ]
#         , initializer=w_init
#     )
#     W1 = tf.get_variable('W1'
#         , shape=[ proj_params['nb_units'], proj_params['nb_units'] ]
#         , initializer=w_init
#     )
#     W2 = tf.get_variable('W2'
#         , shape=[ proj_params['nb_units'], proj_params['nb_actions'] ]
#         , initializer=w_init
#     )


# Path: tests/agents/test_capacities.py
import os, sys, unittest, timeit
import numpy as np
import tensorflow as tf
from agents import capacities


dir = os.path.dirname(os.path.realpath(__file__))
sys.path.append(dir + '/../..')


class TestCapacities(unittest.TestCase):

    def test_tabular_eps_greedy(self):
        nb_states = 3
        nb_actions = 2

        with tf.Graph().as_default():
            tf.set_random_seed(1)

            inputs_t = tf.random_uniform(shape=[2], minval=0, maxval=3, dtype=tf.int32)
            # inputs_t = tf.Print(inputs_t, data=[inputs_t], message='inputs_t')

            Qs = tf.random_uniform(shape=[nb_states, nb_actions])
            # Qs = tf.Print(Qs, data=[Qs], message='Qs', summarize=12)

            q_preds = tf.gather(Qs, inputs_t)
            # q_preds = tf.Print(q_preds, data=[q_preds], message='q_preds', summarize=6)

            N0 = 100

            actions, probs = capacities.tabular_eps_greedy(inputs_t, q_preds, nb_states, nb_actions, N0, 0.)