# Path: lib/actions.py
# ACTION_UP = "W"
# ACTION_DOWN = "S"
# ACTION_LEFT = "A"
# ACTION_RIGHT = "D"
# ACTION_UP_DOWN_NONE = "~WS"
# ACTION_LEFT_RIGHT_NONE = "~AD"
# ACTIONS_UP_DOWN = [ACTION_UP, ACTION_DOWN, ACTION_UP_DOWN_NONE]
# ACTIONS_LEFT_RIGHT = [ACTION_LEFT, ACTION_RIGHT, ACTION_LEFT_RIGHT_NONE]
# ALL_ACTIONS = ACTIONS_UP_DOWN + ACTIONS_LEFT_RIGHT
# ACTION_TO_KEY = {
#     ACTION_UP: "w",
#     ACTION_DOWN: "s",
#     ACTION_LEFT: "a",
#     ACTION_RIGHT: "d",
#     ACTION_UP_DOWN_NONE: None,
#     ACTION_LEFT_RIGHT_NONE: None
# }
# ACTION_UP_VECTOR_INDEX = 0
# ACTION_DOWN_VECTOR_INDEX = 1
# ACTION_LEFT_VECTOR_INDEX = 0
# ACTION_RIGHT_VECTOR_INDEX = 1
# ACTION_UP_DOWN_NONE_VECTOR_INDEX = 2
# ACTION_LEFT_RIGHT_NONE_VECTOR_INDEX = 2
# ACTION_UP_VECTOR = make_one_hot_vector(3, ACTION_UP_VECTOR_INDEX)
# ACTION_DOWN_VECTOR = make_one_hot_vector(3, ACTION_DOWN_VECTOR_INDEX)
# ACTION_LEFT_VECTOR = make_one_hot_vector(3, ACTION_LEFT_VECTOR_INDEX)
# ACTION_RIGHT_VECTOR = make_one_hot_vector(3, ACTION_RIGHT_VECTOR_INDEX)
# ACTION_UP_DOWN_NONE_VECTOR = make_one_hot_vector(3, ACTION_UP_DOWN_NONE_VECTOR_INDEX)
# ACTION_LEFT_RIGHT_NONE_VECTOR = make_one_hot_vector(3, ACTION_LEFT_RIGHT_NONE_VECTOR_INDEX)
# ACTION_UP_DOWN_TO_VECTOR_INDEX = {
#     ACTION_UP: ACTION_UP_VECTOR_INDEX,
#     ACTION_DOWN: ACTION_DOWN_VECTOR_INDEX,
#     ACTION_UP_DOWN_NONE: ACTION_UP_DOWN_NONE_VECTOR_INDEX
# }
# ACTION_LEFT_RIGHT_TO_VECTOR_INDEX = {
#     ACTION_LEFT: ACTION_LEFT_VECTOR_INDEX,
#     ACTION_RIGHT: ACTION_RIGHT_VECTOR_INDEX,
#     ACTION_LEFT_RIGHT_NONE: ACTION_LEFT_RIGHT_NONE_VECTOR_INDEX
# }
# ACTION_UP_DOWN_FROM_VECTOR_INDEX = [
#     ACTION_UP,
#     ACTION_DOWN,
#     ACTION_UP_DOWN_NONE
# ]
# ACTION_LEFT_RIGHT_FROM_VECTOR_INDEX = [
#     ACTION_LEFT,
#     ACTION_RIGHT,
#     ACTION_LEFT_RIGHT_NONE
# ]
# ACTION_UP_DOWN_TO_VECTOR = {
#     ACTION_UP: ACTION_UP_VECTOR,
#     ACTION_DOWN: ACTION_DOWN_VECTOR,
#     ACTION_UP_DOWN_NONE: ACTION_UP_DOWN_NONE_VECTOR
# }
# ACTION_LEFT_RIGHT_TO_VECTOR = {
#     ACTION_LEFT: ACTION_LEFT_VECTOR,
#     ACTION_RIGHT: ACTION_RIGHT_VECTOR,
#     ACTION_LEFT_RIGHT_NONE: ACTION_LEFT_RIGHT_NONE_VECTOR
# }
# ALL_MULTIACTIONS = []
# ACTIONS_TO_MULTIVEC = dict()
# ACTIONS_FROM_MULTIVEC_INDEX = []
# def make_one_hot_vector(size, idx_one):
# def get_random_action_up_down():
# def get_random_action_left_right():
# def keys_to_action_up_down(keys):
# def keys_to_action_left_right(keys):
# def action_up_down_to_vector(action):
# def action_left_right_to_vector(action):
# def action_up_down_to_vector_index(action):
# def action_left_right_to_vector_index(action):
# def action_up_down_from_vector_index(index):
# def action_left_right_from_vector_index(index):
# def action_to_key(action):
# def actions_to_multivec(action_up_down, action_left_right):


import sys
import os
import models as models_reinforced
from lib import actions as actionslib
        plan_str = " ".join(["%s%s" % (a, b) for (a, b) in plan])
        if plan_str not in added:
            result.append(plan)
            added.add(plan_str)
    return result

def create_plans(nb_future_states):
    PLANS = []
    # plans that contain only a specific multiaction
    for multiaction in actionslib.ALL_MULTIACTIONS:
        plan = []
        for i in xrange(nb_future_states):
            plan.append(multiaction)
        PLANS.append(plan)
    # plans that contain multiaction A followed by B at every Nth step
    # e.g [U U UR U U UR U U UR] (U=up, UR=up+right)
    for n in [1, 2, 3]:
        for ma1 in actionslib.ALL_MULTIACTIONS:
            for ma2 in actionslib.ALL_MULTIACTIONS:
                if ma1 != ma2:
                    plan = []
                    for i in xrange(nb_future_states):
                        if i % n == 0:
                            plan.append(ma2)
                        else:
                            plan.append(ma1)
                    PLANS.append(plan)
    # plans that start with multiaction A followed by B after P percent of timesteps
    # e.g. [U U U U U UR UR UR UR UR] for P=50percent
    for p in [0.25, 0.5, 0.75]:

# Path: lib/actions.py
# ACTION_UP = "W"
# ACTION_DOWN = "S"
# ACTION_LEFT = "A"
# ACTION_RIGHT = "D"
# ACTION_UP_DOWN_NONE = "~WS"
# ACTION_LEFT_RIGHT_NONE = "~AD"
# ACTIONS_UP_DOWN = [ACTION_UP, ACTION_DOWN, ACTION_UP_DOWN_NONE]
# ACTIONS_LEFT_RIGHT = [ACTION_LEFT, ACTION_RIGHT, ACTION_LEFT_RIGHT_NONE]
# ALL_ACTIONS = ACTIONS_UP_DOWN + ACTIONS_LEFT_RIGHT
# ACTION_TO_KEY = {
#     ACTION_UP: "w",
#     ACTION_DOWN: "s",
#     ACTION_LEFT: "a",
#     ACTION_RIGHT: "d",
#     ACTION_UP_DOWN_NONE: None,
#     ACTION_LEFT_RIGHT_NONE: None
# }
# ACTION_UP_VECTOR_INDEX = 0
# ACTION_DOWN_VECTOR_INDEX = 1
# ACTION_LEFT_VECTOR_INDEX = 0
# ACTION_RIGHT_VECTOR_INDEX = 1
# ACTION_UP_DOWN_NONE_VECTOR_INDEX = 2
# ACTION_LEFT_RIGHT_NONE_VECTOR_INDEX = 2
# ACTION_UP_VECTOR = make_one_hot_vector(3, ACTION_UP_VECTOR_INDEX)
# ACTION_DOWN_VECTOR = make_one_hot_vector(3, ACTION_DOWN_VECTOR_INDEX)
# ACTION_LEFT_VECTOR = make_one_hot_vector(3, ACTION_LEFT_VECTOR_INDEX)
# ACTION_RIGHT_VECTOR = make_one_hot_vector(3, ACTION_RIGHT_VECTOR_INDEX)
# ACTION_UP_DOWN_NONE_VECTOR = make_one_hot_vector(3, ACTION_UP_DOWN_NONE_VECTOR_INDEX)
# ACTION_LEFT_RIGHT_NONE_VECTOR = make_one_hot_vector(3, ACTION_LEFT_RIGHT_NONE_VECTOR_INDEX)
# ACTION_UP_DOWN_TO_VECTOR_INDEX = {
#     ACTION_UP: ACTION_UP_VECTOR_INDEX,
#     ACTION_DOWN: ACTION_DOWN_VECTOR_INDEX,
#     ACTION_UP_DOWN_NONE: ACTION_UP_DOWN_NONE_VECTOR_INDEX
# }
# ACTION_LEFT_RIGHT_TO_VECTOR_INDEX = {
#     ACTION_LEFT: ACTION_LEFT_VECTOR_INDEX,
#     ACTION_RIGHT: ACTION_RIGHT_VECTOR_INDEX,
#     ACTION_LEFT_RIGHT_NONE: ACTION_LEFT_RIGHT_NONE_VECTOR_INDEX
# }
# ACTION_UP_DOWN_FROM_VECTOR_INDEX = [
#     ACTION_UP,
#     ACTION_DOWN,
#     ACTION_UP_DOWN_NONE
# ]
# ACTION_LEFT_RIGHT_FROM_VECTOR_INDEX = [
#     ACTION_LEFT,
#     ACTION_RIGHT,
#     ACTION_LEFT_RIGHT_NONE
# ]
# ACTION_UP_DOWN_TO_VECTOR = {
#     ACTION_UP: ACTION_UP_VECTOR,
#     ACTION_DOWN: ACTION_DOWN_VECTOR,
#     ACTION_UP_DOWN_NONE: ACTION_UP_DOWN_NONE_VECTOR
# }
# ACTION_LEFT_RIGHT_TO_VECTOR = {
#     ACTION_LEFT: ACTION_LEFT_VECTOR,
#     ACTION_RIGHT: ACTION_RIGHT_VECTOR,
#     ACTION_LEFT_RIGHT_NONE: ACTION_LEFT_RIGHT_NONE_VECTOR
# }
# ALL_MULTIACTIONS = []
# ACTIONS_TO_MULTIVEC = dict()
# ACTIONS_FROM_MULTIVEC_INDEX = []
# def make_one_hot_vector(size, idx_one):
# def get_random_action_up_down():
# def get_random_action_left_right():
# def keys_to_action_up_down(keys):
# def keys_to_action_left_right(keys):
# def action_up_down_to_vector(action):
# def action_left_right_to_vector(action):
# def action_up_down_to_vector_index(action):
# def action_left_right_to_vector_index(action):
# def action_up_down_from_vector_index(index):
# def action_left_right_from_vector_index(index):
# def action_to_key(action):
# def actions_to_multivec(action_up_down, action_left_right):


# Path: train_reinforced/plans.py
import sys
import os
import models as models_reinforced
from lib import actions as actionslib

        plan_str = " ".join(["%s%s" % (a, b) for (a, b) in plan])
        if plan_str not in added:
            result.append(plan)
            added.add(plan_str)
    return result

def create_plans(nb_future_states):
    PLANS = []
    # plans that contain only a specific multiaction
    for multiaction in actionslib.ALL_MULTIACTIONS:
        plan = []
        for i in xrange(nb_future_states):
            plan.append(multiaction)
        PLANS.append(plan)
    # plans that contain multiaction A followed by B at every Nth step
    # e.g [U U UR U U UR U U UR] (U=up, UR=up+right)
    for n in [1, 2, 3]:
        for ma1 in actionslib.ALL_MULTIACTIONS:
            for ma2 in actionslib.ALL_MULTIACTIONS:
                if ma1 != ma2:
                    plan = []
                    for i in xrange(nb_future_states):
                        if i % n == 0:
                            plan.append(ma2)
                        else:
                            plan.append(ma1)
                    PLANS.append(plan)
    # plans that start with multiaction A followed by B after P percent of timesteps
    # e.g. [U U U U U UR UR UR UR UR] for P=50percent
    for p in [0.25, 0.5, 0.75]:

        for ma1 in actionslib.ALL_MULTIACTIONS: