# Path: rlvision/utils.py
# def load_grid_selection(path):
# def process_map_data(path, return_full=False):
# def mask_grid(pos, grid, radius, one_is_free=True):
# def accumulate_map(source_grid, new_grid, one_is_free=True):
# def plot_grid(data, imsize, start=None, pos=None, goal=None, title=None):
# def process_gridworld_data(data_in, imsize):
# def load_mat_data(file_path):
# def find_layout(data, verbose=0):
# def init_h5_db(db_name, save_dir):
# def add_h5_group(group_name, db, db_path=None):
# def add_h5_ds(data, ds_name, db, group_name=None, data_type=np.uint8,
#               renew=False):
# def create_grid_8_dataset(mat_file_path, db_name, save_dir):
# def create_ds_from_splits(mat_file_prefix, db, splits):
# def create_grid_16_dataset(mat_file_prefix, db_name, save_dir, splits=5):
# def create_grid_28_dataset(mat_file_prefix, db_name, save_dir, splits=20):
# def create_grid_40_dataset(mat_file_prefix, db_name, save_dir, splits=100):
# def load_grid8(return_imsize=True):
# def load_grid16(split=None, return_imsize=True):
# def load_grid28(split=None, return_imsize=True):
# def load_grid40(split=None, return_imsize=True):
# 
# Path: rlvision/grid.py
# class Grid(object):
# class GridSampler(object):
# class GridDataSampler(object):
#     def __init__(self, grid_map, value_map, im_size=None,
#                  start_pos=None, is_po=True, mask_radius=3,
#                  grid_type="one-is-free", dstar=False):
#     def get_state(self):
#     def is_pos_valid(self, pos):
#     def rand_start_pos(self):
#     def set_start_pos(self, start_pos):
#     def set_goal_pos(self):
#     def set_curr_pos(self, curr_pos):
#     def update_curr_map(self, map_update, dstar_map_update=None):
#     def get_curr_dstar_visible_map(self, pos):
#     def get_curr_visible_map(self, pos):
#     def get_state_map(self):
#     def update_state(self, pos_update):
#     def action2pos(self, action):
#     def update_state_from_action(self, action, verbose=0):
#     def get_time(self):
#     def get_state_reward(self, max_num_steps=None):
#     def __init__(self, im_data, state_data, label_data, sample_idx, im_size):
#     def find_goal(self, value_grid):
#     def get_grid(self, grid_idx):
#     def compare_grid(self, grid_1, grid_2):
#     def next(self):
#     def __init__(self, grid_data, value_data, im_size, states_xy,
#                  label_data):
#     def compare_pos(self, pos1, pos2):
#     def get_next_state(self, pos, action):
#     def get_goal_pos(self, value_grid):
#     def next(self):
# def sample_data(db, imsize, num_samples=0):
# def sample_data_grid8(num_samples=0):
# def sample_data_grid16(split=None, num_samples=0):
# def sample_data_grid28(split=None, num_samples=0):
# def sample_data_grid40(split=None, num_samples=0):
# def create_train_grid8(db_name, save_dir, num_samples=0):
# def create_train_grid16(db_name, save_dir, num_samples=0):
# def create_train_grid28(db_name, save_dir, num_samples=0):
# def create_train_grid40(db_name, save_dir, num_samples=0):
# def load_train_grid8(return_imsize=True):
# def load_train_grid16(return_imsize=True):
# def load_train_grid28(return_imsize=True):
# def load_train_grid40(return_imsize=True):


import tensorflow as tf
import os
import numpy as np
import random
import time
import datetime
import rlvision
from collections import deque
from rlvision import utils
from rlvision import grid
                action = agent.epsilon_greedy(state) # epsilon-greedy for exploration
                game.update_state_from_action(action)
                reward, done = game.get_state_reward()
                # use updated partially-observable map as next state
                tmp_value_pos = value[game_idx].copy()
                tmp_value_pos.reshape(dim, dim)[game.curr_pos] = 1
                next_state = np.array([game.curr_map.ravel(), tmp_value_pos]).transpose()
                agent.store_episodes(state, action, reward, next_state, done)
                if done in [-1, 1]:
                    if done == 1:
                        total_win = total_win + 1.
                        curr_win = curr_win + 1.
                    ## print exploration map with index as latest step number
                    map_explore = game.curr_map.astype(int)
                    for i in range(len(game.pos_history)):
                        map_explore[game.pos_history[i]] = i + 1
                    map_explore[game.start_pos] = 33 #indicate start position
                    map_explore[game.goal_pos] = 99 #indicate goal position
                    print(map_explore)
                    print ("Episode %d Result: " % (episode) +
                           ("Victory!" if done == 1 else "Defeat!") +
                           (" Total steps: ") +
                           ('[%s]' % ', '.join(map(str, game.pos_history))))
                    print("current success rate: %f; total success rate: %f" % (curr_win / (episode + 1.), total_win / total_games))
                    break

            # Test every 100 episodes and stop early if possible
            if episode % 100 == 0:
                total_reward = 0
                for i in xrange(TEST):

# Path: rlvision/utils.py
# def load_grid_selection(path):
# def process_map_data(path, return_full=False):
# def mask_grid(pos, grid, radius, one_is_free=True):
# def accumulate_map(source_grid, new_grid, one_is_free=True):
# def plot_grid(data, imsize, start=None, pos=None, goal=None, title=None):
# def process_gridworld_data(data_in, imsize):
# def load_mat_data(file_path):
# def find_layout(data, verbose=0):
# def init_h5_db(db_name, save_dir):
# def add_h5_group(group_name, db, db_path=None):
# def add_h5_ds(data, ds_name, db, group_name=None, data_type=np.uint8,
#               renew=False):
# def create_grid_8_dataset(mat_file_path, db_name, save_dir):
# def create_ds_from_splits(mat_file_prefix, db, splits):
# def create_grid_16_dataset(mat_file_prefix, db_name, save_dir, splits=5):
# def create_grid_28_dataset(mat_file_prefix, db_name, save_dir, splits=20):
# def create_grid_40_dataset(mat_file_prefix, db_name, save_dir, splits=100):
# def load_grid8(return_imsize=True):
# def load_grid16(split=None, return_imsize=True):
# def load_grid28(split=None, return_imsize=True):
# def load_grid40(split=None, return_imsize=True):
# 
# Path: rlvision/grid.py
# class Grid(object):
# class GridSampler(object):
# class GridDataSampler(object):
#     def __init__(self, grid_map, value_map, im_size=None,
#                  start_pos=None, is_po=True, mask_radius=3,
#                  grid_type="one-is-free", dstar=False):
#     def get_state(self):
#     def is_pos_valid(self, pos):
#     def rand_start_pos(self):
#     def set_start_pos(self, start_pos):
#     def set_goal_pos(self):
#     def set_curr_pos(self, curr_pos):
#     def update_curr_map(self, map_update, dstar_map_update=None):
#     def get_curr_dstar_visible_map(self, pos):
#     def get_curr_visible_map(self, pos):
#     def get_state_map(self):
#     def update_state(self, pos_update):
#     def action2pos(self, action):
#     def update_state_from_action(self, action, verbose=0):
#     def get_time(self):
#     def get_state_reward(self, max_num_steps=None):
#     def __init__(self, im_data, state_data, label_data, sample_idx, im_size):
#     def find_goal(self, value_grid):
#     def get_grid(self, grid_idx):
#     def compare_grid(self, grid_1, grid_2):
#     def next(self):
#     def __init__(self, grid_data, value_data, im_size, states_xy,
#                  label_data):
#     def compare_pos(self, pos1, pos2):
#     def get_next_state(self, pos, action):
#     def get_goal_pos(self, value_grid):
#     def next(self):
# def sample_data(db, imsize, num_samples=0):
# def sample_data_grid8(num_samples=0):
# def sample_data_grid16(split=None, num_samples=0):
# def sample_data_grid28(split=None, num_samples=0):
# def sample_data_grid40(split=None, num_samples=0):
# def create_train_grid8(db_name, save_dir, num_samples=0):
# def create_train_grid16(db_name, save_dir, num_samples=0):
# def create_train_grid28(db_name, save_dir, num_samples=0):
# def create_train_grid40(db_name, save_dir, num_samples=0):
# def load_train_grid8(return_imsize=True):
# def load_train_grid16(return_imsize=True):
# def load_train_grid28(return_imsize=True):
# def load_train_grid40(return_imsize=True):


# Path: rlvision/exps/dqn_28_exp.py
import tensorflow as tf
import os
import numpy as np
import random
import time
import datetime
import rlvision
from collections import deque
from rlvision import utils
from rlvision import grid

                action = agent.epsilon_greedy(state) # epsilon-greedy for exploration
                game.update_state_from_action(action)
                reward, done = game.get_state_reward()
                # use updated partially-observable map as next state
                tmp_value_pos = value[game_idx].copy()
                tmp_value_pos.reshape(dim, dim)[game.curr_pos] = 1
                next_state = np.array([game.curr_map.ravel(), tmp_value_pos]).transpose()
                agent.store_episodes(state, action, reward, next_state, done)
                if done in [-1, 1]:
                    if done == 1:
                        total_win = total_win + 1.
                        curr_win = curr_win + 1.
                    ## print exploration map with index as latest step number
                    map_explore = game.curr_map.astype(int)
                    for i in range(len(game.pos_history)):
                        map_explore[game.pos_history[i]] = i + 1
                    map_explore[game.start_pos] = 33 #indicate start position
                    map_explore[game.goal_pos] = 99 #indicate goal position
                    print(map_explore)
                    print ("Episode %d Result: " % (episode) +
                           ("Victory!" if done == 1 else "Defeat!") +
                           (" Total steps: ") +
                           ('[%s]' % ', '.join(map(str, game.pos_history))))
                    print("current success rate: %f; total success rate: %f" % (curr_win / (episode + 1.), total_win / total_games))
                    break

            # Test every 100 episodes and stop early if possible
            if episode % 100 == 0:
                total_reward = 0
                for i in xrange(TEST):

                    game = grid.Grid(data[game_idx], value[game_idx], imsize,