def train(self, env, episode_count, render=False):
    mean_step_all = []
    mean_q_all = []
    goal_time_all = []
    reward_all = []
    self.agent.epsilon = self.epsilon
    values = []
    steps = deque(maxlen=100)
    lr = self.learning_rate
    for i in range(episode_count):
        reward_total = 0
        goal_time = 0
        obs = env.reset()
        step = 0
        done = False
        while (not done):
            if render:
                env.render()
            action = self.agent.act(obs)
            (next_obs, reward, done, goal_time) = env.step(action)
            reward_total += reward
            goal_time += goal_time
            state = self.agent.q.observation_to_state(obs)
            future = (0 if done else np.max(self.agent.q.values(next_obs)))
            value = self.agent.q.table[state][action]
            self.agent.q.table[state][action] += (lr * ((reward + (self.gamma * future)) - value))
            obs = next_obs
            values.append(value)
            step += 1
            if ((self.max_step > 0) and (step > self.max_step)):
                done = True
        else:
            mean = np.mean(values)
            steps.append(step)
            mean_step = np.mean(steps)
            print('Episode {}: {}steps(avg{}). epsilon={:.3f}, lr={:.3f}, mean q value={:.2f}'.format(i, step, mean_step, self.agent.epsilon, lr, mean))
            mean_step_all.append(mean_step)
            mean_q_all.append(mean)
            reward_all.append(reward_total)
            if (mean_step > 1000):
                render = True
            if (self.epsilon_decay is not None):
                self.agent.epsilon = self.epsilon_decay(self.agent.epsilon, i)
            if (self.learning_rate_decay is not None):
                lr = self.learning_rate_decay(lr, i)
    plt.xlabel('Episodes')
    plt.ylabel('reward')
    plt.plot(reward_all, label='Q-learning', color='yellow')
    plt.plot(goal_time_all, label='Q-learning', color='green')
    plt.title('reward/Episode')
    plt.show()
    plt.xlabel('Episodes')
    plt.ylabel('goal_time')
    plt.plot(goal_time_all, label='Q-learning', color='green')
    plt.title('goal/Episode')
    plt.show()
def train(self, env, episode_count, render=False):
    mean_step_all = []
    mean_q_all = []
    goal_time_all = []
    reward_all = []
    self.agent.epsilon = self.epsilon
    values = []
    steps = deque(maxlen=100)
    lr = self.learning_rate
    for i in range(episode_count):
        reward_total = 0
        goal_time = 0
        obs = env.reset()
        step = 0
        done = False
        while (not done):
            if render:
                env.render()
            action = self.agent.act(obs)
            (next_obs, reward, done, goal_time) = env.step(action)
            reward_total += reward
            goal_time += goal_time
            state = self.agent.q.observation_to_state(obs)
            future = (0 if done else np.max(self.agent.q.values(next_obs)))
            value = self.agent.q.table[state][action]
            self.agent.q.table[state][action] += (lr * ((reward + (self.gamma * future)) - value))
            obs = next_obs
            values.append(value)
            step += 1
            if ((self.max_step > 0) and (step > self.max_step)):
                done = True
        else:
            mean = np.mean(values)
            steps.append(step)
            mean_step = np.mean(steps)
            print('Episode {}: {}steps(avg{}). epsilon={:.3f}, lr={:.3f}, mean q value={:.2f}'.format(i, step, mean_step, self.agent.epsilon, lr, mean))
            mean_step_all.append(mean_step)
            mean_q_all.append(mean)
            reward_all.append(reward_total)
            if (mean_step > 1000):
                render = True
            if (self.epsilon_decay is not None):
                self.agent.epsilon = self.epsilon_decay(self.agent.epsilon, i)
            if (self.learning_rate_decay is not None):
                lr = self.learning_rate_decay(lr, i)
    plt.xlabel('Episodes')
    plt.ylabel('reward')
    plt.plot(reward_all, label='Q-learning', color='yellow')
    plt.plot(goal_time_all, label='Q-learning', color='green')
    plt.title('reward/Episode')
    plt.show()
    plt.xlabel('Episodes')
    plt.ylabel('goal_time')
    plt.plot(goal_time_all, label='Q-learning', color='green')
    plt.title('goal/Episode')
    plt.show()