def train(coin_name, time_unit, train_env, evaluate_env):
    common_utils.print_fast_rl_params(params)
    params.BATCH_SIZE *= params.TRAIN_STEP_FREQ
    net = value_based_model.DuelingDQNSmallCNN(observation_shape=train_env.observation_space.shape, n_actions=train_env.action_space.n).to(device)
    print(net)
    print('ACTION MEANING: {0}'.format(train_env.get_action_meanings()))
    tgt_net = value_based_model.DuelingDQNSmallCNN(observation_shape=train_env.observation_space.shape, n_actions=train_env.action_space.n).to(device)
    action_selector = EpsilonGreedyTradeDQNActionSelector(epsilon=params.EPSILON_INIT, env=train_env)
    agent = rl_agent.DQNAgent(dqn_model=net, action_selector=action_selector, device=device)
    argmax_action_selector = ArgmaxTradeActionSelector(env=evaluate_env)
    evaluate_agent = rl_agent.DQNAgent(dqn_model=net, action_selector=argmax_action_selector, device=device)
    random_action_selector = RandomTradeDQNActionSelector(env=evaluate_env)
    random_agent = rl_agent.DQNAgent(dqn_model=None, action_selector=random_action_selector, device=device)
    experience_source = experience_single.ExperienceSourceSingleEnvFirstLast(train_env, agent, gamma=params.GAMMA, n_step=params.N_STEP)
    buffer = replay_buffer.ExperienceReplayBuffer(experience_source, buffer_size=params.REPLAY_BUFFER_SIZE)
    optimizer = optim.Adam(net.parameters(), lr=params.LEARNING_RATE)
    step_idx = 0
    last_loss = 0.0
    evaluate_steps = []
    evaluate_dqn_total_profits = []
    evaluate_random_total_profits = []
    early_stopping = EarlyStopping(patience=params.STOP_PATIENCE_COUNT, evaluation_min_threshold=params.TRAIN_STOP_EPISODE_REWARD, verbose=True, delta=0.0, model_save_dir=MODEL_SAVE_DIR, model_save_file_prefix=((((params.ENVIRONMENT_ID.value + '_') + coin_name) + '_') + time_unit.value), agent=agent)
    with utils.SpeedTracker(params=params, frame=False, early_stopping=None) as reward_tracker:
        while (step_idx < params.MAX_GLOBAL_STEP):
            step_idx += params.TRAIN_STEP_FREQ
            last_entry = buffer.populate(params.TRAIN_STEP_FREQ)
            if epsilon_tracker:
                epsilon_tracker.udpate(step_idx)
            episode_rewards = experience_source.pop_episode_reward_lst()
            solved = False
            if episode_rewards:
                for episode_reward in episode_rewards:
                    reward_tracker.set_episode_reward(episode_reward, step_idx, action_selector.epsilon, last_info=last_entry.info, last_loss=last_loss, model=net)
                    if ((reward_tracker.done_episodes % params.TEST_PERIOD_EPISODE) == 0):
                        print(('#' * 200))
                        print('[TEST START]')
                        evaluate(evaluate_env, evaluate_agent)
                        evaluate_steps.append(step_idx)
                        (dqn_total_profit, _) = evaluate_random('DQN', evaluate_env, evaluate_agent, num_episodes=100)
                        evaluate_dqn_total_profits.append(dqn_total_profit)
                        (random_total_profit, _) = evaluate_random('RANDOM', evaluate_env, random_agent, num_episodes=100)
                        evaluate_random_total_profits.append(random_total_profit)
                        solved = early_stopping(dqn_total_profit, step_idx=step_idx)
                        visualizer.draw_performance(evaluate_steps, evaluate_dqn_total_profits, evaluate_random_total_profits)
                        print('[TEST END]')
                        print(('#' * 200))
                    if solved:
                        break
            if solved:
                break
            optimizer.zero_grad()
            batch = buffer.sample(params.BATCH_SIZE)
            loss_v = value_based_model.calc_loss_double_dqn(batch, net, tgt_net, gamma=params.GAMMA, device=device)
            loss_v.backward()
            optimizer.step()
            draw_loss = min(1.0, loss_v.detach().item())
            last_loss = loss_v.detach().item()
            if ((step_idx % params.TARGET_NET_SYNC_STEP_PERIOD) < params.TRAIN_STEP_FREQ):
                tgt_net.sync(net)
    return net
def train(coin_name, time_unit, train_env, evaluate_env):
    common_utils.print_fast_rl_params(params)
    params.BATCH_SIZE *= params.TRAIN_STEP_FREQ
    net = value_based_model.DuelingDQNSmallCNN(observation_shape=train_env.observation_space.shape, n_actions=train_env.action_space.n).to(device)
    print(net)
    print('ACTION MEANING: {0}'.format(train_env.get_action_meanings()))
    tgt_net = value_based_model.DuelingDQNSmallCNN(observation_shape=train_env.observation_space.shape, n_actions=train_env.action_space.n).to(device)
    action_selector = EpsilonGreedyTradeDQNActionSelector(epsilon=params.EPSILON_INIT, env=train_env)
    agent = rl_agent.DQNAgent(dqn_model=net, action_selector=action_selector, device=device)
    argmax_action_selector = ArgmaxTradeActionSelector(env=evaluate_env)
    evaluate_agent = rl_agent.DQNAgent(dqn_model=net, action_selector=argmax_action_selector, device=device)
    random_action_selector = RandomTradeDQNActionSelector(env=evaluate_env)
    random_agent = rl_agent.DQNAgent(dqn_model=None, action_selector=random_action_selector, device=device)
    experience_source = experience_single.ExperienceSourceSingleEnvFirstLast(train_env, agent, gamma=params.GAMMA, n_step=params.N_STEP)
    buffer = replay_buffer.ExperienceReplayBuffer(experience_source, buffer_size=params.REPLAY_BUFFER_SIZE)
    optimizer = optim.Adam(net.parameters(), lr=params.LEARNING_RATE)
    step_idx = 0
    last_loss = 0.0
    evaluate_steps = []
    evaluate_dqn_total_profits = []
    evaluate_random_total_profits = []
    early_stopping = EarlyStopping(patience=params.STOP_PATIENCE_COUNT, evaluation_min_threshold=params.TRAIN_STOP_EPISODE_REWARD, verbose=True, delta=0.0, model_save_dir=MODEL_SAVE_DIR, model_save_file_prefix=((((params.ENVIRONMENT_ID.value + '_') + coin_name) + '_') + time_unit.value), agent=agent)
    with utils.SpeedTracker(params=params, frame=False, early_stopping=None) as reward_tracker:
        while (step_idx < params.MAX_GLOBAL_STEP):
            step_idx += params.TRAIN_STEP_FREQ
            last_entry = buffer.populate(params.TRAIN_STEP_FREQ)
            if epsilon_tracker:
                epsilon_tracker.udpate(step_idx)
            episode_rewards = experience_source.pop_episode_reward_lst()
            solved = False
            if episode_rewards:
                for episode_reward in episode_rewards:
                    reward_tracker.set_episode_reward(episode_reward, step_idx, action_selector.epsilon, last_info=last_entry.info, last_loss=last_loss, model=net)
                    if ((reward_tracker.done_episodes % params.TEST_PERIOD_EPISODE) == 0):
                        print(('#' * 200))
                        print('[TEST START]')
                        evaluate(evaluate_env, evaluate_agent)
                        evaluate_steps.append(step_idx)
                        (dqn_total_profit, _) = evaluate_random('DQN', evaluate_env, evaluate_agent, num_episodes=100)
                        evaluate_dqn_total_profits.append(dqn_total_profit)
                        (random_total_profit, _) = evaluate_random('RANDOM', evaluate_env, random_agent, num_episodes=100)
                        evaluate_random_total_profits.append(random_total_profit)
                        solved = early_stopping(dqn_total_profit, step_idx=step_idx)
                        visualizer.draw_performance(evaluate_steps, evaluate_dqn_total_profits, evaluate_random_total_profits)
                        print('[TEST END]')
                        print(('#' * 200))
                    if solved:
                        break
            if solved:
                break
            optimizer.zero_grad()
            batch = buffer.sample(params.BATCH_SIZE)
            loss_v = value_based_model.calc_loss_double_dqn(batch, net, tgt_net, gamma=params.GAMMA, device=device)
            loss_v.backward()
            optimizer.step()
            draw_loss = min(1.0, loss_v.detach().item())
            last_loss = loss_v.detach().item()
            if ((step_idx % params.TARGET_NET_SYNC_STEP_PERIOD) < params.TRAIN_STEP_FREQ):
                tgt_net.sync(net)
    return net