def learn(env, q_func, lr=0.0005, max_timesteps=100000, buffer_size=50000, exploration_fraction=0.1, exploration_final_eps=0.02, train_freq=1, batch_size=32, print_freq=1, checkpoint_freq=10000, learning_starts=1000, gamma=1.0, target_network_update_freq=500, prioritized_replay=False, prioritized_replay_alpha=0.6, prioritized_replay_beta0=0.4, prioritized_replay_beta_iters=None, prioritized_replay_eps=1e-06, num_cpu=16, callback=None):
    'Train a deepq model.\n\n    Parameters\n    -------\n    env : gym.Env\n        environment to train on\n    q_func: (tf.Variable, int, str, bool) -> tf.Variable\n        the model that takes the following inputs:\n            observation_in: object\n                the output of observation placeholder\n            num_actions: int\n                number of actions\n            scope: str\n            reuse: bool\n                should be passed to outer variable scope\n        and returns a tensor of shape (batch_size, num_actions) with values of every action.\n    lr: float\n        learning rate for adam optimizer\n    max_timesteps: int\n        number of env steps to optimizer for\n    buffer_size: int\n        size of the replay buffer\n    exploration_fraction: float\n        fraction of entire training period over which the exploration rate is annealed\n    exploration_final_eps: float\n        final value of random action probability\n    train_freq: int\n        update the model every `train_freq` steps.\n    batch_size: int\n        size of a batched sampled from replay buffer for training\n    print_freq: int\n        how often to print out training progress\n        set to None to disable printing\n    checkpoint_freq: int\n        how often to save the model. This is so that the best version is restored\n        at the end of the training. If you do not wish to restore the best version at\n        the end of the training set this variable to None.\n    learning_starts: int\n        how many steps of the model to collect transitions for before learning starts\n    gamma: float\n        discount factor\n    target_network_update_freq: int\n        update the target network every `target_network_update_freq` steps.\n    prioritized_replay: True\n        if True prioritized replay buffer will be used.\n    prioritized_replay_alpha: float\n        alpha parameter for prioritized replay buffer\n    prioritized_replay_beta0: float\n        initial value of beta for prioritized replay buffer\n    prioritized_replay_beta_iters: int\n        number of iterations over which beta will be annealed from initial value\n        to 1.0. If set to None equals to max_timesteps.\n    prioritized_replay_eps: float\n        epsilon to add to the TD errors when updating priorities.\n    num_cpu: int\n        number of cpus to use for training\n    callback: (locals, globals) -> None\n        function called at every steps with state of the algorithm.\n        If callback returns true training stops.\n\n    Returns\n    -------\n    act: ActWrapper\n        Wrapper over act function. Adds ability to save it and load it.\n        See header of baselines/deepq/categorical.py for details on the act function.\n    '
    sess = U.make_session(num_cpu=num_cpu)
    sess.__enter__()

    def make_obs_ph(name):
        return U.BatchInput(env.observation_space.shape, name=name)
    (act, train, update_target, debug) = build_train(make_obs_ph=make_obs_ph, q_func=q_func, num_actions=env.action_space.n, optimizer=tf.train.AdamOptimizer(learning_rate=lr), gamma=gamma, grad_norm_clipping=10)
    act_params = {'make_obs_ph': make_obs_ph, 'q_func': q_func, 'num_actions': env.action_space.n}
    if prioritized_replay:
        replay_buffer = PrioritizedReplayBuffer(buffer_size, alpha=prioritized_replay_alpha)
        if (prioritized_replay_beta_iters is None):
            prioritized_replay_beta_iters = max_timesteps
        beta_schedule = LinearSchedule(prioritized_replay_beta_iters, initial_p=prioritized_replay_beta0, final_p=1.0)
    else:
        replay_buffer = ReplayBuffer(buffer_size)
        beta_schedule = None
    exploration = LinearSchedule(schedule_timesteps=int((exploration_fraction * max_timesteps)), initial_p=1.0, final_p=exploration_final_eps)
    U.initialize()
    update_target()
    episode_rewards = [0.0]
    saved_mean_reward = None
    obs = env.reset()
    with tempfile.TemporaryDirectory() as td:
        model_saved = False
        model_file = os.path.join(td, 'model')
        for t in range(max_timesteps):
            if (callback is not None):
                if callback(locals(), globals()):
                    break
            action = act(np.array(obs)[None], update_eps=exploration.value(t))[0]
            (new_obs, rew, done, _) = env.step(action)
            replay_buffer.add(obs, action, rew, new_obs, float(done))
            obs = new_obs
            episode_rewards[(- 1)] += rew
            if done:
                obs = env.reset()
                episode_rewards.append(0.0)
            if ((t > learning_starts) and ((t % train_freq) == 0)):
                if prioritized_replay:
                    experience = replay_buffer.sample(batch_size, beta=beta_schedule.value(t))
                    (obses_t, actions, rewards, obses_tp1, dones, weights, batch_idxes) = experience
                else:
                    (obses_t, actions, rewards, obses_tp1, dones) = replay_buffer.sample(batch_size)
                    (weights, batch_idxes) = (np.ones_like(rewards), None)
                td_errors = train(obses_t, actions, rewards, obses_tp1, dones, weights)
                if prioritized_replay:
                    new_priorities = (np.abs(td_errors) + prioritized_replay_eps)
                    replay_buffer.update_priorities(batch_idxes, new_priorities)
            if ((t > learning_starts) and ((t % target_network_update_freq) == 0)):
                update_target()
            mean_100ep_reward = round(np.mean(episode_rewards[(- 101):(- 1)]), 1)
            num_episodes = len(episode_rewards)
            if (done and (print_freq is not None) and ((len(episode_rewards) % print_freq) == 0)):
                logger.record_tabular('steps', t)
                logger.record_tabular('episodes', num_episodes)
                logger.record_tabular('mean 100 episode reward', mean_100ep_reward)
                logger.record_tabular('% time spent exploring', int((100 * exploration.value(t))))
                logger.dump_tabular()
            if ((checkpoint_freq is not None) and (t > learning_starts) and (num_episodes > 100) and ((t % checkpoint_freq) == 0)):
                if ((saved_mean_reward is None) or (mean_100ep_reward > saved_mean_reward)):
                    if (print_freq is not None):
                        logger.log('Saving model due to mean reward increase: {} -> {}'.format(saved_mean_reward, mean_100ep_reward))
                    U.save_state(model_file)
                    model_saved = True
                    saved_mean_reward = mean_100ep_reward
        if model_saved:
            if (print_freq is not None):
                logger.log('Restored model with mean reward: {}'.format(saved_mean_reward))
            U.load_state(model_file)
    return ActWrapper(act, act_params)
Train a deepq model.

Parameters
-------
env : gym.Env
    environment to train on
q_func: (tf.Variable, int, str, bool) -> tf.Variable
    the model that takes the following inputs:
        observation_in: object
            the output of observation placeholder
        num_actions: int
            number of actions
        scope: str
        reuse: bool
            should be passed to outer variable scope
    and returns a tensor of shape (batch_size, num_actions) with values of every action.
lr: float
    learning rate for adam optimizer
max_timesteps: int
    number of env steps to optimizer for
buffer_size: int
    size of the replay buffer
exploration_fraction: float
    fraction of entire training period over which the exploration rate is annealed
exploration_final_eps: float
    final value of random action probability
train_freq: int
    update the model every `train_freq` steps.
batch_size: int
    size of a batched sampled from replay buffer for training
print_freq: int
    how often to print out training progress
    set to None to disable printing
checkpoint_freq: int
    how often to save the model. This is so that the best version is restored
    at the end of the training. If you do not wish to restore the best version at
    the end of the training set this variable to None.
learning_starts: int
    how many steps of the model to collect transitions for before learning starts
gamma: float
    discount factor
target_network_update_freq: int
    update the target network every `target_network_update_freq` steps.
prioritized_replay: True
    if True prioritized replay buffer will be used.
prioritized_replay_alpha: float
    alpha parameter for prioritized replay buffer
prioritized_replay_beta0: float
    initial value of beta for prioritized replay buffer
prioritized_replay_beta_iters: int
    number of iterations over which beta will be annealed from initial value
    to 1.0. If set to None equals to max_timesteps.
prioritized_replay_eps: float
    epsilon to add to the TD errors when updating priorities.
num_cpu: int
    number of cpus to use for training
callback: (locals, globals) -> None
    function called at every steps with state of the algorithm.
    If callback returns true training stops.

Returns
-------
act: ActWrapper
    Wrapper over act function. Adds ability to save it and load it.
    See header of baselines/deepq/categorical.py for details on the act function.
def learn(env, q_func, lr=0.0005, max_timesteps=100000, buffer_size=50000, exploration_fraction=0.1, exploration_final_eps=0.02, train_freq=1, batch_size=32, print_freq=1, checkpoint_freq=10000, learning_starts=1000, gamma=1.0, target_network_update_freq=500, prioritized_replay=False, prioritized_replay_alpha=0.6, prioritized_replay_beta0=0.4, prioritized_replay_beta_iters=None, prioritized_replay_eps=1e-06, num_cpu=16, callback=None):
    'Train a deepq model.\n\n    Parameters\n    -------\n    env : gym.Env\n        environment to train on\n    q_func: (tf.Variable, int, str, bool) -> tf.Variable\n        the model that takes the following inputs:\n            observation_in: object\n                the output of observation placeholder\n            num_actions: int\n                number of actions\n            scope: str\n            reuse: bool\n                should be passed to outer variable scope\n        and returns a tensor of shape (batch_size, num_actions) with values of every action.\n    lr: float\n        learning rate for adam optimizer\n    max_timesteps: int\n        number of env steps to optimizer for\n    buffer_size: int\n        size of the replay buffer\n    exploration_fraction: float\n        fraction of entire training period over which the exploration rate is annealed\n    exploration_final_eps: float\n        final value of random action probability\n    train_freq: int\n        update the model every `train_freq` steps.\n    batch_size: int\n        size of a batched sampled from replay buffer for training\n    print_freq: int\n        how often to print out training progress\n        set to None to disable printing\n    checkpoint_freq: int\n        how often to save the model. This is so that the best version is restored\n        at the end of the training. If you do not wish to restore the best version at\n        the end of the training set this variable to None.\n    learning_starts: int\n        how many steps of the model to collect transitions for before learning starts\n    gamma: float\n        discount factor\n    target_network_update_freq: int\n        update the target network every `target_network_update_freq` steps.\n    prioritized_replay: True\n        if True prioritized replay buffer will be used.\n    prioritized_replay_alpha: float\n        alpha parameter for prioritized replay buffer\n    prioritized_replay_beta0: float\n        initial value of beta for prioritized replay buffer\n    prioritized_replay_beta_iters: int\n        number of iterations over which beta will be annealed from initial value\n        to 1.0. If set to None equals to max_timesteps.\n    prioritized_replay_eps: float\n        epsilon to add to the TD errors when updating priorities.\n    num_cpu: int\n        number of cpus to use for training\n    callback: (locals, globals) -> None\n        function called at every steps with state of the algorithm.\n        If callback returns true training stops.\n\n    Returns\n    -------\n    act: ActWrapper\n        Wrapper over act function. Adds ability to save it and load it.\n        See header of baselines/deepq/categorical.py for details on the act function.\n    '
    sess = U.make_session(num_cpu=num_cpu)
    sess.__enter__()

    def make_obs_ph(name):
        return U.BatchInput(env.observation_space.shape, name=name)
    (act, train, update_target, debug) = build_train(make_obs_ph=make_obs_ph, q_func=q_func, num_actions=env.action_space.n, optimizer=tf.train.AdamOptimizer(learning_rate=lr), gamma=gamma, grad_norm_clipping=10)
    act_params = {'make_obs_ph': make_obs_ph, 'q_func': q_func, 'num_actions': env.action_space.n}
    if prioritized_replay:
        replay_buffer = PrioritizedReplayBuffer(buffer_size, alpha=prioritized_replay_alpha)
        if (prioritized_replay_beta_iters is None):
            prioritized_replay_beta_iters = max_timesteps
        beta_schedule = LinearSchedule(prioritized_replay_beta_iters, initial_p=prioritized_replay_beta0, final_p=1.0)
    else:
        replay_buffer = ReplayBuffer(buffer_size)
        beta_schedule = None
    exploration = LinearSchedule(schedule_timesteps=int((exploration_fraction * max_timesteps)), initial_p=1.0, final_p=exploration_final_eps)
    U.initialize()
    update_target()
    episode_rewards = [0.0]
    saved_mean_reward = None
    obs = env.reset()
    with tempfile.TemporaryDirectory() as td:
        model_saved = False
        model_file = os.path.join(td, 'model')
        for t in range(max_timesteps):
            if (callback is not None):
                if callback(locals(), globals()):
                    break
            action = act(np.array(obs)[None], update_eps=exploration.value(t))[0]
            (new_obs, rew, done, _) = env.step(action)
            replay_buffer.add(obs, action, rew, new_obs, float(done))
            obs = new_obs
            episode_rewards[(- 1)] += rew
            if done:
                obs = env.reset()
                episode_rewards.append(0.0)
            if ((t > learning_starts) and ((t % train_freq) == 0)):
                if prioritized_replay:
                    experience = replay_buffer.sample(batch_size, beta=beta_schedule.value(t))
                    (obses_t, actions, rewards, obses_tp1, dones, weights, batch_idxes) = experience
                else:
                    (obses_t, actions, rewards, obses_tp1, dones) = replay_buffer.sample(batch_size)
                    (weights, batch_idxes) = (np.ones_like(rewards), None)
                td_errors = train(obses_t, actions, rewards, obses_tp1, dones, weights)
                if prioritized_replay:
                    new_priorities = (np.abs(td_errors) + prioritized_replay_eps)
                    replay_buffer.update_priorities(batch_idxes, new_priorities)
            if ((t > learning_starts) and ((t % target_network_update_freq) == 0)):
                update_target()
            mean_100ep_reward = round(np.mean(episode_rewards[(- 101):(- 1)]), 1)
            num_episodes = len(episode_rewards)
            if (done and (print_freq is not None) and ((len(episode_rewards) % print_freq) == 0)):
                logger.record_tabular('steps', t)
                logger.record_tabular('episodes', num_episodes)
                logger.record_tabular('mean 100 episode reward', mean_100ep_reward)
                logger.record_tabular('% time spent exploring', int((100 * exploration.value(t))))
                logger.dump_tabular()
            if ((checkpoint_freq is not None) and (t > learning_starts) and (num_episodes > 100) and ((t % checkpoint_freq) == 0)):
                if ((saved_mean_reward is None) or (mean_100ep_reward > saved_mean_reward)):
                    if (print_freq is not None):
                        logger.log('Saving model due to mean reward increase: {} -> {}'.format(saved_mean_reward, mean_100ep_reward))
                    U.save_state(model_file)
                    model_saved = True
                    saved_mean_reward = mean_100ep_reward
        if model_saved:
            if (print_freq is not None):
                logger.log('Restored model with mean reward: {}'.format(saved_mean_reward))
            U.load_state(model_file)
    return ActWrapper(act, act_params)