def run(self):
    self.learning_model = AgentModel(self.obs_dim, self.n_actions, 32)
    self.critic_model = BaselineModel(self.obs_dim, 32)
    model = copy.deepcopy(self.learning_model)
    self.evaluation_batcher = EpisodeBatcher(n_timesteps=self.config['max_episode_steps'], n_slots=self.config['n_evaluation_episodes'], create_agent=self._create_agent, create_env=self._create_env, env_args={'n_envs': self.config['n_envs'], 'max_episode_steps': self.config['max_episode_steps'], 'env_name': self.config['env_name']}, agent_args={'n_actions': self.n_actions, 'model': model}, n_threads=self.config['n_evaluation_threads'], seeds=[(self.config['env_seed'] + (k * 10)) for k in range(self.config['n_evaluation_threads'])])
    model = copy.deepcopy(self.learning_model)
    self.train_batcher = Batcher(n_timesteps=self.config['a2c_timesteps'], n_slots=(self.config['n_envs'] * self.config['n_threads']), create_agent=self._create_agent, create_env=self._create_train_env, env_args={'n_envs': self.config['n_envs'], 'max_episode_steps': self.config['max_episode_steps'], 'env_name': self.config['env_name']}, agent_args={'n_actions': self.n_actions, 'model': model}, n_threads=self.config['n_threads'], seeds=[(self.config['env_seed'] + (k * 10)) for k in range(self.config['n_threads'])])
    optimizer = torch.optim.Adam(nn.Sequential(self.learning_model, self.critic_model).parameters(), lr=self.config['lr'])
    _start_time = time.time()
    self.iteration = 0
    n_episodes = self.config['n_evaluation_episodes']
    agent_info = DictTensor({'stochastic': torch.tensor([False]).repeat(n_episodes)})
    self.evaluation_batcher.execute(n_episodes=n_episodes, agent_info=agent_info)
    self.evaluation_iteration = self.iteration
    self.train_batcher.update(self.learning_model.state_dict())
    n_episodes = (self.config['n_envs'] * self.config['n_threads'])
    agent_info = DictTensor({'stochastic': torch.tensor([True]).repeat(n_episodes)})
    self.train_batcher.reset(agent_info=agent_info)
    while ((time.time() - _start_time) < self.config['time_limit']):
        self.train_batcher.execute()
        trajectories = self.train_batcher.get(blocking=True)
        dt = self.get_loss(trajectories)
        [self.logger.add_scalar(k, dt[k].item(), self.iteration) for k in dt.keys()]
        ld = (self.config['critic_coef'] * dt['critic_loss'])
        lr = (self.config['a2c_coef'] * dt['a2c_loss'])
        le = (self.config['entropy_coef'] * dt['entropy_loss'])
        floss = ((ld - le) - lr)
        floss = ((floss / n_episodes) * trajectories.n_elems())
        optimizer.zero_grad()
        floss.backward()
        optimizer.step()
        self.train_batcher.update(self.learning_model.state_dict())
        self.iteration += 1
        evaluation_trajectories = self.evaluation_batcher.get(blocking=False)
        if (not (evaluation_trajectories is None)):
            cumulated_reward = (evaluation_trajectories['_reward'] * evaluation_trajectories.mask()).sum(1).mean()
            self.logger.add_scalar('evaluation_reward', cumulated_reward.item(), self.evaluation_iteration)
            print(('At iteration %d, reward is %f' % (self.evaluation_iteration, cumulated_reward.item())))
            self.evaluation_batcher.update(self.learning_model.state_dict())
            self.evaluation_iteration = self.iteration
            self.evaluation_batcher.reexecute()
    self.train_batcher.close()
    self.evaluation_batcher.get()
    self.evaluation_batcher.close()
    self.logger.update_csv()
    self.logger.close()
def run(self):
    self.learning_model = AgentModel(self.obs_dim, self.n_actions, 32)
    self.critic_model = BaselineModel(self.obs_dim, 32)
    model = copy.deepcopy(self.learning_model)
    self.evaluation_batcher = EpisodeBatcher(n_timesteps=self.config['max_episode_steps'], n_slots=self.config['n_evaluation_episodes'], create_agent=self._create_agent, create_env=self._create_env, env_args={'n_envs': self.config['n_envs'], 'max_episode_steps': self.config['max_episode_steps'], 'env_name': self.config['env_name']}, agent_args={'n_actions': self.n_actions, 'model': model}, n_threads=self.config['n_evaluation_threads'], seeds=[(self.config['env_seed'] + (k * 10)) for k in range(self.config['n_evaluation_threads'])])
    model = copy.deepcopy(self.learning_model)
    self.train_batcher = Batcher(n_timesteps=self.config['a2c_timesteps'], n_slots=(self.config['n_envs'] * self.config['n_threads']), create_agent=self._create_agent, create_env=self._create_train_env, env_args={'n_envs': self.config['n_envs'], 'max_episode_steps': self.config['max_episode_steps'], 'env_name': self.config['env_name']}, agent_args={'n_actions': self.n_actions, 'model': model}, n_threads=self.config['n_threads'], seeds=[(self.config['env_seed'] + (k * 10)) for k in range(self.config['n_threads'])])
    optimizer = torch.optim.Adam(nn.Sequential(self.learning_model, self.critic_model).parameters(), lr=self.config['lr'])
    _start_time = time.time()
    self.iteration = 0
    n_episodes = self.config['n_evaluation_episodes']
    agent_info = DictTensor({'stochastic': torch.tensor([False]).repeat(n_episodes)})
    self.evaluation_batcher.execute(n_episodes=n_episodes, agent_info=agent_info)
    self.evaluation_iteration = self.iteration
    self.train_batcher.update(self.learning_model.state_dict())
    n_episodes = (self.config['n_envs'] * self.config['n_threads'])
    agent_info = DictTensor({'stochastic': torch.tensor([True]).repeat(n_episodes)})
    self.train_batcher.reset(agent_info=agent_info)
    while ((time.time() - _start_time) < self.config['time_limit']):
        self.train_batcher.execute()
        trajectories = self.train_batcher.get(blocking=True)
        dt = self.get_loss(trajectories)
        [self.logger.add_scalar(k, dt[k].item(), self.iteration) for k in dt.keys()]
        ld = (self.config['critic_coef'] * dt['critic_loss'])
        lr = (self.config['a2c_coef'] * dt['a2c_loss'])
        le = (self.config['entropy_coef'] * dt['entropy_loss'])
        floss = ((ld - le) - lr)
        floss = ((floss / n_episodes) * trajectories.n_elems())
        optimizer.zero_grad()
        floss.backward()
        optimizer.step()
        self.train_batcher.update(self.learning_model.state_dict())
        self.iteration += 1
        evaluation_trajectories = self.evaluation_batcher.get(blocking=False)
        if (not (evaluation_trajectories is None)):
            cumulated_reward = (evaluation_trajectories['_reward'] * evaluation_trajectories.mask()).sum(1).mean()
            self.logger.add_scalar('evaluation_reward', cumulated_reward.item(), self.evaluation_iteration)
            print(('At iteration %d, reward is %f' % (self.evaluation_iteration, cumulated_reward.item())))
            self.evaluation_batcher.update(self.learning_model.state_dict())
            self.evaluation_iteration = self.iteration
            self.evaluation_batcher.reexecute()
    self.train_batcher.close()
    self.evaluation_batcher.get()
    self.evaluation_batcher.close()
    self.logger.update_csv()
    self.logger.close()