def rollout(self, subpolicy, failure_observations):
    "\n\t\t\tThis is where we collect the batch of data\n\t\t\tfrom simulation. Since this is an on-policy algorithm, we'll need to collect a fresh batch\n\t\t\tof data each time we iterate the actor/critic networks.\n\n\t\t\tParameters:\n\t\t\t\tNone\n\n\t\t\tReturn:\n\t\t\t\tbatch_obs - the observations collected this batch. Shape: (number of timesteps, dimension of observation)\n\t\t\t\tbatch_acts - the actions collected this batch. Shape: (number of timesteps, dimension of action)\n\t\t\t\tbatch_log_probs - the log probabilities of each action taken this batch. Shape: (number of timesteps)\n\t\t\t\tbatch_rtgs - the Rewards-To-Go of each timestep in this batch. Shape: (number of timesteps)\n\t\t\t\tbatch_lens - the lengths of each episode this batch. Shape: (number of episodes)\n\t\t"
    batch_obs = []
    batch_acts = []
    batch_log_probs = []
    batch_rews = []
    batch_rtgs = []
    batch_lens = []
    batch_infractions = []
    ep_rews = []
    t = 0
    while (t < self.timesteps_per_batch):
        act_list = []
        ep_rews = []
        obs = self.env.reset()
        done = False
        count_infractions = 0
        count_infractions_acc = 0
        count_infractions_steer = 0
        for ep_t in range(self.max_timesteps_per_episode):
            a_predicted_clf = clf_control(self.env.v_ego)
            (delta, target_id, crosstrack_error) = self.env.car.tracker.stanley_control(self.env.x_ego, self.env.y_ego, self.env.yaw_ego, self.env.v_ego, self.env.delta_ego)
            if self.render:
                self.env.render()
            t += 1
            batch_obs.append(obs)
            if self.discrete:
                (action, log_prob) = self.get_action_discrete(obs)
            else:
                (action, log_prob) = self.get_action(obs)
            if (abs(round(float(action[0]), 1)) < abs(round(float(a_predicted_clf), 1))):
                count_infractions_acc = (count_infractions_acc + 1)
            if (abs(round(float(action[1]), 1)) < (abs(round(float(delta), 1)) - 0.2)):
                count_infractions_steer = (count_infractions_steer + 1)
            (obs, rew, done, info) = self.env.step(action)
            count_infractions = (count_infractions_acc + count_infractions_steer)
            ep_rews.append(rew)
            batch_acts.append(action)
            batch_log_probs.append(log_prob)
            act_list.append(info)
            if done:
                break
        batch_lens.append((ep_t + 1))
        batch_rews.append(ep_rews)
        batch_infractions.append(count_infractions)
    batch_obs = torch.tensor(batch_obs, dtype=torch.float)
    if self.discrete:
        batch_acts = torch.tensor(batch_acts, dtype=torch.long).view((- 1))
    else:
        batch_acts = torch.tensor(batch_acts, dtype=torch.float)
    batch_log_probs = torch.tensor(batch_log_probs, dtype=torch.float)
    batch_rtgs = self.compute_rtgs(batch_rews)
    self.logger['batch_rews'] = batch_rews
    self.logger['batch_lens'] = batch_lens
    self.logger['batch_infractions'] = batch_infractions
    return (batch_obs, batch_acts, batch_log_probs, batch_rtgs, batch_lens)
This is where we collect the batch of data
from simulation. Since this is an on-policy algorithm, we'll need to collect a fresh batch
of data each time we iterate the actor/critic networks.

Parameters:
        None

Return:
        batch_obs - the observations collected this batch. Shape: (number of timesteps, dimension of observation)
        batch_acts - the actions collected this batch. Shape: (number of timesteps, dimension of action)
        batch_log_probs - the log probabilities of each action taken this batch. Shape: (number of timesteps)
        batch_rtgs - the Rewards-To-Go of each timestep in this batch. Shape: (number of timesteps)
        batch_lens - the lengths of each episode this batch. Shape: (number of episodes)
def rollout(self, subpolicy, failure_observations):
    "\n\t\t\tThis is where we collect the batch of data\n\t\t\tfrom simulation. Since this is an on-policy algorithm, we'll need to collect a fresh batch\n\t\t\tof data each time we iterate the actor/critic networks.\n\n\t\t\tParameters:\n\t\t\t\tNone\n\n\t\t\tReturn:\n\t\t\t\tbatch_obs - the observations collected this batch. Shape: (number of timesteps, dimension of observation)\n\t\t\t\tbatch_acts - the actions collected this batch. Shape: (number of timesteps, dimension of action)\n\t\t\t\tbatch_log_probs - the log probabilities of each action taken this batch. Shape: (number of timesteps)\n\t\t\t\tbatch_rtgs - the Rewards-To-Go of each timestep in this batch. Shape: (number of timesteps)\n\t\t\t\tbatch_lens - the lengths of each episode this batch. Shape: (number of episodes)\n\t\t"
    batch_obs = []
    batch_acts = []
    batch_log_probs = []
    batch_rews = []
    batch_rtgs = []
    batch_lens = []
    batch_infractions = []
    ep_rews = []
    t = 0
    while (t < self.timesteps_per_batch):
        act_list = []
        ep_rews = []
        obs = self.env.reset()
        done = False
        count_infractions = 0
        count_infractions_acc = 0
        count_infractions_steer = 0
        for ep_t in range(self.max_timesteps_per_episode):
            a_predicted_clf = clf_control(self.env.v_ego)
            (delta, target_id, crosstrack_error) = self.env.car.tracker.stanley_control(self.env.x_ego, self.env.y_ego, self.env.yaw_ego, self.env.v_ego, self.env.delta_ego)
            if self.render:
                self.env.render()
            t += 1
            batch_obs.append(obs)
            if self.discrete:
                (action, log_prob) = self.get_action_discrete(obs)
            else:
                (action, log_prob) = self.get_action(obs)
            if (abs(round(float(action[0]), 1)) < abs(round(float(a_predicted_clf), 1))):
                count_infractions_acc = (count_infractions_acc + 1)
            if (abs(round(float(action[1]), 1)) < (abs(round(float(delta), 1)) - 0.2)):
                count_infractions_steer = (count_infractions_steer + 1)
            (obs, rew, done, info) = self.env.step(action)
            count_infractions = (count_infractions_acc + count_infractions_steer)
            ep_rews.append(rew)
            batch_acts.append(action)
            batch_log_probs.append(log_prob)
            act_list.append(info)
            if done:
                break
        batch_lens.append((ep_t + 1))
        batch_rews.append(ep_rews)
        batch_infractions.append(count_infractions)
    batch_obs = torch.tensor(batch_obs, dtype=torch.float)
    if self.discrete:
        batch_acts = torch.tensor(batch_acts, dtype=torch.long).view((- 1))
    else:
        batch_acts = torch.tensor(batch_acts, dtype=torch.float)
    batch_log_probs = torch.tensor(batch_log_probs, dtype=torch.float)
    batch_rtgs = self.compute_rtgs(batch_rews)
    self.logger['batch_rews'] = batch_rews
    self.logger['batch_lens'] = batch_lens
    self.logger['batch_infractions'] = batch_infractions
    return (batch_obs, batch_acts, batch_log_probs, batch_rtgs, batch_lens)