def build_graph(self):
    '\n        Creates the computation graph\n        '
    ' Create Variables '
    with tf.variable_scope(self.name):
        self.step_sizes = self._create_step_size_vars()
        ' --- Build inner update graph for adapting the policy and sampling trajectories --- '
        (self.adapted_policies_params, self.adapt_input_ph_dict) = self._build_inner_adaption()
        ' ----- Build graph for the meta-update ----- '
        self.meta_op_phs_dict = OrderedDict()
        (obs_phs, action_phs, adv_phs, dist_info_old_phs, all_phs_dict) = self._make_input_placeholders('step0')
        self.meta_op_phs_dict.update(all_phs_dict)
        (distribution_info_vars, current_policy_params) = ([], [])
        (all_surr_objs, all_inner_kls) = ([], [])
    for i in range(self.meta_batch_size):
        dist_info_sym = self.policy.distribution_info_sym(obs_phs[i], params=None)
        distribution_info_vars.append(dist_info_sym)
        current_policy_params.append(self.policy.policy_params)
    initial_distribution_info_vars = distribution_info_vars
    initial_action_phs = action_phs
    with tf.variable_scope(self.name):
        ' Inner updates'
        for step_id in range(1, (self.num_inner_grad_steps + 1)):
            (surr_objs, adapted_policy_params) = ([], [])
            for i in range(self.meta_batch_size):
                surr_loss = self._adapt_objective_sym(action_phs[i], adv_phs[i], dist_info_old_phs[i], distribution_info_vars[i])
                adapted_params_var = self._adapt_sym(surr_loss, current_policy_params[i])
                adapted_policy_params.append(adapted_params_var)
                surr_objs.append(surr_loss)
            all_surr_objs.append(surr_objs)
            (obs_phs, action_phs, adv_phs, dist_info_old_phs, all_phs_dict) = self._make_input_placeholders(('step%i' % step_id))
            self.meta_op_phs_dict.update(all_phs_dict)
            distribution_info_vars = [self.policy.distribution_info_sym(obs_phs[i], params=adapted_policy_params[i]) for i in range(self.meta_batch_size)]
            current_policy_params = adapted_policy_params
        ' Outer objective '
        (surr_objs, outer_kls) = ([], [])
        for i in range(self.meta_batch_size):
            likelihood_ratio = self.policy.distribution.likelihood_ratio_sym(action_phs[i], dist_info_old_phs[i], distribution_info_vars[i])
            outer_kl = tf.reduce_mean(self.policy.distribution.kl_sym(dist_info_old_phs[i], distribution_info_vars[i]))
            surr_obj = (- tf.reduce_mean((likelihood_ratio * adv_phs[i])))
            if self.exploration:
                adj_avg_rewards = tf.placeholder(dtype=tf.float32, shape=[None], name=(((('adj_avg_rewards' + '_') + str(self.num_inner_grad_steps)) + '_') + str(i)))
                self.meta_op_phs_dict[('step%i_task%i_%s' % (self.num_inner_grad_steps, i, 'adj_avg_rewards'))] = adj_avg_rewards
                log_likelihood_inital = self.policy.distribution.log_likelihood_sym(initial_action_phs[i], initial_distribution_info_vars[i])
                surr_obj += ((- tf.reduce_mean(adj_avg_rewards)) * tf.reduce_mean(log_likelihood_inital))
            surr_objs.append(surr_obj)
            outer_kls.append(outer_kl)
        mean_outer_kl = tf.reduce_mean(tf.stack(outer_kls))
        ' Mean over meta tasks '
        meta_objective = tf.reduce_mean(tf.stack(surr_objs, 0))
        self.optimizer.build_graph(loss=meta_objective, target=self.policy, input_ph_dict=self.meta_op_phs_dict, leq_constraint=(mean_outer_kl, self.step_size))
Creates the computation graph
def build_graph(self):
    '\n        \n        '
    ' Create Variables '
    with tf.variable_scope(self.name):
        self.step_sizes = self._create_step_size_vars()
        ' --- Build inner update graph for adapting the policy and sampling trajectories --- '
        (self.adapted_policies_params, self.adapt_input_ph_dict) = self._build_inner_adaption()
        ' ----- Build graph for the meta-update ----- '
        self.meta_op_phs_dict = OrderedDict()
        (obs_phs, action_phs, adv_phs, dist_info_old_phs, all_phs_dict) = self._make_input_placeholders('step0')
        self.meta_op_phs_dict.update(all_phs_dict)
        (distribution_info_vars, current_policy_params) = ([], [])
        (all_surr_objs, all_inner_kls) = ([], [])
    for i in range(self.meta_batch_size):
        dist_info_sym = self.policy.distribution_info_sym(obs_phs[i], params=None)
        distribution_info_vars.append(dist_info_sym)
        current_policy_params.append(self.policy.policy_params)
    initial_distribution_info_vars = distribution_info_vars
    initial_action_phs = action_phs
    with tf.variable_scope(self.name):
        ' Inner updates'
        for step_id in range(1, (self.num_inner_grad_steps + 1)):
            (surr_objs, adapted_policy_params) = ([], [])
            for i in range(self.meta_batch_size):
                surr_loss = self._adapt_objective_sym(action_phs[i], adv_phs[i], dist_info_old_phs[i], distribution_info_vars[i])
                adapted_params_var = self._adapt_sym(surr_loss, current_policy_params[i])
                adapted_policy_params.append(adapted_params_var)
                surr_objs.append(surr_loss)
            all_surr_objs.append(surr_objs)
            (obs_phs, action_phs, adv_phs, dist_info_old_phs, all_phs_dict) = self._make_input_placeholders(('step%i' % step_id))
            self.meta_op_phs_dict.update(all_phs_dict)
            distribution_info_vars = [self.policy.distribution_info_sym(obs_phs[i], params=adapted_policy_params[i]) for i in range(self.meta_batch_size)]
            current_policy_params = adapted_policy_params
        ' Outer objective '
        (surr_objs, outer_kls) = ([], [])
        for i in range(self.meta_batch_size):
            likelihood_ratio = self.policy.distribution.likelihood_ratio_sym(action_phs[i], dist_info_old_phs[i], distribution_info_vars[i])
            outer_kl = tf.reduce_mean(self.policy.distribution.kl_sym(dist_info_old_phs[i], distribution_info_vars[i]))
            surr_obj = (- tf.reduce_mean((likelihood_ratio * adv_phs[i])))
            if self.exploration:
                adj_avg_rewards = tf.placeholder(dtype=tf.float32, shape=[None], name=(((('adj_avg_rewards' + '_') + str(self.num_inner_grad_steps)) + '_') + str(i)))
                self.meta_op_phs_dict[('step%i_task%i_%s' % (self.num_inner_grad_steps, i, 'adj_avg_rewards'))] = adj_avg_rewards
                log_likelihood_inital = self.policy.distribution.log_likelihood_sym(initial_action_phs[i], initial_distribution_info_vars[i])
                surr_obj += ((- tf.reduce_mean(adj_avg_rewards)) * tf.reduce_mean(log_likelihood_inital))
            surr_objs.append(surr_obj)
            outer_kls.append(outer_kl)
        mean_outer_kl = tf.reduce_mean(tf.stack(outer_kls))
        ' Mean over meta tasks '
        meta_objective = tf.reduce_mean(tf.stack(surr_objs, 0))
        self.optimizer.build_graph(loss=meta_objective, target=self.policy, input_ph_dict=self.meta_op_phs_dict, leq_constraint=(mean_outer_kl, self.step_size))