<commit_before>import numpy as np
from mushroom.algorithms.policy_search import PolicyGradient


class eNAC(PolicyGradient):
    """
    ENAC algorithm.
    "Policy Gradient Methods for Robotics", <NAME>., <NAME>.  2006.

    """
    def __init__(self, policy, mdp_info, params, features):
        self.__name__ = 'eNAC'

        super(eNAC, self).__init__(policy, mdp_info, params, features)

        self.psi = None
        self.fisher_list = list()
        self.grad_list = list()
        self.eligibility_list = list()

    def _compute_gradient(self, J):
        n_ep = len(self.fisher_list)

        fisher = np.mean(self.fisher_list, axis=0)
        g = np.mean(self.grad_list, axis=0)
        eligibility = np.mean(self.eligibility_list, axis=0)
        Jpol = np.mean(J)

        if fisher.shape[0] == np.linalg.matrix_rank(fisher):
            tmp = np.linalg.solve(n_ep*fisher-np.outer(eligibility, eligibility), eligibility)
            Q = (1 + eligibility.dot(tmp)) / n_ep
            b = Q.dot(Jpol - eligibility.dot(np.linalg.solve(fisher, g)))
            gradient = g - eligibility.dot(b)
            nat_grad = np.linalg.solve(fisher, gradient)
        else:
            H = np.linalg.pinv(fisher)
            b = (1 + eligibility.dot(np.linalg.pinv(n_ep*fisher - np.outer(eligibility, eligibility)).dot(eligibility)))\
                    *(Jpol - eligibility.dot(H).dot(g)) / n_ep
            gradient = g - eligibility.dot(b)
            nat_grad = H.dot(gradient)

        self.fisher_list = list()
        self.grad_list = list()
        self.eligibility_list = list()

        return gradient, nat_grad

    def _step_update(self, x, u, r):
        self.psi += self.policy.diff_log(x, u)

    def _episode_end_update(self):
        f_m = np.outer(self.psi, self.psi)
        self.fisher_list.append(f_m)

        gradient = self.J_episode*self.psi
        self.grad_list.append(gradient)

        self.eligibility_list.append(self.psi)

    def _init_update(self):
        self.psi = np.zeros(self.policy.weights_size)<commit_msg>pep8 fixes<commit_after>import numpy as np
from mushroom.algorithms.policy_search import PolicyGradient


class eNAC(PolicyGradient):
    """
    ENAC algorithm.
    "Policy Gradient Methods for Robotics", <NAME>., <NAME>.  2006.

    """
    def __init__(self, policy, mdp_info, params, features):
        self.__name__ = 'eNAC'

        super(eNAC, self).__init__(policy, mdp_info, params, features)

        self.psi = None
        self.fisher_list = list()
        self.grad_list = list()
        self.eligibility_list = list()

    def _compute_gradient(self, J):
        n_ep = len(self.fisher_list)

        fisher = np.mean(self.fisher_list, axis=0)
        g = np.mean(self.grad_list, axis=0)
        eligibility = np.mean(self.eligibility_list, axis=0)
        J_pol = np.mean(J)

        if fisher.shape[0] == np.linalg.matrix_rank(fisher):
            tmp = np.linalg.solve(
                n_ep * fisher - np.outer(eligibility, eligibility), eligibility)
            Q = (1 + eligibility.dot(tmp)) / n_ep
            b = Q.dot(J_pol - eligibility.dot(np.linalg.solve(fisher, g)))
            gradient = g - eligibility.dot(b)
            nat_grad = np.linalg.solve(fisher, gradient)
        else:
            H = np.linalg.pinv(fisher)
            b = (1 + eligibility.dot(np.linalg.pinv(n_ep * fisher - np.outer(
                eligibility, eligibility)).dot(eligibility))) * (
                    J_pol - eligibility.dot(H).dot(g)) / n_ep
            gradient = g - eligibility.dot(b)
            nat_grad = H.dot(gradient)

        self.fisher_list = list()
        self.grad_list = list()
        self.eligibility_list = list()

        return gradient, nat_grad

    def _step_update(self, x, u, r):
        self.psi += self.policy.diff_log(x, u)

    def _episode_end_update(self):
        f_m = np.outer(self.psi, self.psi)
        self.fisher_list.append(f_m)

        gradient = self.J_episode * self.psi
        self.grad_list.append(gradient)

        self.eligibility_list.append(self.psi)

    def _init_update(self):
        self.psi = np.zeros(self.policy.weights_size)
{'fraction_non_alphanumeric': 0.12400674211413436, 'fraction_numerical': 0.005056585600770528, 'mean_word_length': 2.283794466403162, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1677756', 'n_tokens_mistral': 1490, 'n_tokens_neox': 1379, 'n_words': 256}