<commit_before>import numpy as np
from rl.policy.base_policy import Policy


class AnnealedGaussian(Policy):

    '''
    Noise policy, mainly for DDPG.
    Original inspiration from
    https://github.com/matthiasplappert/keras-rl/blob/master/rl/random.py
    '''

    def __init__(self, env_spec,
                 mu, sigma, sigma_min,
                 **kwargs):  # absorb generic param without breaking
        super(AnnealedGaussian, self).__init__(env_spec)
        self.size = self.env_spec['action_dim']
        self.n_steps_annealing = self.env_spec['timestep_limit'] / 2
        self.mu = mu
        self.sigma = sigma
        self.n_steps = 0

        if sigma_min is not None:
            self.m = -float(sigma - sigma_min) / float(self.n_steps_annealing)
            self.c = sigma
            self.sigma_min = sigma_min
        else:
            self.m = 0.
            self.c = sigma
            self.sigma_min = sigma

    @property
    def current_sigma(self):
        sigma = max(self.sigma_min, self.m * float(self.n_steps) + self.c)
        return sigma

    def select_action(self, state):
        agent = self.agent
        state = np.expand_dims(state, axis=0)
        action = agent.actor.predict(state)[0] + self.sample()
        return action

    def update(self, sys_vars):
        pass


class GaussianWhiteNoise(AnnealedGaussian):

    def __init__(self, env_spec,
                 mu=0., sigma=.3, sigma_min=None,
                 **kwargs):  # absorb generic param without breaking
        super(GaussianWhiteNoise, self).__init__(
            env_spec, mu=mu, sigma=sigma, sigma_min=sigma_min)

    def sample(self):
        sample = np.random.normal(self.mu, self.current_sigma, self.size)
        self.n_steps += 1
        return sample


# Based on
# http://math.stackexchange.com/questions/1287634/implementing-ornstein-uhlenbeck-in-matlab
class OUNoise(AnnealedGaussian):

    def __init__(self, env_spec,
                 theta=.15, mu=0., sigma=.3, dt=1e-2, x0=None, sigma_min=None,
                 **kwargs):  # absorb generic param without breaking
        super(OUNoise, self).__init__(
            env_spec, mu=mu, sigma=sigma, sigma_min=sigma_min,
            **kwargs)
        self.theta = theta
        self.mu = mu
        self.dt = dt<commit_msg>correct description in noise<commit_after>import numpy as np
from rl.policy.base_policy import Policy


class AnnealedGaussian(Policy):

    '''
    Noise policy, mainly for DDPG.
    Original inspiration from
    https://github.com/matthiasplappert/keras-rl/blob/master/rl/random.py
    '''

    def __init__(self, env_spec,
                 mu, sigma, sigma_min,
                 **kwargs):  # absorb generic param without breaking
        super(AnnealedGaussian, self).__init__(env_spec)
        self.size = self.env_spec['action_dim']
        self.n_steps_annealing = self.env_spec['timestep_limit'] / 2
        self.mu = mu
        self.sigma = sigma
        self.n_steps = 0

        if sigma_min is not None:
            self.m = -float(sigma - sigma_min) / float(self.n_steps_annealing)
            self.c = sigma
            self.sigma_min = sigma_min
        else:
            self.m = 0.
            self.c = sigma
            self.sigma_min = sigma

    @property
    def current_sigma(self):
        sigma = max(self.sigma_min, self.m * float(self.n_steps) + self.c)
        return sigma

    def select_action(self, state):
        agent = self.agent
        state = np.expand_dims(state, axis=0)
        action = agent.actor.predict(state)[0] + self.sample()
        return action

    def update(self, sys_vars):
        pass


class GaussianWhiteNoise(AnnealedGaussian):

    def __init__(self, env_spec,
                 mu=0., sigma=.3, sigma_min=None,
                 **kwargs):  # absorb generic param without breaking
        super(GaussianWhiteNoise, self).__init__(
            env_spec, mu=mu, sigma=sigma, sigma_min=sigma_min)

    def sample(self):
        sample = np.random.normal(self.mu, self.current_sigma, self.size)
        self.n_steps += 1
        return sample


class OUNoise(AnnealedGaussian):

    '''
    Based on
    http://math.stackexchange.com/questions/1287634/implementing-ornstein-uhlenbeck-in-matlab
    '''

    def __init__(self, env_spec,
                 theta=.15, mu=0., sigma=.3, dt=1e-2, x0=None, sigma_min=None,
                 **kwargs):  # absorb generic param without breaking
        super(OUNoise, self).__init__(
            env_spec, mu=mu, sigma=sigma, sigma_min=sigma_min,
            **kwargs)
        self.theta = theta
        self.mu = mu
        self.dt = dt
{'fraction_non_alphanumeric': 0.10501296456352636, 'fraction_numerical': 0.00950734658599827, 'mean_word_length': 2.1640464798359536, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7276105', 'n_tokens_mistral': 1515, 'n_tokens_neox': 1411, 'n_words': 310}