optimizer that implements the rmsprop algorithm the gist of rmsprop is to maintain a moving discounted average of the square of gradients divide the gradient by the root of this average this implementation of rmsprop uses plain momentum not nesterov momentum the centered version additionally maintains a moving average of the gradients and uses that average to estimate the variance args learningrate a float a keras optimizers schedules learningrateschedule instance or a callable that takes no arguments and returns the actual value to use the learning rate defaults to 0 001 rho float defaults to 0 9 discounting factor for the old gradients momentum float defaults to 0 0 if not 0 0 the optimizer tracks the momentum value with a decay rate equals to 1 momentum epsilon a small constant for numerical stability this epsilon is epsilon hat in the kingma and ba paper in the formula just before section 2 1 not the epsilon in algorithm 1 of the paper defaults to 1e7 centered boolean if true gradients are normalized by the estimated variance of the gradient if false by the uncentered second moment setting this to true may help with training but is slightly more expensive in terms of computation and memory defaults to false baseoptimizerkeywordargs usage opt keras optimizers rmsproplearningrate0 1 var1 keras backend variable10 0 loss lambda var1 2 2 0 dloss dvar1 var1 opt minimizeloss var1 var1 9 683772 reference hinton 2012 http www cs toronto edutijmencsc321slideslectureslideslec6 pdf update step given gradient and the associated model variable lr ops castlearningrate variable dtype gradient ops castgradient variable dtype velocity self velocitiesself getvariableindexvariable momentum none if self momentum 0 momentum self momentumsself getvariableindexvariable averagegrad none if self centered averagegrad self averagegradients self getvariableindexvariable rho self rho self assign velocity ops add ops multiplyrho velocity ops multiply1 rho ops squaregradient if self centered self assign averagegrad ops add ops multiplyrho averagegrad ops multiply1 rho gradient denominator velocity ops squareaveragegrad self epsilon else denominator ops addvelocity self epsilon increment ops divide ops multiplylr gradient ops sqrtdenominator if self momentum 0 self assign momentum ops addops multiplyself momentum momentum increment self assignsubvariable momentum else self assignsubvariable increment def getconfigself config super getconfig config update rho self rho momentum self momentum epsilon self epsilon centered self centered return config rmsprop doc rmsprop doc replace baseoptimizerkeywordargs optimizer baseoptimizerkeywordargs optimizer that implements the rmsprop algorithm the gist of rmsprop is to maintain a moving discounted average of the square of gradients divide the gradient by the root of this average this implementation of rmsprop uses plain momentum not nesterov momentum the centered version additionally maintains a moving average of the gradients and uses that average to estimate the variance args learning_rate a float a keras optimizers schedules learningrateschedule instance or a callable that takes no arguments and returns the actual value to use the learning rate defaults to 0 001 rho float defaults to 0 9 discounting factor for the old gradients momentum float defaults to 0 0 if not 0 0 the optimizer tracks the momentum value with a decay rate equals to 1 momentum epsilon a small constant for numerical stability this epsilon is epsilon hat in the kingma and ba paper in the formula just before section 2 1 not the epsilon in algorithm 1 of the paper defaults to 1e 7 centered boolean if true gradients are normalized by the estimated variance of the gradient if false by the uncentered second moment setting this to true may help with training but is slightly more expensive in terms of computation and memory defaults to false base_optimizer_keyword_args usage opt keras optimizers rmsprop learning_rate 0 1 var1 keras backend variable 10 0 loss lambda var1 2 2 0 d loss d var1 var1 opt minimize loss var1 var1 9 683772 reference hinton 2012 http www cs toronto edu tijmen csc321 slides lecture_slides_lec6 pdf update step given gradient and the associated model variable
from keras import ops
from keras.api_export import keras_export
from keras.optimizers import optimizer


@keras_export(["keras.optimizers.RMSprop"])
class RMSprop(optimizer.Optimizer):
    

    def __init__(
        self,
        learning_rate=0.001,
        rho=0.9,
        momentum=0.0,
        epsilon=1e-7,
        centered=False,
        weight_decay=None,
        clipnorm=None,
        clipvalue=None,
        global_clipnorm=None,
        use_ema=False,
        ema_momentum=0.99,
        ema_overwrite_frequency=100,
        name="rmsprop",
        **kwargs,
    ):
        super().__init__(
            learning_rate=learning_rate,
            weight_decay=weight_decay,
            clipnorm=clipnorm,
            clipvalue=clipvalue,
            global_clipnorm=global_clipnorm,
            use_ema=use_ema,
            ema_momentum=ema_momentum,
            ema_overwrite_frequency=ema_overwrite_frequency,
            name=name,
            **kwargs,
        )
        self.rho = rho
        self.momentum = momentum
        self.epsilon = epsilon
        self.centered = centered

    def build(self, var_list):
        if self.built:
            return

        super().build(var_list)

        self._velocities = []
        for var in var_list:
            self._velocities.append(
                self.add_variable_from_reference(var, "velocity")
            )

        self._momentums = []
        if self.momentum > 0:
            for var in var_list:
                self._momentums.append(
                    self.add_variable_from_reference(var, "momentum")
                )

        self._average_gradients = []
        if self.centered:
            for var in var_list:
                self._average_gradients.append(
                    self.add_variable_from_reference(var, "average_gradient")
                )

    def update_step(self, gradient, variable, learning_rate):
        
        lr = ops.cast(learning_rate, variable.dtype)
        gradient = ops.cast(gradient, variable.dtype)

        velocity = self._velocities[self._get_variable_index(variable)]
        momentum = None
        if self.momentum > 0:
            momentum = self._momentums[self._get_variable_index(variable)]
        average_grad = None
        if self.centered:
            average_grad = self._average_gradients[
                self._get_variable_index(variable)
            ]

        rho = self.rho

        self.assign(
            velocity,
            ops.add(
                ops.multiply(rho, velocity),
                ops.multiply(1 - rho, ops.square(gradient)),
            ),
        )
        if self.centered:
            self.assign(
                average_grad,
                ops.add(
                    ops.multiply(rho, average_grad),
                    ops.multiply(1 - rho, gradient),
                ),
            )
            denominator = velocity - ops.square(average_grad) + self.epsilon
        else:
            denominator = ops.add(velocity, self.epsilon)
        increment = ops.divide(
            ops.multiply(lr, gradient), ops.sqrt(denominator)
        )
        if self.momentum > 0:
            self.assign(
                momentum,
                ops.add(ops.multiply(self.momentum, momentum), increment),
            )
            self.assign_sub(variable, momentum)
        else:
            self.assign_sub(variable, increment)

    def get_config(self):
        config = super().get_config()

        config.update(
            {
                "rho": self.rho,
                "momentum": self.momentum,
                "epsilon": self.epsilon,
                "centered": self.centered,
            }
        )
        return config


RMSprop.__doc__ = RMSprop.__doc__.replace(
    "{{base_optimizer_keyword_args}}", optimizer.base_optimizer_keyword_args
)