gradient descent with momentum optimizer update rule for parameter w with gradient g when momentum is 0 python w w learningrate g update rule when momentum is larger than 0 python velocity momentum velocity learningrate g w w velocity when nesterovtrue this rule becomes python velocity momentum velocity learningrate g w w momentum velocity learningrate g args learningrate a float a keras optimizers schedules learningrateschedule instance or a callable that takes no arguments and returns the actual value to use the learning rate defaults to 0 01 momentum float hyperparameter 0 that accelerates gradient descent in the relevant direction and dampens oscillations 0 is vanilla gradient descent defaults to 0 0 nesterov boolean whether to apply nesterov momentum defaults to false baseoptimizerkeywordargs initialize optimizer variables sgd optimizer has one variable momentums only set if self momentum is not 0 args varlist list of model variables to build sgd variables on update step given gradient and the associated model variable learningrate ops castlearningrate variable dtype gradient ops castgradient variable dtype m none if self momentum 0 m self momentumsself getvariableindexvariable if m is not none momentum ops castself momentum variable dtype self assign m ops subtract ops multiplym momentum ops multiplygradient learningrate if self nesterov self assignadd variable ops subtract ops multiplym momentum ops multiplygradient learningrate else self assignaddvariable m else self assignsubvariable ops multiplygradient learningrate def getconfigself config super getconfig config update momentum self momentum nesterov self nesterov return config sgd doc sgd doc replace baseoptimizerkeywordargs optimizer baseoptimizerkeywordargs gradient descent with momentum optimizer update rule for parameter w with gradient g when momentum is 0 python w w learning_rate g update rule when momentum is larger than 0 python velocity momentum velocity learning_rate g w w velocity when nesterov true this rule becomes python velocity momentum velocity learning_rate g w w momentum velocity learning_rate g args learning_rate a float a keras optimizers schedules learningrateschedule instance or a callable that takes no arguments and returns the actual value to use the learning rate defaults to 0 01 momentum float hyperparameter 0 that accelerates gradient descent in the relevant direction and dampens oscillations 0 is vanilla gradient descent defaults to 0 0 nesterov boolean whether to apply nesterov momentum defaults to false base_optimizer_keyword_args initialize optimizer variables sgd optimizer has one variable momentums only set if self momentum is not 0 args var_list list of model variables to build sgd variables on update step given gradient and the associated model variable
from keras import ops
from keras.api_export import keras_export
from keras.optimizers import optimizer


@keras_export("keras.optimizers.SGD")
class SGD(optimizer.Optimizer):
    

    def __init__(
        self,
        learning_rate=0.01,
        momentum=0.0,
        nesterov=False,
        weight_decay=None,
        clipnorm=None,
        clipvalue=None,
        global_clipnorm=None,
        use_ema=False,
        ema_momentum=0.99,
        ema_overwrite_frequency=None,
        name="SGD",
        **kwargs,
    ):
        super().__init__(
            learning_rate=learning_rate,
            name=name,
            weight_decay=weight_decay,
            clipnorm=clipnorm,
            clipvalue=clipvalue,
            global_clipnorm=global_clipnorm,
            use_ema=use_ema,
            ema_momentum=ema_momentum,
            ema_overwrite_frequency=ema_overwrite_frequency,
            **kwargs,
        )
        if not isinstance(momentum, float) or momentum < 0 or momentum > 1:
            raise ValueError("`momentum` must be a float between [0, 1].")
        self.momentum = momentum
        self.nesterov = nesterov

    def build(self, variables):
        
        if self.built:
            return
        super().build(variables)
        self.momentums = []
        if self.momentum != 0:
            for variable in variables:
                self.momentums.append(
                    self.add_variable_from_reference(
                        reference_variable=variable, name="momentum"
                    )
                )

    def update_step(self, gradient, variable, learning_rate):
        
        learning_rate = ops.cast(learning_rate, variable.dtype)
        gradient = ops.cast(gradient, variable.dtype)
        m = None
        if self.momentum != 0:
            m = self.momentums[self._get_variable_index(variable)]

        if m is not None:
            momentum = ops.cast(self.momentum, variable.dtype)
            self.assign(
                m,
                ops.subtract(
                    ops.multiply(m, momentum),
                    ops.multiply(gradient, learning_rate),
                ),
            )
            if self.nesterov:
                self.assign_add(
                    variable,
                    ops.subtract(
                        ops.multiply(m, momentum),
                        ops.multiply(gradient, learning_rate),
                    ),
                )
            else:
                self.assign_add(variable, m)
        else:
            self.assign_sub(variable, ops.multiply(gradient, learning_rate))

    def get_config(self):
        config = super().get_config()
        config.update(
            {
                "momentum": self.momentum,
                "nesterov": self.nesterov,
            }
        )
        return config


SGD.__doc__ = SGD.__doc__.replace(
    "{{base_optimizer_keyword_args}}", optimizer.base_optimizer_keyword_args
)