optimizer that implements the adagrad algorithm adagrad is an optimizer with parameterspecific learning rates which are adapted relative to how frequently a parameter gets updated during training the more updates a parameter receives the smaller the updates args learningrate a float a keras optimizers schedules learningrateschedule instance or a callable that takes no arguments and returns the actual value to use the learning rate defaults to 0 001 note that adagrad tends to benefit from higher initial learning rate values compared to other optimizers to match the exact form in the original paper use 1 0 initialaccumulatorvalue floating point value starting value for the accumulators perparameter momentum values must be nonnegative epsilon small floating point value for maintaining numerical stability baseoptimizerkeywordargs reference duchi et al 2011 http www jmlr orgpapersvolume12duchi11aduchi11a pdf update step given gradient and the associated model variable lr ops castlearningrate variable dtype gradient ops castgradient variable dtype accumulator self accumulatorsself getvariableindexvariable self assignaddaccumulator ops squaregradient self assignsub variable ops divide ops multiplylr gradient ops sqrtops addaccumulator self epsilon def getconfigself config super getconfig config update initialaccumulatorvalue self initialaccumulatorvalue epsilon self epsilon return config adagrad doc adagrad doc replace baseoptimizerkeywordargs optimizer baseoptimizerkeywordargs optimizer that implements the adagrad algorithm adagrad is an optimizer with parameter specific learning rates which are adapted relative to how frequently a parameter gets updated during training the more updates a parameter receives the smaller the updates args learning_rate a float a keras optimizers schedules learningrateschedule instance or a callable that takes no arguments and returns the actual value to use the learning rate defaults to 0 001 note that adagrad tends to benefit from higher initial learning rate values compared to other optimizers to match the exact form in the original paper use 1 0 initial_accumulator_value floating point value starting value for the accumulators per parameter momentum values must be non negative epsilon small floating point value for maintaining numerical stability base_optimizer_keyword_args reference duchi et al 2011 http www jmlr org papers volume12 duchi11a duchi11a pdf update step given gradient and the associated model variable
from keras import initializers
from keras import ops
from keras.api_export import keras_export
from keras.optimizers import optimizer


@keras_export(["keras.optimizers.Adagrad"])
class Adagrad(optimizer.Optimizer):
    

    def __init__(
        self,
        learning_rate=0.001,
        initial_accumulator_value=0.1,
        epsilon=1e-7,
        weight_decay=None,
        clipnorm=None,
        clipvalue=None,
        global_clipnorm=None,
        use_ema=False,
        ema_momentum=0.99,
        ema_overwrite_frequency=None,
        name="adagrad",
        **kwargs,
    ):
        super().__init__(
            learning_rate=learning_rate,
            weight_decay=weight_decay,
            clipnorm=clipnorm,
            clipvalue=clipvalue,
            global_clipnorm=global_clipnorm,
            use_ema=use_ema,
            ema_momentum=ema_momentum,
            ema_overwrite_frequency=ema_overwrite_frequency,
            name=name,
            **kwargs,
        )
        self.initial_accumulator_value = initial_accumulator_value
        self.epsilon = epsilon

    def build(self, var_list):
        if self.built:
            return
        super().build(var_list)
        self._accumulators = []
        initializer = initializers.Constant(self.initial_accumulator_value)
        for var in var_list:
            self._accumulators.append(
                self.add_variable(
                    shape=var.shape,
                    initializer=initializer,
                    dtype=var.dtype,
                    name="accumulator",
                )
            )

    def update_step(self, gradient, variable, learning_rate):
        
        lr = ops.cast(learning_rate, variable.dtype)
        gradient = ops.cast(gradient, variable.dtype)

        accumulator = self._accumulators[self._get_variable_index(variable)]

        self.assign_add(accumulator, ops.square(gradient))
        self.assign_sub(
            variable,
            ops.divide(
                ops.multiply(lr, gradient),
                ops.sqrt(ops.add(accumulator, self.epsilon)),
            ),
        )

    def get_config(self):
        config = super().get_config()

        config.update(
            {
                "initial_accumulator_value": self.initial_accumulator_value,
                "epsilon": self.epsilon,
            }
        )
        return config


Adagrad.__doc__ = Adagrad.__doc__.replace(
    "{{base_optimizer_keyword_args}}", optimizer.base_optimizer_keyword_args
)