def init self learningrate0 001 learningratepower0 5 initialaccumulatorvalue0 1 l1regularizationstrength0 0 l2regularizationstrength0 0 l2shrinkageregularizationstrength0 0 beta0 0 weightdecaynone clipnormnone clipvaluenone globalclipnormnone useemafalse emamomentum0 99 emaoverwritefrequencynone nameftrl kwargs super init learningratelearningrate namename weightdecayweightdecay clipnormclipnorm clipvalueclipvalue globalclipnormglobalclipnorm useemauseema emamomentumemamomentum emaoverwritefrequencyemaoverwritefrequency kwargs if initialaccumulatorvalue 0 0 raise valueerror initialaccumulatorvalue needs to be positive or zero received initialaccumulatorvalue finitialaccumulatorvalue if learningratepower 0 0 raise valueerror learningratepower needs to be negative or zero received flearningratepowerlearningratepower if l1regularizationstrength 0 0 raise valueerror l1regularizationstrength needs to be positive or zero received l1regularizationstrength fl1regularizationstrength if l2regularizationstrength 0 0 raise valueerror l2regularizationstrength needs to be positive or zero received l2regularizationstrength fl2regularizationstrength if l2shrinkageregularizationstrength 0 0 raise valueerror l2shrinkageregularizationstrength needs to be positive or zero received l2shrinkageregularizationstrength fl2shrinkageregularizationstrength self learningratepower learningratepower self initialaccumulatorvalue initialaccumulatorvalue self l1regularizationstrength l1regularizationstrength self l2regularizationstrength l2regularizationstrength self l2shrinkageregularizationstrength l2shrinkageregularizationstrength self beta beta def buildself varlist if self built return super buildvarlist self accumulators self linears for var in varlist self accumulators append self addvariable shapevar shape dtypevar dtype nameaccumulator initializerinitializers constant self initialaccumulatorvalue self linears append self addvariablefromreference referencevariablevar namelinear def updatestepself gradient variable learningrate optimizer that implements the ftrl algorithm follow the regularized leader ftrl is an optimization algorithm developed at google for click through rate prediction in the early 2010s it is most suitable for shallow models with large and sparse feature spaces the algorithm is described by mcmahan et al 2013 https research google com pubs archive 41159 pdf the keras version has support for both online l2 regularization the l2 regularization described in the paper above and shrinkage type l2 regularization which is the addition of an l2 penalty to the loss function initialization python n 0 sigma 0 z 0 update rule for one variable w python prev_n n n n g 2 sigma n lr_power prev_n lr_power lr z z g sigma w if abs z lambda_1 w 0 else w sgn z lambda_1 z beta sqrt n alpha lambda_2 notation lr is the learning rate g is the gradient for the variable lambda_1 is the l1 regularization strength lambda_2 is the l2 regularization strength lr_power is the power to scale n check the documentation for the l2_shrinkage_regularization_strength parameter for more details when shrinkage is enabled in which case gradient is replaced with a gradient with shrinkage args learning_rate a float a keras optimizers schedules learningrateschedule instance or a callable that takes no arguments and returns the actual value to use the learning rate defaults to 0 001 learning_rate_power a float value must be less or equal to zero controls how the learning rate decreases during training use zero for a fixed learning rate initial_accumulator_value the starting value for accumulators only zero or positive values are allowed l1_regularization_strength a float value must be greater than or equal to zero defaults to 0 0 l2_regularization_strength a float value must be greater than or equal to zero defaults to 0 0 l2_shrinkage_regularization_strength a float value must be greater than or equal to zero this differs from l2 above in that the l2 above is a stabilization penalty whereas this l2 shrinkage is a magnitude penalty when input is sparse shrinkage will only happen on the active weights beta a float value representing the beta value from the paper defaults to 0 0 base_optimizer_keyword_args initialize optimizer variables args var_list list of model variables to build ftrl variables on update step given gradient and the associated model variable noqa e501
from keras import initializers
from keras import ops
from keras.api_export import keras_export
from keras.optimizers import optimizer


@keras_export(["keras.optimizers.Ftrl"])
class Ftrl(optimizer.Optimizer):
    r

    def __init__(
        self,
        learning_rate=0.001,
        learning_rate_power=-0.5,
        initial_accumulator_value=0.1,
        l1_regularization_strength=0.0,
        l2_regularization_strength=0.0,
        l2_shrinkage_regularization_strength=0.0,
        beta=0.0,
        weight_decay=None,
        clipnorm=None,
        clipvalue=None,
        global_clipnorm=None,
        use_ema=False,
        ema_momentum=0.99,
        ema_overwrite_frequency=None,
        name="ftrl",
        **kwargs,
    ):
        super().__init__(
            learning_rate=learning_rate,
            name=name,
            weight_decay=weight_decay,
            clipnorm=clipnorm,
            clipvalue=clipvalue,
            global_clipnorm=global_clipnorm,
            use_ema=use_ema,
            ema_momentum=ema_momentum,
            ema_overwrite_frequency=ema_overwrite_frequency,
            **kwargs,
        )

        if initial_accumulator_value < 0.0:
            raise ValueError(
                "`initial_accumulator_value` needs to be positive or zero. "
                "Received: initial_accumulator_value="
                f"{initial_accumulator_value}."
            )
        if learning_rate_power > 0.0:
            raise ValueError(
                "`learning_rate_power` needs to be negative or zero. Received: "
                f"learning_rate_power={learning_rate_power}."
            )
        if l1_regularization_strength < 0.0:
            raise ValueError(
                "`l1_regularization_strength` needs to be positive or zero. "
                "Received: l1_regularization_strength="
                f"{l1_regularization_strength}."
            )
        if l2_regularization_strength < 0.0:
            raise ValueError(
                "`l2_regularization_strength` needs to be positive or zero. "
                "Received: l2_regularization_strength="
                f"{l2_regularization_strength}."
            )
        if l2_shrinkage_regularization_strength < 0.0:
            raise ValueError(
                "`l2_shrinkage_regularization_strength` needs to be positive "
                "or zero. Received: l2_shrinkage_regularization_strength"
                f"={l2_shrinkage_regularization_strength}."
            )

        self.learning_rate_power = learning_rate_power
        self.initial_accumulator_value = initial_accumulator_value
        self.l1_regularization_strength = l1_regularization_strength
        self.l2_regularization_strength = l2_regularization_strength
        self.l2_shrinkage_regularization_strength = (
            l2_shrinkage_regularization_strength
        )
        self.beta = beta

    def build(self, var_list):
        
        if self.built:
            return
        super().build(var_list)
        self._accumulators = []
        self._linears = []
        for var in var_list:
            self._accumulators.append(
                self.add_variable(
                    shape=var.shape,
                    dtype=var.dtype,
                    name="accumulator",
                    initializer=initializers.Constant(
                        self.initial_accumulator_value,
                    ),
                )
            )
            self._linears.append(
                self.add_variable_from_reference(
                    reference_variable=var, name="linear"
                )
            )

    def update_step(self, gradient, variable, learning_rate):
        

        lr = ops.cast(learning_rate, variable.dtype)
        gradient = ops.cast(gradient, variable.dtype)

        accum = self._accumulators[self._get_variable_index(variable)]
        linear = self._linears[self._get_variable_index(variable)]

        lr_power = self.learning_rate_power
        l2_reg = self.l2_regularization_strength
        l2_reg = l2_reg + self.beta / (2.0 * lr)

        grad_to_use = ops.add(
            gradient,
            ops.multiply(
                2 * self.l2_shrinkage_regularization_strength, variable
            ),
        )
        new_accum = ops.add(accum, ops.square(gradient))
        self.assign_add(
            linear,
            ops.subtract(
                grad_to_use,
                ops.multiply(
                    ops.divide(
                        ops.subtract(
                            ops.power(new_accum, -lr_power),
                            ops.power(accum, -lr_power),
                        ),
                        lr,
                    ),
                    variable,
                ),
            ),
        )
        quadratic = ops.add(
            ops.divide(ops.power(new_accum, (-lr_power)), lr), 2 * l2_reg
        )
        linear_clipped = ops.clip(
            linear,
            -self.l1_regularization_strength,
            self.l1_regularization_strength,
        )
        self.assign(
            variable,
            ops.divide(ops.subtract(linear_clipped, linear), quadratic),
        )
        self.assign(accum, new_accum)

    def get_config(self):
        config = super().get_config()

        config.update(
            {
                "learning_rate_power": self.learning_rate_power,
                "initial_accumulator_value": self.initial_accumulator_value,
                "l1_regularization_strength": self.l1_regularization_strength,
                "l2_regularization_strength": self.l2_regularization_strength,
                "l2_shrinkage_regularization_strength": self.l2_shrinkage_regularization_strength,  
                "beta": self.beta,
            }
        )
        return config


Ftrl.__doc__ = Ftrl.__doc__.replace(
    "{{base_optimizer_keyword_args}}", optimizer.base_optimizer_keyword_args
)