<commit_before># Copyright 2021 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""Base class of optimizer."""

import abc
import re

import tensorflow.compat.v2 as tf
from absl import logging

from keras import backend
from keras import initializers
from keras.optimizers.optimizer_v2 import utils as optimizer_utils
from keras.optimizers.schedules import learning_rate_schedule
from keras.utils import tf_utils

# isort: off
from tensorflow.python.util.tf_export import keras_export
from tensorflow.tools.docs import doc_controls


class _BaseOptimizer(tf.__internal__.tracking.AutoTrackable):
    """Optimizer base class, which only supports non-distribute use case."""

    def __init__(
        self,
        name,
        weight_decay=None,
        clipnorm=None,
        clipvalue=None,
        global_clipnorm=None,
        use_ema=False,
        ema_momentum=0.99,
        ema_overwrite_frequency=None,
        jit_compile=True,
        **kwargs,
    ):
        self.name = name
        self.weight_decay = weight_decay
        self.clipnorm = clipnorm
        self.global_clipnorm = global_clipnorm
        self.clipvalue = clipvalue
        self.use_ema = use_ema
        self.jit_compile = jit_compile
        if not tf.config.list_physical_devices("GPU"):
            # Optimizer only benefits from XLA when training on GPU. So if no
            # GPU is found, we turn off XLA.
            self.jit_compile = False
        if use_ema:
            # Verify the arguments related to EMA.
            if ema_momentum > 1 or ema_momentum < 0:
                raise ValueError(
                    "`ema_momentum` must be in the range [0, 1]. "
                    f"Received: ema_momentum={ema_momentum}"
                )
            if ema_overwrite_frequency and (
                not isinstance(ema_overwrite_frequency, int)
                or ema_overwrite_frequency < 1
            ):
                raise ValueError(
                    "`ema_overwrite_frequency` must be an integer > 1 or None. "
                    "Received: ema_overwrite_frequency="
                    f"{ema_overwrite_frequency}"
                )
        self.ema_momentum = ema_momentum
        self.ema_overwrite_frequency = ema_overwrite_frequency

        if self.clipnorm is not None and self.global_clipnorm is not None:
            raise ValueError(
                "At most one of `clipnorm` and `global_clipnorm` can "
                f"be set. Received: clipnorm={self.clipnorm}, "
                f"global_clipnorm={self.global_clipnorm}."
            )

        self._variables = []
        self._create_iteration_variable()
        self._process_kwargs(kwargs)

    def _create_iteration_variable(self):
        """Create the iterations counter variable."""
        with tf.init_scope():
            # Lift the variable creation to init scope to avoid environment
            # issue.
            self._iterations = tf.Variable(
                0, name="iteration", dtype=tf.int64, trainable=False
            )
        self._variables.append(self._iterations)

    def _process_kwargs(self, kwargs):
        # Remove the `is_legacy_optimizer` arg, which is for serialization only.
        kwargs.pop("is_legacy_optimizer", None)
        legacy_kwargs = {
            "lr",
            "decay",
            "gradient_transformers",
            "gradient_aggregator",
        }
        for k in kwargs:
            if k in legacy_kwargs:
                logging.warning(
                    "%s is deprecated in `optimizer_experimental.Optimizer`"
                    ", please check the docstring for valid arguments.",
                    k,
                )
            else:
                raise TypeError(
                    f"{k} is not a valid argument, kwargs should be empty "
                    " for `optimizer_experimental.Optimizer`."
                )

    def _create_or_restore_slot_variable(self, **kwargs):
        raise ValueError(
            "You are trying to restore a checkpoint from a legacy Keras "
            "optimizer into a v2.11+ Optimizer, which can cause "
            "errors. Please update the optimizer referenced in your code "
            "to be an instance of "
            "`tf.keras.optimizers.legacy.Optimizer`, e.g.: "
            f"`tf.keras.optimizer.legacy.{self.__class__.__name__}`."
        )

    def _var_key(self, variable):
        """Get a unique identifier of the given variable."""
        # Get the distributed variable if it exists.
        # TODO(b/199214315): replace _unique_id with ref() after fixing ref()
        # issues on AggregatingVariable.
        return variable._unique_id

    def _deduplicate_sparse_grad(self, grads):
        """Deduplicate sparse gradient.

        For sparse gradients, i.e., gradient is of type `tf.IndexedSlices`,
        it is possible that `gradient.indices` has duplicated indices.
        This function adds up values for the duplicated indices, and returns
        a `tf.IndexedSlices` with indices of unique values.
        """
        processed_grads = []<commit_msg>keras: fix typo in keras optimizers migration recommendation<commit_after># Copyright 2021 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""Base class of optimizer."""

import abc
import re

import tensorflow.compat.v2 as tf
from absl import logging

from keras import backend
from keras import initializers
from keras.optimizers.optimizer_v2 import utils as optimizer_utils
from keras.optimizers.schedules import learning_rate_schedule
from keras.utils import tf_utils

# isort: off
from tensorflow.python.util.tf_export import keras_export
from tensorflow.tools.docs import doc_controls


class _BaseOptimizer(tf.__internal__.tracking.AutoTrackable):
    """Optimizer base class, which only supports non-distribute use case."""

    def __init__(
        self,
        name,
        weight_decay=None,
        clipnorm=None,
        clipvalue=None,
        global_clipnorm=None,
        use_ema=False,
        ema_momentum=0.99,
        ema_overwrite_frequency=None,
        jit_compile=True,
        **kwargs,
    ):
        self.name = name
        self.weight_decay = weight_decay
        self.clipnorm = clipnorm
        self.global_clipnorm = global_clipnorm
        self.clipvalue = clipvalue
        self.use_ema = use_ema
        self.jit_compile = jit_compile
        if not tf.config.list_physical_devices("GPU"):
            # Optimizer only benefits from XLA when training on GPU. So if no
            # GPU is found, we turn off XLA.
            self.jit_compile = False
        if use_ema:
            # Verify the arguments related to EMA.
            if ema_momentum > 1 or ema_momentum < 0:
                raise ValueError(
                    "`ema_momentum` must be in the range [0, 1]. "
                    f"Received: ema_momentum={ema_momentum}"
                )
            if ema_overwrite_frequency and (
                not isinstance(ema_overwrite_frequency, int)
                or ema_overwrite_frequency < 1
            ):
                raise ValueError(
                    "`ema_overwrite_frequency` must be an integer > 1 or None. "
                    "Received: ema_overwrite_frequency="
                    f"{ema_overwrite_frequency}"
                )
        self.ema_momentum = ema_momentum
        self.ema_overwrite_frequency = ema_overwrite_frequency

        if self.clipnorm is not None and self.global_clipnorm is not None:
            raise ValueError(
                "At most one of `clipnorm` and `global_clipnorm` can "
                f"be set. Received: clipnorm={self.clipnorm}, "
                f"global_clipnorm={self.global_clipnorm}."
            )

        self._variables = []
        self._create_iteration_variable()
        self._process_kwargs(kwargs)

    def _create_iteration_variable(self):
        """Create the iterations counter variable."""
        with tf.init_scope():
            # Lift the variable creation to init scope to avoid environment
            # issue.
            self._iterations = tf.Variable(
                0, name="iteration", dtype=tf.int64, trainable=False
            )
        self._variables.append(self._iterations)

    def _process_kwargs(self, kwargs):
        # Remove the `is_legacy_optimizer` arg, which is for serialization only.
        kwargs.pop("is_legacy_optimizer", None)
        legacy_kwargs = {
            "lr",
            "decay",
            "gradient_transformers",
            "gradient_aggregator",
        }
        for k in kwargs:
            if k in legacy_kwargs:
                logging.warning(
                    "%s is deprecated in `optimizer_experimental.Optimizer`"
                    ", please check the docstring for valid arguments.",
                    k,
                )
            else:
                raise TypeError(
                    f"{k} is not a valid argument, kwargs should be empty "
                    " for `optimizer_experimental.Optimizer`."
                )

    def _create_or_restore_slot_variable(self, **kwargs):
        raise ValueError(
            "You are trying to restore a checkpoint from a legacy Keras "
            "optimizer into a v2.11+ Optimizer, which can cause "
            "errors. Please update the optimizer referenced in your code "
            "to be an instance of "
            "`tf.keras.optimizers.legacy.Optimizer`, e.g.: "
            f"`tf.keras.optimizers.legacy.{self.__class__.__name__}`."
        )

    def _var_key(self, variable):
        """Get a unique identifier of the given variable."""
        # Get the distributed variable if it exists.
        # TODO(b/199214315): replace _unique_id with ref() after fixing ref()
        # issues on AggregatingVariable.
        return variable._unique_id

    def _deduplicate_sparse_grad(self, grads):
        """Deduplicate sparse gradient.

        For sparse gradients, i.e., gradient is of type `tf.IndexedSlices`,
        it is possible that `gradient.indices` has duplicated indices.
        This function adds up values for the duplicated indices, and returns
        a `tf.IndexedSlices` with indices of unique values.
        """
        processed_grads = []
{'fraction_non_alphanumeric': 0.08633030726256984, 'fraction_numerical': 0.005935754189944134, 'mean_word_length': 2.1684181415929205, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '930671', 'n_tokens_mistral': 3121, 'n_tokens_neox': 2974, 'n_words': 955}