





"""simple docstring"""




import random
import timeit
from functools import wraps
from typing import Callable, Optional

from ..configuration_utils import PretrainedConfig
from ..models.auto.modeling_tf_auto import TF_MODEL_MAPPING, TF_MODEL_WITH_LM_HEAD_MAPPING
from ..utils import is_pyanvml_available, is_tf_available, logging
from .benchmark_utils import (
    Benchmark,
    Memory,
    MemorySummary,
    measure_peak_memory_cpu,
    start_memory_tracing,
    stop_memory_tracing,
)


if is_tf_available():
   import tensorflow as tf
   from tensorflow.python.framework.errors_impl import ResourceExhaustedError

   from .benchmark_args_tf import TensorFlowBenchmarkArguments

if is_pyanvml_available():
   import pyanvml.pyanvml as nvml

lowerCamelCase_     :						List[Any]						=					logging.get_logger(__name__)





def 						UpperCAmelCase__		(     _UpperCAmelCase ,					_UpperCAmelCase					):






  """simple docstring"""





  def run_func(_UpperCAmelCase					):
    @wraps(_UpperCAmelCase					)
    def run_in_eager_mode(*_UpperCAmelCase ,					**_UpperCAmelCase					):
      return func(*_UpperCAmelCase ,					**_UpperCAmelCase					)

    @wraps(_UpperCAmelCase					)
    @tf.function(experimental_compile=_UpperCAmelCase					)
    def run_in_graph_mode(*_UpperCAmelCase ,					**_UpperCAmelCase					):
      return func(*_UpperCAmelCase ,					**_UpperCAmelCase					)

    if do_eager_mode is True:
      if use_xla is not False:
        raise ValueError(
            'Cannot run model in XLA, if `args.eager_mode` is set to `True`. Please set `args.eager_mode=False`.'					)
      return run_in_eager_mode
    else:
      return run_in_graph_mode

  return run_func





def 						UpperCAmelCase__		(     _UpperCAmelCase ,					_UpperCAmelCase ,					_UpperCAmelCase					):






  """simple docstring"""





  A_  :       Union[str, Any]										=  random.Random()
  A_  :       List[str]										=  [rng.randint(0 ,					vocab_size - 1					) for i in range(batch_size * sequence_length					)]
  return tf.constant(_UpperCAmelCase ,					shape=(batch_size, sequence_length) ,					dtype=tf.intaa					)





class  _UpperCAmelCase  (   UpperCAmelCase__		):



      '''simple docstring'''
      lowercase_      : TensorFlowBenchmarkArguments
      lowercase_      : PretrainedConfig
      lowercase_      : str	    =   "TensorFlow"

      @property
      def 					lowerCamelCase_			(     self  ):

        """simple docstring"""

        return tf.__version__

      def 					lowerCamelCase_			(     self					,      snake_case_					,      snake_case_					,      snake_case_  ):

        """simple docstring"""

        A_  :       str										=  self.args.strategy
        if strategy is None:
          raise ValueError('A device strategy has to be initialized before using TensorFlow.'  )
        A_  :       int										=  self._prepare_inference_func(snake_case_					,      snake_case_					,      snake_case_  )
        return self._measure_speed(_inference  )

      def 					lowerCamelCase_			(     self					,      snake_case_					,      snake_case_					,      snake_case_  ):

        """simple docstring"""

        A_  :       Any										=  self.args.strategy
        if strategy is None:
          raise ValueError('A device strategy has to be initialized before using TensorFlow.'  )
        A_  :       str										=  self._prepare_train_func(snake_case_					,      snake_case_					,      snake_case_  )
        return self._measure_speed(_train  )

      def 					lowerCamelCase_			(     self					,      snake_case_					,      snake_case_					,      snake_case_  ):

        """simple docstring"""

        if self.args.is_gpu:
          tf.config.experimental.set_memory_growth(self.args.gpu_list[self.args.device_idx]					,      snake_case_  )
        A_  :       Any										=  self.args.strategy
        if strategy is None:
          raise ValueError('A device strategy has to be initialized before using TensorFlow.'  )
        A_  :       Any										=  self._prepare_inference_func(snake_case_					,      snake_case_					,      snake_case_  )
        return self._measure_memory(_inference  )

      def 					lowerCamelCase_			(     self					,      snake_case_					,      snake_case_					,      snake_case_  ):

        """simple docstring"""

        if self.args.is_gpu:
          tf.config.experimental.set_memory_growth(self.args.gpu_list[self.args.device_idx]					,      snake_case_  )
        A_  :       List[str]										=  self.args.strategy
        if strategy is None:
          raise ValueError('A device strategy has to be initialized before using TensorFlow.'  )

        A_  :       str										=  self._prepare_train_func(snake_case_					,      snake_case_					,      snake_case_  )
        return self._measure_memory(_train  )

      def 					lowerCamelCase_			(     self					,      snake_case_					,      snake_case_					,      snake_case_  ):

        """simple docstring"""

        A_  :       str										=  self.config_dict[model_name]

        if self.args.fpaa:
          raise NotImplementedError('Mixed precision is currently not supported.'  )

        A_  :       Optional[Any]										=  (
            hasattr(snake_case_					,      'architectures'  )
            and isinstance(config.architectures					,      snake_case_  )
            and len(config.architectures  ) > 0
        )
        if not self.args.only_pretrain_model and has_model_class_in_config:
          try:
            A_  :       str										=  'TF' + config.architectures[0]  # prepend 'TF' for tensorflow model
            A_  :       Optional[Any]										=  __import__('transformers'					,      fromlist=[model_class]  )
            A_  :       int										=  getattr(snake_case_					,      snake_case_  )
            A_  :       List[str]										=  model_cls(snake_case_  )
          except ImportError:
            raise ImportError(
                F"""{model_class} does not exist. If you just want to test the pretrained model, you might want to"""
                ' set `--only_pretrain_model` or `args.only_pretrain_model=True`.'  )
        else:
          A_  :       List[str]										=  TF_MODEL_MAPPING[config.__class__](snake_case_  )

        # encoder-decoder has vocab size saved differently
        A_  :       Optional[int]										=  config.vocab_size if hasattr(snake_case_					,      'vocab_size'  ) else config.encoder.vocab_size
        A_  :       Optional[int]										=  random_input_ids(snake_case_					,      snake_case_					,      snake_case_  )

        @run_with_tf_optimizations(self.args.eager_mode					,      self.args.use_xla  )
        def encoder_decoder_forward():
          return model(snake_case_					,      decoder_input_ids=snake_case_					,      training=snake_case_  )

        @run_with_tf_optimizations(self.args.eager_mode					,      self.args.use_xla  )
        def encoder_forward():
          return model(snake_case_					,      training=snake_case_  )

        A_  :       str										=  encoder_decoder_forward if config.is_encoder_decoder else encoder_forward

        return _inference

      def 					lowerCamelCase_			(     self					,      snake_case_					,      snake_case_					,      snake_case_  ):

        """simple docstring"""

        A_  :       List[Any]										=  self.config_dict[model_name]

        if self.args.eager_mode is not False:
          raise ValueError('Training cannot be done in eager mode. Please make sure that `args.eager_mode = False`.'  )

        if self.args.fpaa:
          raise NotImplementedError('Mixed precision is currently not supported.'  )

        A_  :       int										=  (
            hasattr(snake_case_					,      'architectures'  )
            and isinstance(config.architectures					,      snake_case_  )
            and len(config.architectures  ) > 0
        )
        if not self.args.only_pretrain_model and has_model_class_in_config:
          try:
            A_  :       Any										=  'TF' + config.architectures[0]  # prepend 'TF' for tensorflow model
            A_  :       Optional[Any]										=  __import__('transformers'					,      fromlist=[model_class]  )
            A_  :       Tuple										=  getattr(snake_case_					,      snake_case_  )
            A_  :       Union[str, Any]										=  model_cls(snake_case_  )
          except ImportError:
            raise ImportError(
                F"""{model_class} does not exist. If you just want to test the pretrained model, you might want to"""
                ' set `--only_pretrain_model` or `args.only_pretrain_model=True`.'  )
        else:
          A_  :       Any										=  TF_MODEL_WITH_LM_HEAD_MAPPING[config.__class__](snake_case_  )

        # encoder-decoder has vocab size saved differently
        A_  :       Optional[int]										=  config.vocab_size if hasattr(snake_case_					,      'vocab_size'  ) else config.encoder.vocab_size
        A_  :       Dict										=  random_input_ids(snake_case_					,      snake_case_					,      snake_case_  )

        @run_with_tf_optimizations(self.args.eager_mode					,      self.args.use_xla  )
        def encoder_decoder_train():
          A_  :       int										=  model(snake_case_					,      decoder_input_ids=snake_case_					,      labels=snake_case_					,      training=snake_case_  )[0]
          A_  :       Union[str, Any]										=  tf.gradients(snake_case_					,      model.trainable_variables  )
          return gradients

        @run_with_tf_optimizations(self.args.eager_mode					,      self.args.use_xla  )
        def encoder_train():
          A_  :       Any										=  model(snake_case_					,      labels=snake_case_					,      training=snake_case_  )[0]
          A_  :       List[Any]										=  tf.gradients(snake_case_					,      model.trainable_variables  )
          return gradients

        A_  :       Tuple										=  encoder_decoder_train if config.is_encoder_decoder else encoder_train

        return _train

      def 					lowerCamelCase_			(     self					,      snake_case_  ):

        """simple docstring"""

        with self.args.strategy.scope():
          try:
            if self.args.is_tpu or self.args.use_xla:
              # run additional 10 times to stabilize compilation for tpu
              logger.info('Do inference on TPU. Running model 5 times to stabilize compilation'  )
              timeit.repeat(snake_case_					,      repeat=1					,      number=5  )

            # as written in https://docs.python.org/2/library/timeit.html#timeit.Timer.repeat, min should be taken rather than the average
            A_  :       List[str]										=  timeit.repeat(
                snake_case_					,      repeat=self.args.repeat					,      number=1_0					,      )

            return min(snake_case_  ) / 10.0
          except ResourceExhaustedError as e:
            self.print_fn(F"""Doesn't fit on GPU. {e}"""  )

      def 					lowerCamelCase_			(     self					,      snake_case_  ):

        """simple docstring"""

        logger.info(
            'Note that TensorFlow allocates more memory than '
            'it might need to speed up computation. '
            'The memory reported here corresponds to the memory '
            'reported by `nvidia-smi`, which can vary depending '
            'on total available memory on the GPU that is used.'  )
        with self.args.strategy.scope():
          try:
            if self.args.trace_memory_line_by_line:
              if not self.args.eager_mode:
                raise ValueError(
                    '`args.eager_mode` is set to `False`. Make sure to run model in eager mode to measure memory'
                    ' consumption line by line.'  )
              A_  :       Optional[Any]										=  start_memory_tracing('transformers'  )

            if self.args.is_tpu:
              # tpu
              raise NotImplementedError(
                  'Memory Benchmarking is currently not implemented for TPU. Please disable memory benchmarking'
                  ' with `args.memory=False`'  )
            elif self.args.is_gpu:
              # gpu
              if not is_pyanvml_available():
                logger.warning(
                    'py3nvml not installed, we won\'t log GPU memory usage. '
                    'Install py3nvml (pip install py3nvml) to log information about GPU.'  )
                A_  :       Tuple										=  'N/A'
              else:
                logger.info(
                    'Measuring total GPU usage on GPU device. Make sure to not have additional processes'
                    ' running on the same GPU.'  )
                # init nvml
                nvml.nvmlInit()
                func()
                A_  :       List[str]										=  nvml.nvmlDeviceGetHandleByIndex(self.args.device_idx  )
                A_  :       int										=  nvml.nvmlDeviceGetMemoryInfo(snake_case_  )
                A_  :       int										=  meminfo.used
                A_  :       int										=  Memory(snake_case_  )
                # shutdown nvml
                nvml.nvmlShutdown()
            else:
              # cpu
              if self.args.trace_memory_line_by_line:
                logger.info(
                    'When enabling line by line tracing, the max peak memory for CPU is inaccurate in'
                    ' TensorFlow.'  )
                A_  :       Any										=  None
              else:
                A_  :       List[str]										=  measure_peak_memory_cpu(snake_case_  )
                A_  :       Any										=  Memory(snake_case_  ) if isinstance(snake_case_					,      snake_case_  ) else memory_bytes
            if self.args.trace_memory_line_by_line:
              A_  :       List[str]										=  stop_memory_tracing(snake_case_  )
              if memory is None:
                A_  :       Union[str, Any]										=  summary.total
            else:
              A_  :       Any										=  None

            return memory, summary
          except ResourceExhaustedError as e:
            self.print_fn(F"""Doesn't fit on GPU. {e}"""  )
            return "N/A", None






"""simple docstring"""




import copy
import random

from transformers import CLIPTokenizer

class  _UpperCAmelCase  (   UpperCAmelCase__		):



      '''simple docstring'''

      def __init__(     self					,      *snake_case_					,      **snake_case_  ):

        """simple docstring"""

        super().__init__(*snake_case_					,      **snake_case_  )
        A_  :       Tuple										=  {}

      def 					lowerCamelCase_			(     self					,      snake_case_					,      *snake_case_					,      **snake_case_  ):

        """simple docstring"""

        A_  :       str										=  super().add_tokens(snake_case_					,      *snake_case_					,      **snake_case_  )
        if num_added_tokens == 0:
          raise ValueError(
              F"""The tokenizer already contains the token {placeholder_token}. Please pass a different"""
              ' `placeholder_token` that is not already in the tokenizer.'  )

      def 					lowerCamelCase_			(     self					,      snake_case_					,      *snake_case_					,      snake_case_=1					,      **snake_case_  ):

        """simple docstring"""

        A_  :       Tuple										=  []
        if num_vec_per_token == 1:
          self.try_adding_tokens(snake_case_					,      *snake_case_					,      **snake_case_  )
          output.append(snake_case_  )
        else:
          A_  :       Tuple										=  []
          for i in range(snake_case_  ):
            A_  :       List[str]										=  placeholder_token + F"""_{i}"""
            self.try_adding_tokens(snake_case_					,      *snake_case_					,      **snake_case_  )
            output.append(snake_case_  )
        # handle cases where there is a new placeholder token that contains the current placeholder token but is larger
        for token in self.token_map:
          if token in placeholder_token:
            raise ValueError(
                F"""The tokenizer already has placeholder token {token} that can get confused with"""
                F""" {placeholder_token}keep placeholder tokens independent"""  )
        A_  :       Any										=  output

      def 					lowerCamelCase_			(     self					,      snake_case_					,      snake_case_=False					,      snake_case_=1.0  ):

        """simple docstring"""

        if isinstance(snake_case_					,      snake_case_  ):
          A_  :       Optional[Any]										=  []
          for i in range(len(snake_case_  )  ):
            output.append(self.replace_placeholder_tokens_in_text(text[i]					,      vector_shuffle=snake_case_  )  )
          return output
        for placeholder_token in self.token_map:
          if placeholder_token in text:
            A_  :       List[Any]										=  self.token_map[placeholder_token]
            A_  :       Optional[int]										=  tokens[: 1 + int(len(snake_case_  ) * prop_tokens_to_load  )]
            if vector_shuffle:
              A_  :       Optional[Any]										=  copy.copy(snake_case_  )
              random.shuffle(snake_case_  )
            A_  :       List[str]										=  text.replace(snake_case_					,      ' '.join(snake_case_  )  )
        return text

      def __call__(     self					,      snake_case_					,      *snake_case_					,      snake_case_=False					,      snake_case_=1.0					,      **snake_case_  ):

        """simple docstring"""

        return super().__call__(
            self.replace_placeholder_tokens_in_text(
                snake_case_					,      vector_shuffle=snake_case_					,      prop_tokens_to_load=snake_case_  )					,      *snake_case_					,      **snake_case_					,      )

      def 					lowerCamelCase_			(     self					,      snake_case_					,      *snake_case_					,      snake_case_=False					,      snake_case_=1.0					,      **snake_case_  ):

        """simple docstring"""

        return super().encode(
            self.replace_placeholder_tokens_in_text(
                snake_case_					,      vector_shuffle=snake_case_					,      prop_tokens_to_load=snake_case_  )					,      *snake_case_					,      **snake_case_					,      )