


"""simple docstring"""







from math import sqrt




def    UpperCAmelCase__       (lowerCAmelCase_   ):




					'''simple docstring'''


					assert isinstance(lowerCAmelCase_  ,						lowerCAmelCase_   ) and (
					    number >= 0
					), "'number' must been an int and positive"

					__SCREAMING_SNAKE_CASE						   =				True

					# 0 and 1 are none primes.
					if number <= 1:
										__SCREAMING_SNAKE_CASE						   =				False

					for divisor in range(2  ,						int(round(sqrt(lowerCAmelCase_   )   )   ) + 1   ):
										# if 'number' divisible by 'divisor' then sets 'status'
										# of false and break up the loop.
										if number % divisor == 0:
															__SCREAMING_SNAKE_CASE						   =				False
															break

    # precondition
					assert isinstance(lowerCAmelCase_  ,						lowerCAmelCase_   ), "'status' must been from type bool"

					return status




def    UpperCAmelCase__       (lowerCAmelCase_   ):




					'''simple docstring'''


					assert isinstance(lowerCAmelCase_  ,						lowerCAmelCase_   ) and (n > 2), "'N' must been an int and > 2"

					# beginList: contains all natural numbers from 2 up to N
					__SCREAMING_SNAKE_CASE						   =				list(range(2  ,						n + 1   )   )

					__SCREAMING_SNAKE_CASE						   =				[]  # this list will be returns.

					# actual sieve of erathostenes
					for i in range(len(lowerCAmelCase_   )   ):
										for j in range(i + 1  ,						len(lowerCAmelCase_   )   ):
															if (begin_list[i] != 0) and (begin_list[j] % begin_list[i] == 0):
																				__SCREAMING_SNAKE_CASE						   =				0

    # filters actual prime numbers.
					__SCREAMING_SNAKE_CASE						   =				[x for x in begin_list if x != 0]

					# precondition
					assert isinstance(lowerCAmelCase_  ,						lowerCAmelCase_   ), "'ans' must been from type list"

					return ans




def    UpperCAmelCase__       (lowerCAmelCase_   ):




					'''simple docstring'''


					assert isinstance(lowerCAmelCase_  ,						lowerCAmelCase_   ) and (n > 2), "'N' must been an int and > 2"

					__SCREAMING_SNAKE_CASE						   =				[]

					# iterates over all numbers between 2 up to N+1
					# if a number is prime then appends to list 'ans'
					for number in range(2  ,						n + 1   ):
										if is_prime(lowerCAmelCase_   ):
															ans.append(lowerCAmelCase_   )

    # precondition
					assert isinstance(lowerCAmelCase_  ,						lowerCAmelCase_   ), "'ans' must been from type list"

					return ans




def    UpperCAmelCase__       (lowerCAmelCase_   ):




					'''simple docstring'''


					assert isinstance(lowerCAmelCase_  ,						lowerCAmelCase_   ) and number >= 0, "'number' must been an int and >= 0"

					__SCREAMING_SNAKE_CASE						   =				[]  # this list will be returns of the function.

					# potential prime number factors.

					__SCREAMING_SNAKE_CASE						   =				2

					__SCREAMING_SNAKE_CASE						   =				number

					if number == 0 or number == 1:
										ans.append(lowerCAmelCase_   )

					# if 'number' not prime then builds the prime factorization of 'number'
					elif not is_prime(lowerCAmelCase_   ):
										while quotient != 1:
															if is_prime(lowerCAmelCase_   ) and (quotient % factor == 0):
																				ans.append(lowerCAmelCase_   )
																				quotient /= factor
															else:
																				factor += 1

					else:
										ans.append(lowerCAmelCase_   )

					# precondition
					assert isinstance(lowerCAmelCase_  ,						lowerCAmelCase_   ), "'ans' must been from type list"

					return ans




def    UpperCAmelCase__       (lowerCAmelCase_   ):




					'''simple docstring'''


					assert isinstance(lowerCAmelCase_  ,						lowerCAmelCase_   ) and (
					    number >= 0
					), "'number' bust been an int and >= 0"

					__SCREAMING_SNAKE_CASE						   =				0

					# prime factorization of 'number'
					__SCREAMING_SNAKE_CASE						   =				prime_factorization(lowerCAmelCase_   )

					__SCREAMING_SNAKE_CASE						   =				max(lowerCAmelCase_   )

					# precondition
					assert isinstance(lowerCAmelCase_  ,						lowerCAmelCase_   ), "'ans' must been from type int"

					return ans




def    UpperCAmelCase__       (lowerCAmelCase_   ):




					'''simple docstring'''


					assert isinstance(lowerCAmelCase_  ,						lowerCAmelCase_   ) and (
					    number >= 0
					), "'number' bust been an int and >= 0"

					__SCREAMING_SNAKE_CASE						   =				0

					# prime factorization of 'number'
					__SCREAMING_SNAKE_CASE						   =				prime_factorization(lowerCAmelCase_   )

					__SCREAMING_SNAKE_CASE						   =				min(lowerCAmelCase_   )

					# precondition
					assert isinstance(lowerCAmelCase_  ,						lowerCAmelCase_   ), "'ans' must been from type int"

					return ans




def    UpperCAmelCase__       (lowerCAmelCase_   ):




					'''simple docstring'''


					assert isinstance(lowerCAmelCase_  ,						lowerCAmelCase_   ), "'number' must been an int"
					assert isinstance(number % 2 == 0  ,						lowerCAmelCase_   ), "compare bust been from type bool"

					return number % 2 == 0




def    UpperCAmelCase__       (lowerCAmelCase_   ):




					'''simple docstring'''


					assert isinstance(lowerCAmelCase_  ,						lowerCAmelCase_   ), "'number' must been an int"
					assert isinstance(number % 2 != 0  ,						lowerCAmelCase_   ), "compare bust been from type bool"

					return number % 2 != 0




def    UpperCAmelCase__       (lowerCAmelCase_   ):




					'''simple docstring'''


					assert (
					    isinstance(lowerCAmelCase_  ,						lowerCAmelCase_   ) and (number > 2) and is_even(lowerCAmelCase_   )
					), "'number' must been an int, even and > 2"

					__SCREAMING_SNAKE_CASE						   =				[]  # this list will returned

					# creates a list of prime numbers between 2 up to 'number'
					__SCREAMING_SNAKE_CASE						   =				get_prime_numbers(lowerCAmelCase_   )
					__SCREAMING_SNAKE_CASE						   =				len(lowerCAmelCase_   )

					# run variable for while-loops.
					__SCREAMING_SNAKE_CASE						   =				0
					__SCREAMING_SNAKE_CASE						   =				None

					# exit variable. for break up the loops
					__SCREAMING_SNAKE_CASE						   =				True

					while i < len_pn and loop:
										__SCREAMING_SNAKE_CASE						   =				i + 1

										while j < len_pn and loop:
															if prime_numbers[i] + prime_numbers[j] == number:
																				__SCREAMING_SNAKE_CASE						   =				False
																				ans.append(prime_numbers[i]   )
																				ans.append(prime_numbers[j]   )

															j += 1

										i += 1

					# precondition
					assert (
					    isinstance(lowerCAmelCase_  ,						lowerCAmelCase_   )
					    and (len(lowerCAmelCase_   ) == 2)
					    and (ans[0] + ans[1] == number)
					    and is_prime(ans[0]   )
					    and is_prime(ans[1]   )
					), "'ans' must contains two primes. And sum of elements must been eq 'number'"

					return ans




def    UpperCAmelCase__       (lowerCAmelCase_  ,						lowerCAmelCase_   ):




					'''simple docstring'''


					assert (
					    isinstance(lowerCAmelCase_  ,						lowerCAmelCase_   )
					    and isinstance(lowerCAmelCase_  ,						lowerCAmelCase_   )
					    and (numbera >= 0)
					    and (numbera >= 0)
					), "'number1' and 'number2' must been positive integer."

					__SCREAMING_SNAKE_CASE						   =				0

					while numbera != 0:
										__SCREAMING_SNAKE_CASE						   =				numbera % numbera
										__SCREAMING_SNAKE_CASE						   =				numbera
										__SCREAMING_SNAKE_CASE						   =				rest

					# precondition
					assert isinstance(lowerCAmelCase_  ,						lowerCAmelCase_   ) and (
					    numbera >= 0
					), "'number' must been from type int and positive"

					return numbera




def    UpperCAmelCase__       (lowerCAmelCase_  ,						lowerCAmelCase_   ):




					'''simple docstring'''


					assert (
					    isinstance(lowerCAmelCase_  ,						lowerCAmelCase_   )
					    and isinstance(lowerCAmelCase_  ,						lowerCAmelCase_   )
					    and (numbera >= 1)
					    and (numbera >= 1)
					), "'number1' and 'number2' must been positive integer."

					__SCREAMING_SNAKE_CASE						   =				1  # actual answer that will be return.

					# for kgV (x,1)
					if numbera > 1 and numbera > 1:
										# builds the prime factorization of 'number1' and 'number2'
										__SCREAMING_SNAKE_CASE						   =				prime_factorization(lowerCAmelCase_   )
										__SCREAMING_SNAKE_CASE						   =				prime_factorization(lowerCAmelCase_   )

					elif numbera == 1 or numbera == 1:
										__SCREAMING_SNAKE_CASE						   =				[]
										__SCREAMING_SNAKE_CASE						   =				[]
										__SCREAMING_SNAKE_CASE						   =				max(lowerCAmelCase_  ,						lowerCAmelCase_   )

					__SCREAMING_SNAKE_CASE						   =				0
					__SCREAMING_SNAKE_CASE						   =				0

					__SCREAMING_SNAKE_CASE						   =				[]  # captured numbers int both 'primeFac1' and 'primeFac2'

					# iterates through primeFac1
					for n in prime_fac_a:
										if n not in done:
															if n in prime_fac_a:
																				__SCREAMING_SNAKE_CASE						   =				prime_fac_a.count(lowerCAmelCase_   )
																				__SCREAMING_SNAKE_CASE						   =				prime_fac_a.count(lowerCAmelCase_   )

																				for _ in range(max(lowerCAmelCase_  ,						lowerCAmelCase_   )   ):
																									ans *= n

															else:
																				__SCREAMING_SNAKE_CASE						   =				prime_fac_a.count(lowerCAmelCase_   )

																				for _ in range(lowerCAmelCase_   ):
																									ans *= n

															done.append(lowerCAmelCase_   )

    # iterates through primeFac2
					for n in prime_fac_a:
										if n not in done:
															__SCREAMING_SNAKE_CASE						   =				prime_fac_a.count(lowerCAmelCase_   )

															for _ in range(lowerCAmelCase_   ):
																				ans *= n

															done.append(lowerCAmelCase_   )

    # precondition
					assert isinstance(lowerCAmelCase_  ,						lowerCAmelCase_   ) and (
					    ans >= 0
					), "'ans' must been from type int and positive"

					return ans




def    UpperCAmelCase__       (lowerCAmelCase_   ):




					'''simple docstring'''


					assert isinstance(lowerCAmelCase_  ,						lowerCAmelCase_   ) and (n >= 0), "'number' must been a positive int"

					__SCREAMING_SNAKE_CASE						   =				0
					__SCREAMING_SNAKE_CASE						   =				2  # this variable holds the answer

					while index < n:
										index += 1

										ans += 1  # counts to the next number

										# if ans not prime then
										# runs to the next prime number.
										while not is_prime(lowerCAmelCase_   ):
															ans += 1

    # precondition
					assert isinstance(lowerCAmelCase_  ,						lowerCAmelCase_   ) and is_prime(
					    lowerCAmelCase_   ), "'ans' must been a prime number and from type int"

					return ans




def    UpperCAmelCase__       (lowerCAmelCase_  ,						lowerCAmelCase_   ):




					'''simple docstring'''


					assert (
					    is_prime(lowerCAmelCase_   ) and is_prime(lowerCAmelCase_   ) and (p_number_a < p_number_a)
					), "The arguments must been prime numbers and 'pNumber1' < 'pNumber2'"

					__SCREAMING_SNAKE_CASE						   =				p_number_a + 1  # jump to the next number

					__SCREAMING_SNAKE_CASE						   =				[]  # this list will be returns.

					# if number is not prime then
					# fetch the next prime number.
					while not is_prime(lowerCAmelCase_   ):
										number += 1

					while number < p_number_a:
										ans.append(lowerCAmelCase_   )

										number += 1

										# fetch the next prime number.
										while not is_prime(lowerCAmelCase_   ):
															number += 1

    # precondition
					assert (
					    isinstance(lowerCAmelCase_  ,						lowerCAmelCase_   )
					    and ans[0] != p_number_a
					    and ans[len(lowerCAmelCase_   ) - 1] != p_number_a
					), "'ans' must been a list without the arguments"

					# 'ans' contains not 'pNumber1' and 'pNumber2' !
					return ans




def    UpperCAmelCase__       (lowerCAmelCase_   ):




					'''simple docstring'''


					assert isinstance(lowerCAmelCase_  ,						lowerCAmelCase_   ) and (n >= 1), "'n' must been int and >= 1"

					__SCREAMING_SNAKE_CASE						   =				[]  # will be returned.

					for divisor in range(1  ,						n + 1   ):
										if n % divisor == 0:
															ans.append(lowerCAmelCase_   )

    # precondition
					assert ans[0] == 1 and ans[len(lowerCAmelCase_   ) - 1] == n, "Error in function getDivisiors(...)"

					return ans




def    UpperCAmelCase__       (lowerCAmelCase_   ):




					'''simple docstring'''


					assert isinstance(lowerCAmelCase_  ,						lowerCAmelCase_   ) and (
					    number > 1
					), "'number' must been an int and >= 1"

					__SCREAMING_SNAKE_CASE						   =				get_divisors(lowerCAmelCase_   )

					# precondition
					assert (
					    isinstance(lowerCAmelCase_  ,						lowerCAmelCase_   )
					    and (divisors[0] == 1)
					    and (divisors[len(lowerCAmelCase_   ) - 1] == number)
					), "Error in help-function getDivisiors(...)"

					# summed all divisors up to 'number' (exclusive), hence [:-1]
					return sum(divisors[:-1]   ) == number




def    UpperCAmelCase__       (lowerCAmelCase_  ,						lowerCAmelCase_   ):




					'''simple docstring'''


					assert (
					    isinstance(lowerCAmelCase_  ,						lowerCAmelCase_   )
					    and isinstance(lowerCAmelCase_  ,						lowerCAmelCase_   )
					    and (denominator != 0)
					), "The arguments must been from type int and 'denominator' != 0"

					# build the greatest common divisor of numerator and denominator.
					__SCREAMING_SNAKE_CASE						   =				gcd(abs(lowerCAmelCase_   )  ,						abs(lowerCAmelCase_   )   )

					# precondition
					assert (
					    isinstance(lowerCAmelCase_  ,						lowerCAmelCase_   )
					    and (numerator % gcd_of_fraction == 0)
					    and (denominator % gcd_of_fraction == 0)
					), "Error in function gcd(...,...)"

					return (numerator // gcd_of_fraction, denominator // gcd_of_fraction)




def    UpperCAmelCase__       (lowerCAmelCase_   ):




					'''simple docstring'''


					assert isinstance(lowerCAmelCase_  ,						lowerCAmelCase_   ) and (n >= 0), "'n' must been a int and >= 0"

					__SCREAMING_SNAKE_CASE						   =				1  # this will be return.

					for factor in range(1  ,						n + 1   ):
										ans *= factor

					return ans




def    UpperCAmelCase__       (lowerCAmelCase_   ):




					'''simple docstring'''


					assert isinstance(lowerCAmelCase_  ,						lowerCAmelCase_   ) and (n >= 0), "'n' must been an int and >= 0"

					__SCREAMING_SNAKE_CASE						   =				0
					__SCREAMING_SNAKE_CASE						   =				1
					__SCREAMING_SNAKE_CASE						   =				1  # this will be return

					for _ in range(n - 1   ):
										__SCREAMING_SNAKE_CASE						   =				ans
										ans += fiba
										__SCREAMING_SNAKE_CASE						   =				tmp

					return ans

"""simple docstring"""


import random
import timeit
from functools import wraps
from typing import Callable, Optional

from ..configuration_utils import PretrainedConfig
from ..models.auto.modeling_tf_auto import TF_MODEL_MAPPING, TF_MODEL_WITH_LM_HEAD_MAPPING
from ..utils import is_pyanvml_available, is_tf_available, logging
from .benchmark_utils import (
    Benchmark,
    Memory,
    MemorySummary,
    measure_peak_memory_cpu,
    start_memory_tracing,
    stop_memory_tracing,
)


if is_tf_available():
    import tensorflow as tf
    from tensorflow.python.framework.errors_impl import ResourceExhaustedError

    from .benchmark_args_tf import TensorFlowBenchmarkArguments

if is_pyanvml_available():
    import pyanvml.pyanvml as nvml

UpperCAmelCase__     						=  logging.get_logger(__name__)






def 			__UpperCAmelCase    (	lowercase      ,lowercase						):







  """simple docstring"""
  def run_func(lowercase						):
    @wraps(lowercase						)
    def run_in_eager_mode(*lowercase      ,**lowercase						):
      return func(*lowercase      ,**lowercase						)

    @wraps(lowercase						)
    @tf.function(experimental_compile=lowercase						)
    def run_in_graph_mode(*lowercase      ,**lowercase						):
      return func(*lowercase      ,**lowercase						)

    if do_eager_mode is True:
      if use_xla is not False:
        raise ValueError(
            """Cannot run model in XLA, if `args.eager_mode` is set to `True`. Please set `args.eager_mode=False`."""						)
      return run_in_eager_mode
    else:
      return run_in_graph_mode

  return run_func






def 			__UpperCAmelCase    (	lowercase      ,lowercase      ,lowercase						):







  """simple docstring"""
  _UpperCAmelCase         =					random.Random()
  _UpperCAmelCase         =					[rng.randint(0      ,vocab_size - 1						) for i in range(batch_size * sequence_length						)]
  return tf.constant(lowercase      ,shape=(batch_size, sequence_length)      ,dtype=tf.intaa						)


class 				a (						lowerCAmelCase_      ):
    _snake_case   :     TensorFlowBenchmarkArguments
    _snake_case   :     PretrainedConfig
    _snake_case   :     str   							=							"TensorFlow"


    @property
    def    lowerCAmelCase_     (    self		:     Union[str, Any]    ):
      return tf.__version__


    def    lowerCAmelCase_     (    self		:     List[str]      , __lowerCAmelCase		:     str      , __lowerCAmelCase		:     int      , __lowerCAmelCase		:     int    ):
      # initialize GPU on separate process
      _UpperCAmelCase         =					self.args.strategy
      if strategy is None:
        raise ValueError("""A device strategy has to be initialized before using TensorFlow."""    )
      _UpperCAmelCase         =					self._prepare_inference_func(__lowerCAmelCase      , __lowerCAmelCase      , __lowerCAmelCase    )
      return self._measure_speed(_inference    )


    def    lowerCAmelCase_     (    self		:     Optional[int]      , __lowerCAmelCase		:     str      , __lowerCAmelCase		:     int      , __lowerCAmelCase		:     int    ):
      _UpperCAmelCase         =					self.args.strategy
      if strategy is None:
        raise ValueError("""A device strategy has to be initialized before using TensorFlow."""    )
      _UpperCAmelCase         =					self._prepare_train_func(__lowerCAmelCase      , __lowerCAmelCase      , __lowerCAmelCase    )
      return self._measure_speed(_train    )


    def    lowerCAmelCase_     (    self		:     Optional[Any]      , __lowerCAmelCase		:     str      , __lowerCAmelCase		:     int      , __lowerCAmelCase		:     int    ):
      # initialize GPU on separate process
      if self.args.is_gpu:
        tf.config.experimental.set_memory_growth(self.args.gpu_list[self.args.device_idx]      , __lowerCAmelCase    )
      _UpperCAmelCase         =					self.args.strategy
      if strategy is None:
        raise ValueError("""A device strategy has to be initialized before using TensorFlow."""    )
      _UpperCAmelCase         =					self._prepare_inference_func(__lowerCAmelCase      , __lowerCAmelCase      , __lowerCAmelCase    )
      return self._measure_memory(_inference    )


    def    lowerCAmelCase_     (    self		:     str      , __lowerCAmelCase		:     str      , __lowerCAmelCase		:     int      , __lowerCAmelCase		:     int    ):
      if self.args.is_gpu:
        tf.config.experimental.set_memory_growth(self.args.gpu_list[self.args.device_idx]      , __lowerCAmelCase    )
      _UpperCAmelCase         =					self.args.strategy
      if strategy is None:
        raise ValueError("""A device strategy has to be initialized before using TensorFlow."""    )

      _UpperCAmelCase         =					self._prepare_train_func(__lowerCAmelCase      , __lowerCAmelCase      , __lowerCAmelCase    )
      return self._measure_memory(_train    )


    def    lowerCAmelCase_     (    self		:     List[Any]      , __lowerCAmelCase		:     str      , __lowerCAmelCase		:     int      , __lowerCAmelCase		:     int    ):
      _UpperCAmelCase         =					self.config_dict[model_name]

      if self.args.fpaa:
        raise NotImplementedError("""Mixed precision is currently not supported."""    )

      _UpperCAmelCase         =					(
          hasattr(__lowerCAmelCase      , """architectures"""    )
          and isinstance(config.architectures      , __lowerCAmelCase    )
          and len(config.architectures    ) > 0
      )
      if not self.args.only_pretrain_model and has_model_class_in_config:
        try:
          _UpperCAmelCase         =					"""TF""" + config.architectures[0]  # prepend 'TF' for tensorflow model
          _UpperCAmelCase         =					__import__("""transformers"""      , fromlist=[model_class]    )
          _UpperCAmelCase         =					getattr(__lowerCAmelCase      , __lowerCAmelCase    )
          _UpperCAmelCase         =					model_cls(__lowerCAmelCase    )
        except ImportError:
          raise ImportError(
              f'''{model_class} does not exist. If you just want to test the pretrained model, you might want to'''
              """ set `--only_pretrain_model` or `args.only_pretrain_model=True`."""    )
      else:
        _UpperCAmelCase         =					TF_MODEL_MAPPING[config.__class__](__lowerCAmelCase    )

      # encoder-decoder has vocab size saved differently
      _UpperCAmelCase         =					config.vocab_size if hasattr(__lowerCAmelCase      , """vocab_size"""    ) else config.encoder.vocab_size
      _UpperCAmelCase         =					random_input_ids(__lowerCAmelCase      , __lowerCAmelCase      , __lowerCAmelCase    )

      @run_with_tf_optimizations(self.args.eager_mode      , self.args.use_xla    )
      def encoder_decoder_forward():
        return model(__lowerCAmelCase      , decoder_input_ids=__lowerCAmelCase      , training=__lowerCAmelCase    )

      @run_with_tf_optimizations(self.args.eager_mode      , self.args.use_xla    )
      def encoder_forward():
        return model(__lowerCAmelCase      , training=__lowerCAmelCase    )

      _UpperCAmelCase         =					encoder_decoder_forward if config.is_encoder_decoder else encoder_forward

      return _inference


    def    lowerCAmelCase_     (    self		:     List[str]      , __lowerCAmelCase		:     str      , __lowerCAmelCase		:     int      , __lowerCAmelCase		:     int    ):
      _UpperCAmelCase         =					self.config_dict[model_name]

      if self.args.eager_mode is not False:
        raise ValueError("""Training cannot be done in eager mode. Please make sure that `args.eager_mode = False`."""    )

      if self.args.fpaa:
        raise NotImplementedError("""Mixed precision is currently not supported."""    )

      _UpperCAmelCase         =					(
          hasattr(__lowerCAmelCase      , """architectures"""    )
          and isinstance(config.architectures      , __lowerCAmelCase    )
          and len(config.architectures    ) > 0
      )
      if not self.args.only_pretrain_model and has_model_class_in_config:
        try:
          _UpperCAmelCase         =					"""TF""" + config.architectures[0]  # prepend 'TF' for tensorflow model
          _UpperCAmelCase         =					__import__("""transformers"""      , fromlist=[model_class]    )
          _UpperCAmelCase         =					getattr(__lowerCAmelCase      , __lowerCAmelCase    )
          _UpperCAmelCase         =					model_cls(__lowerCAmelCase    )
        except ImportError:
          raise ImportError(
              f'''{model_class} does not exist. If you just want to test the pretrained model, you might want to'''
              """ set `--only_pretrain_model` or `args.only_pretrain_model=True`."""    )
      else:
        _UpperCAmelCase         =					TF_MODEL_WITH_LM_HEAD_MAPPING[config.__class__](__lowerCAmelCase    )

      # encoder-decoder has vocab size saved differently
      _UpperCAmelCase         =					config.vocab_size if hasattr(__lowerCAmelCase      , """vocab_size"""    ) else config.encoder.vocab_size
      _UpperCAmelCase         =					random_input_ids(__lowerCAmelCase      , __lowerCAmelCase      , __lowerCAmelCase    )

      @run_with_tf_optimizations(self.args.eager_mode      , self.args.use_xla    )
      def encoder_decoder_train():
        _UpperCAmelCase         =					model(__lowerCAmelCase      , decoder_input_ids=__lowerCAmelCase      , labels=__lowerCAmelCase      , training=__lowerCAmelCase    )[0]
        _UpperCAmelCase         =					tf.gradients(__lowerCAmelCase      , model.trainable_variables    )
        return gradients

      @run_with_tf_optimizations(self.args.eager_mode      , self.args.use_xla    )
      def encoder_train():
        _UpperCAmelCase         =					model(__lowerCAmelCase      , labels=__lowerCAmelCase      , training=__lowerCAmelCase    )[0]
        _UpperCAmelCase         =					tf.gradients(__lowerCAmelCase      , model.trainable_variables    )
        return gradients

      _UpperCAmelCase         =					encoder_decoder_train if config.is_encoder_decoder else encoder_train

      return _train


    def    lowerCAmelCase_     (    self		:     Union[str, Any]      , __lowerCAmelCase		:     Any    ):
      with self.args.strategy.scope():
        try:
          if self.args.is_tpu or self.args.use_xla:
            # run additional 10 times to stabilize compilation for tpu
            logger.info("""Do inference on TPU. Running model 5 times to stabilize compilation"""    )
            timeit.repeat(__lowerCAmelCase      , repeat=1      , number=5    )

          # as written in https://docs.python.org/2/library/timeit.html#timeit.Timer.repeat, min should be taken rather than the average
          _UpperCAmelCase         =					timeit.repeat(
              __lowerCAmelCase      , repeat=self.args.repeat      , number=10      , )

          return min(__lowerCAmelCase    ) / 10.0
        except ResourceExhaustedError as e:
          self.print_fn(f'''Doesn\'t fit on GPU. {e}'''    )


    def    lowerCAmelCase_     (    self		:     List[Any]      , __lowerCAmelCase		:     Callable[[], None]    ):
      logger.info(
          """Note that TensorFlow allocates more memory than """
          """it might need to speed up computation. """
          """The memory reported here corresponds to the memory """
          """reported by `nvidia-smi`, which can vary depending """
          """on total available memory on the GPU that is used."""    )
      with self.args.strategy.scope():
        try:
          if self.args.trace_memory_line_by_line:
            if not self.args.eager_mode:
              raise ValueError(
                  """`args.eager_mode` is set to `False`. Make sure to run model in eager mode to measure memory"""
                  """ consumption line by line."""    )
            _UpperCAmelCase         =					start_memory_tracing("""transformers"""    )

          if self.args.is_tpu:
            # tpu
            raise NotImplementedError(
                """Memory Benchmarking is currently not implemented for TPU. Please disable memory benchmarking"""
                """ with `args.memory=False`"""    )
          elif self.args.is_gpu:
            # gpu
            if not is_pyanvml_available():
              logger.warning(
                  """py3nvml not installed, we won't log GPU memory usage. """
                  """Install py3nvml (pip install py3nvml) to log information about GPU."""    )
              _UpperCAmelCase         =					"""N/A"""
            else:
              logger.info(
                  """Measuring total GPU usage on GPU device. Make sure to not have additional processes"""
                  """ running on the same GPU."""    )
              # init nvml
              nvml.nvmlInit()
              func()
              _UpperCAmelCase         =					nvml.nvmlDeviceGetHandleByIndex(self.args.device_idx    )
              _UpperCAmelCase         =					nvml.nvmlDeviceGetMemoryInfo(__lowerCAmelCase    )
              _UpperCAmelCase         =					meminfo.used
              _UpperCAmelCase         =					Memory(__lowerCAmelCase    )
              # shutdown nvml
              nvml.nvmlShutdown()
          else:
            # cpu
            if self.args.trace_memory_line_by_line:
              logger.info(
                  """When enabling line by line tracing, the max peak memory for CPU is inaccurate in"""
                  """ TensorFlow."""    )
              _UpperCAmelCase         =					None
            else:
              _UpperCAmelCase         =					measure_peak_memory_cpu(__lowerCAmelCase    )
              _UpperCAmelCase         =					Memory(__lowerCAmelCase    ) if isinstance(__lowerCAmelCase      , __lowerCAmelCase    ) else memory_bytes
          if self.args.trace_memory_line_by_line:
            _UpperCAmelCase         =					stop_memory_tracing(__lowerCAmelCase    )
            if memory is None:
              _UpperCAmelCase         =					summary.total
          else:
            _UpperCAmelCase         =					None

          return memory, summary
        except ResourceExhaustedError as e:
          self.print_fn(f'''Doesn\'t fit on GPU. {e}'''    )
          return "N/A", None
