


import random
import timeit
from functools import wraps
from typing import Callable, Optional

from ..configuration_utils import PretrainedConfig
from ..models.auto.modeling_tf_auto import TF_MODEL_MAPPING, TF_MODEL_WITH_LM_HEAD_MAPPING
from ..utils import is_pyanvml_available, is_tf_available, logging
from .benchmark_utils import (
    Benchmark,
    Memory,
    MemorySummary,
    measure_peak_memory_cpu,
    start_memory_tracing,
    stop_memory_tracing,
)


if is_tf_available():
 import tensorflow as tf
 from tensorflow.python.framework.errors_impl import ResourceExhaustedError

 from .benchmark_args_tf import TensorFlowBenchmarkArguments

if is_pyanvml_available():
 import pyanvml.pyanvml as nvml

SCREAMING_SNAKE_CASE       :      str												= logging.get_logger(__name__)



def 				UpperCamelCase_(      lowerCamelCase_							, lowerCamelCase_ )    ->						List[Any]:
    def run_func(lowerCamelCase_ ):
        @wraps(lowerCamelCase_ )
        def run_in_eager_mode(*lowerCamelCase_							, **lowerCamelCase_ ):
            return func(*lowerCamelCase_							, **lowerCamelCase_ )

        @wraps(lowerCamelCase_ )
        @tf.function(experimental_compile=lowerCamelCase_ )
        def run_in_graph_mode(*lowerCamelCase_							, **lowerCamelCase_ ):
            return func(*lowerCamelCase_							, **lowerCamelCase_ )

        if do_eager_mode is True:
            if use_xla is not False:
                raise ValueError(
                    'Cannot run model in XLA, if `args.eager_mode` is set to `True`. Please set `args.eager_mode=False`.' )
            return run_in_eager_mode
        else:
            return run_in_graph_mode

    return run_func



def 				UpperCamelCase_(      lowerCamelCase_							, lowerCamelCase_							, lowerCamelCase_ )    ->						["tf.Tensor"]:
    _lowercase       :							int      		=		random.Random()
    _lowercase       :							List[Any]      		=		[rng.randint(0							, vocab_size - 1 ) for i in range(batch_size * sequence_length )]
    return tf.constant(lowerCamelCase_							, shape=(batch_size, sequence_length)							, dtype=tf.intaa )






class _lowerCamelCase(						_a   ):
       lowercase_ :	TensorFlowBenchmarkArguments
       lowercase_ :	PretrainedConfig
       lowercase_ :	str     =			"TensorFlow"
       @property
       def 			UpperCamelCase     (					self)       ->   Optional[Any]:




           """simple docstring"""

           return tf.__version__
       def 			UpperCamelCase     (					self,      lowerCamelCase,      lowerCamelCase,      lowerCamelCase)       ->   float:




           """simple docstring"""

           _lowercase       :							Union[str, Any]      		=		self.args.strategy
           if strategy is None:
               raise ValueError('A device strategy has to be initialized before using TensorFlow.')
           _lowercase       :							Optional[int]      		=		self._prepare_inference_func(lowerCamelCase,      lowerCamelCase,      lowerCamelCase)
           return self._measure_speed(_inference)
       def 			UpperCamelCase     (					self,      lowerCamelCase,      lowerCamelCase,      lowerCamelCase)       ->   float:




           """simple docstring"""

           _lowercase       :							Dict      		=		self.args.strategy
           if strategy is None:
               raise ValueError('A device strategy has to be initialized before using TensorFlow.')
           _lowercase       :							str      		=		self._prepare_train_func(lowerCamelCase,      lowerCamelCase,      lowerCamelCase)
           return self._measure_speed(_train)
       def 			UpperCamelCase     (					self,      lowerCamelCase,      lowerCamelCase,      lowerCamelCase)       ->   [Memory, Optional[MemorySummary]]:




           """simple docstring"""

           if self.args.is_gpu:
               tf.config.experimental.set_memory_growth(self.args.gpu_list[self.args.device_idx],      lowerCamelCase)
           _lowercase       :							Optional[int]      		=		self.args.strategy
           if strategy is None:
               raise ValueError('A device strategy has to be initialized before using TensorFlow.')
           _lowercase       :							Any      		=		self._prepare_inference_func(lowerCamelCase,      lowerCamelCase,      lowerCamelCase)
           return self._measure_memory(_inference)
       def 			UpperCamelCase     (					self,      lowerCamelCase,      lowerCamelCase,      lowerCamelCase)       ->   [Memory, Optional[MemorySummary]]:




           """simple docstring"""

           if self.args.is_gpu:
               tf.config.experimental.set_memory_growth(self.args.gpu_list[self.args.device_idx],      lowerCamelCase)
           _lowercase       :							Optional[int]      		=		self.args.strategy
           if strategy is None:
               raise ValueError('A device strategy has to be initialized before using TensorFlow.')

           _lowercase       :							Union[str, Any]      		=		self._prepare_train_func(lowerCamelCase,      lowerCamelCase,      lowerCamelCase)
           return self._measure_memory(_train)
       def 			UpperCamelCase     (					self,      lowerCamelCase,      lowerCamelCase,      lowerCamelCase)       ->   Callable[[], None]:




           """simple docstring"""

           _lowercase       :							List[str]      		=		self.config_dict[model_name]

           if self.args.fpaa:
               raise NotImplementedError('Mixed precision is currently not supported.')

           _lowercase       :							Union[str, Any]      		=		(
               hasattr(lowerCamelCase,      'architectures')
               and isinstance(config.architectures,      lowerCamelCase)
               and len(config.architectures) > 0
           )
           if not self.args.only_pretrain_model and has_model_class_in_config:
               try:
                   _lowercase       :							List[Any]      		=		'TF' + config.architectures[0]  # prepend 'TF' for tensorflow model
                   _lowercase       :							Union[str, Any]      		=		__import__('transformers',      fromlist=[model_class])
                   _lowercase       :							Dict      		=		getattr(lowerCamelCase,      lowerCamelCase)
                   _lowercase       :							Union[str, Any]      		=		model_cls(lowerCamelCase)
               except ImportError:
                   raise ImportError(
                       F'''{model_class} does not exist. If you just want to test the pretrained model, you might want to'''
                       ' set `--only_pretrain_model` or `args.only_pretrain_model=True`.')
           else:
               _lowercase       :							Dict      		=		TF_MODEL_MAPPING[config.__class__](lowerCamelCase)

           # encoder-decoder has vocab size saved differently
           _lowercase       :							Tuple      		=		config.vocab_size if hasattr(lowerCamelCase,      'vocab_size') else config.encoder.vocab_size
           _lowercase       :							List[Any]      		=		random_input_ids(lowerCamelCase,      lowerCamelCase,      lowerCamelCase)

           @run_with_tf_optimizations(self.args.eager_mode,      self.args.use_xla)
           def encoder_decoder_forward():
               return model(lowerCamelCase,      decoder_input_ids=lowerCamelCase,      training=lowerCamelCase)

           @run_with_tf_optimizations(self.args.eager_mode,      self.args.use_xla)
           def encoder_forward():
               return model(lowerCamelCase,      training=lowerCamelCase)

           _lowercase       :							str      		=		encoder_decoder_forward if config.is_encoder_decoder else encoder_forward

           return _inference
       def 			UpperCamelCase     (					self,      lowerCamelCase,      lowerCamelCase,      lowerCamelCase)       ->   Callable[[], None]:




           """simple docstring"""

           _lowercase       :							List[Any]      		=		self.config_dict[model_name]

           if self.args.eager_mode is not False:
               raise ValueError('Training cannot be done in eager mode. Please make sure that `args.eager_mode = False`.')

           if self.args.fpaa:
               raise NotImplementedError('Mixed precision is currently not supported.')

           _lowercase       :							Optional[Any]      		=		(
               hasattr(lowerCamelCase,      'architectures')
               and isinstance(config.architectures,      lowerCamelCase)
               and len(config.architectures) > 0
           )
           if not self.args.only_pretrain_model and has_model_class_in_config:
               try:
                   _lowercase       :							List[Any]      		=		'TF' + config.architectures[0]  # prepend 'TF' for tensorflow model
                   _lowercase       :							str      		=		__import__('transformers',      fromlist=[model_class])
                   _lowercase       :							Tuple      		=		getattr(lowerCamelCase,      lowerCamelCase)
                   _lowercase       :							Optional[int]      		=		model_cls(lowerCamelCase)
               except ImportError:
                   raise ImportError(
                       F'''{model_class} does not exist. If you just want to test the pretrained model, you might want to'''
                       ' set `--only_pretrain_model` or `args.only_pretrain_model=True`.')
           else:
               _lowercase       :							str      		=		TF_MODEL_WITH_LM_HEAD_MAPPING[config.__class__](lowerCamelCase)

           # encoder-decoder has vocab size saved differently
           _lowercase       :							List[str]      		=		config.vocab_size if hasattr(lowerCamelCase,      'vocab_size') else config.encoder.vocab_size
           _lowercase       :							Union[str, Any]      		=		random_input_ids(lowerCamelCase,      lowerCamelCase,      lowerCamelCase)

           @run_with_tf_optimizations(self.args.eager_mode,      self.args.use_xla)
           def encoder_decoder_train():
               _lowercase       :							Any      		=		model(lowerCamelCase,      decoder_input_ids=lowerCamelCase,      labels=lowerCamelCase,      training=lowerCamelCase)[0]
               _lowercase       :							List[str]      		=		tf.gradients(lowerCamelCase,      model.trainable_variables)
               return gradients

           @run_with_tf_optimizations(self.args.eager_mode,      self.args.use_xla)
           def encoder_train():
               _lowercase       :							Optional[Any]      		=		model(lowerCamelCase,      labels=lowerCamelCase,      training=lowerCamelCase)[0]
               _lowercase       :							List[str]      		=		tf.gradients(lowerCamelCase,      model.trainable_variables)
               return gradients

           _lowercase       :							str      		=		encoder_decoder_train if config.is_encoder_decoder else encoder_train

           return _train
       def 			UpperCamelCase     (					self,      lowerCamelCase)       ->   float:




           """simple docstring"""

           with self.args.strategy.scope():
               try:
                   if self.args.is_tpu or self.args.use_xla:
                       # run additional 10 times to stabilize compilation for tpu
                       logger.info('Do inference on TPU. Running model 5 times to stabilize compilation')
                       timeit.repeat(lowerCamelCase,      repeat=1,      number=5)

                   # as written in https://docs.python.org/2/library/timeit.html#timeit.Timer.repeat, min should be taken rather than the average
                   _lowercase       :							Dict      		=		timeit.repeat(
                       lowerCamelCase,      repeat=self.args.repeat,      number=10,      )

                   return min(lowerCamelCase) / 1_0.0
               except ResourceExhaustedError as e:
                   self.print_fn(F'''Doesn\'t fit on GPU. {e}''')




       def 			UpperCamelCase     (					self,      lowerCamelCase)       ->   [Memory, MemorySummary]:




           """simple docstring"""

           logger.info(
               'Note that TensorFlow allocates more memory than '
               'it might need to speed up computation. '
               'The memory reported here corresponds to the memory '
               'reported by `nvidia-smi`, which can vary depending '
               'on total available memory on the GPU that is used.')
           with self.args.strategy.scope():
               try:
                   if self.args.trace_memory_line_by_line:
                       if not self.args.eager_mode:
                           raise ValueError(
                               '`args.eager_mode` is set to `False`. Make sure to run model in eager mode to measure memory'
                               ' consumption line by line.')
                       _lowercase       :							str      		=		start_memory_tracing('transformers')

                   if self.args.is_tpu:
                       # tpu
                       raise NotImplementedError(
                           'Memory Benchmarking is currently not implemented for TPU. Please disable memory benchmarking'
                           ' with `args.memory=False`')
                   elif self.args.is_gpu:
                       # gpu
                       if not is_pyanvml_available():
                           logger.warning(
                               'py3nvml not installed, we won\'t log GPU memory usage. '
                               'Install py3nvml (pip install py3nvml) to log information about GPU.')
                           _lowercase       :							Dict      		=		'N/A'
                       else:
                           logger.info(
                               'Measuring total GPU usage on GPU device. Make sure to not have additional processes'
                               ' running on the same GPU.')
                           # init nvml
                           nvml.nvmlInit()
                           func()
                           _lowercase       :							int      		=		nvml.nvmlDeviceGetHandleByIndex(self.args.device_idx)
                           _lowercase       :							str      		=		nvml.nvmlDeviceGetMemoryInfo(lowerCamelCase)
                           _lowercase       :							Dict      		=		meminfo.used
                           _lowercase       :							Any      		=		Memory(lowerCamelCase)
                           # shutdown nvml
                           nvml.nvmlShutdown()
                   else:
                       # cpu
                       if self.args.trace_memory_line_by_line:
                           logger.info(
                               'When enabling line by line tracing, the max peak memory for CPU is inaccurate in'
                               ' TensorFlow.')
                           _lowercase       :							str      		=		None
                       else:
                           _lowercase       :							Any      		=		measure_peak_memory_cpu(lowerCamelCase)
                           _lowercase       :							Dict      		=		Memory(lowerCamelCase) if isinstance(lowerCamelCase,      lowerCamelCase) else memory_bytes
                   if self.args.trace_memory_line_by_line:
                       _lowercase       :							Optional[int]      		=		stop_memory_tracing(lowerCamelCase)
                       if memory is None:
                           _lowercase       :							List[str]      		=		summary.total
                   else:
                       _lowercase       :							Optional[int]      		=		None

                   return memory, summary
               except ResourceExhaustedError as e:
                   self.print_fn(F'''Doesn\'t fit on GPU. {e}''')
                   return "N/A", None






"""simple docstring"""
import numpy as np






def 		lowercase							(     _snake_case						:      int     ,   _snake_case						:      Optional[Any]     ,   _snake_case						:      Optional[int]     ,   _snake_case						:      int     ,   _snake_case						:      Union[str, Any]     )     ->Dict:
	"""simple docstring"""





	__snake_case	:						Union[str, Any]						=				int(np.ceil((x_end - xa) / h     )     )
	__snake_case	:						Dict						=				np.zeros((n + 1,)     )
	__snake_case	:						List[Any]						=				ya
	__snake_case	:						int						=				xa

	for k in range(_snake_case     ):
		__snake_case	:						Any						=				f(_snake_case     ,   y[k]     )
		__snake_case	:						List[Any]						=				f(x + 0.5 * h     ,   y[k] + 0.5 * h * ka     )
		__snake_case	:						int						=				f(x + 0.5 * h     ,   y[k] + 0.5 * h * ka     )
		__snake_case	:						Optional[int]						=				f(x + h     ,   y[k] + h * ka     )
		__snake_case	:						Optional[int]						=				y[k] + (1 / 6) * h * (ka + 2 * ka + 2 * ka + ka)
		x += h

	return y


if __name__ == "__main__":
			import doctest

			doctest.testmod()





