






import random
import timeit
from functools import wraps
from typing import Callable, Optional

from ..configuration_utils import PretrainedConfig
from ..models.auto.modeling_tf_auto import TF_MODEL_MAPPING, TF_MODEL_WITH_LM_HEAD_MAPPING
from ..utils import is_pyanvml_available, is_tf_available, logging
from .benchmark_utils import (
    Benchmark,
    Memory,
    MemorySummary,
    measure_peak_memory_cpu,
    start_memory_tracing,
    stop_memory_tracing,
)


if is_tf_available():
							import tensorflow as tf
							from tensorflow.python.framework.errors_impl import ResourceExhaustedError

							from .benchmark_args_tf import TensorFlowBenchmarkArguments

if is_pyanvml_available():
							import pyanvml.pyanvml as nvml

A_       :int       =						logging.get_logger(__name__)






def 							A     (       a_							,a_					)     ->						Optional[int]:
					def run_func(a_					):
										@wraps(a_					)
										def run_in_eager_mode(*a_							,**a_					):
															return func(*a_							,**a_					)

										@wraps(a_					)
										@tf.function(experimental_compile=a_					)
										def run_in_graph_mode(*a_							,**a_					):
															return func(*a_							,**a_					)

										if do_eager_mode is True:
															if use_xla is not False:
																				raise ValueError(
																				    'Cannot run model in XLA, if `args.eager_mode` is set to `True`. Please set `args.eager_mode=False`.'					)
															return run_in_eager_mode
										else:
															return run_in_graph_mode

					return run_func






def 							A     (       a_							,a_							,a_					)     ->						["tf.Tensor"]:
					__UpperCamelCase      :					List[Any]       		=random.Random()
					__UpperCamelCase      :					Dict       		=[rng.randint(0							,vocab_size - 1					) for i in range(batch_size * sequence_length					)]
					return tf.constant(a_							,shape=(batch_size, sequence_length)							,dtype=tf.intaa					)




class 				__A     (			a ):





				"""simple docstring"""







				UpperCamelCase__ :      TensorFlowBenchmarkArguments
				UpperCamelCase__ :      PretrainedConfig
				UpperCamelCase__ :      str			 ="TensorFlow"


				@property
				def       __lowercase							( self    ):




									"""simple docstring"""

									return tf.__version__


				def       __lowercase							( self							,  lowerCamelCase__							,  lowerCamelCase__							,  lowerCamelCase__    ):




									"""simple docstring"""

									__UpperCamelCase      :					List[str]       		=self.args.strategy
									if strategy is None:
														raise ValueError('A device strategy has to be initialized before using TensorFlow.'    )
									__UpperCamelCase      :					Dict       		=self._prepare_inference_func(lowerCamelCase__							,  lowerCamelCase__							,  lowerCamelCase__    )
									return self._measure_speed(_inference    )


				def       __lowercase							( self							,  lowerCamelCase__							,  lowerCamelCase__							,  lowerCamelCase__    ):




									"""simple docstring"""

									__UpperCamelCase      :					Optional[int]       		=self.args.strategy
									if strategy is None:
														raise ValueError('A device strategy has to be initialized before using TensorFlow.'    )
									__UpperCamelCase      :					Union[str, Any]       		=self._prepare_train_func(lowerCamelCase__							,  lowerCamelCase__							,  lowerCamelCase__    )
									return self._measure_speed(_train    )


				def       __lowercase							( self							,  lowerCamelCase__							,  lowerCamelCase__							,  lowerCamelCase__    ):




									"""simple docstring"""

									if self.args.is_gpu:
														tf.config.experimental.set_memory_growth(self.args.gpu_list[self.args.device_idx]							,  lowerCamelCase__    )
									__UpperCamelCase      :					List[Any]       		=self.args.strategy
									if strategy is None:
														raise ValueError('A device strategy has to be initialized before using TensorFlow.'    )
									__UpperCamelCase      :					Union[str, Any]       		=self._prepare_inference_func(lowerCamelCase__							,  lowerCamelCase__							,  lowerCamelCase__    )
									return self._measure_memory(_inference    )


				def       __lowercase							( self							,  lowerCamelCase__							,  lowerCamelCase__							,  lowerCamelCase__    ):




									"""simple docstring"""

									if self.args.is_gpu:
														tf.config.experimental.set_memory_growth(self.args.gpu_list[self.args.device_idx]							,  lowerCamelCase__    )
									__UpperCamelCase      :					Tuple       		=self.args.strategy
									if strategy is None:
														raise ValueError('A device strategy has to be initialized before using TensorFlow.'    )

									__UpperCamelCase      :					Dict       		=self._prepare_train_func(lowerCamelCase__							,  lowerCamelCase__							,  lowerCamelCase__    )
									return self._measure_memory(_train    )


				def       __lowercase							( self							,  lowerCamelCase__							,  lowerCamelCase__							,  lowerCamelCase__    ):




									"""simple docstring"""

									__UpperCamelCase      :					int       		=self.config_dict[model_name]

									if self.args.fpaa:
														raise NotImplementedError('Mixed precision is currently not supported.'    )

									__UpperCamelCase      :					Dict       		=(
									    hasattr(lowerCamelCase__							,  'architectures'    )
									    and isinstance(config.architectures							,  lowerCamelCase__    )
									    and len(config.architectures    ) > 0
									)
									if not self.args.only_pretrain_model and has_model_class_in_config:
														try:
																			__UpperCamelCase      :					Optional[int]       		='TF' + config.architectures[0]  # prepend 'TF' for tensorflow model
																			__UpperCamelCase      :					List[Any]       		=__import__('transformers'							,  fromlist=[model_class]    )
																			__UpperCamelCase      :					Union[str, Any]       		=getattr(lowerCamelCase__							,  lowerCamelCase__    )
																			__UpperCamelCase      :					Tuple       		=model_cls(lowerCamelCase__    )
														except ImportError:
																			raise ImportError(
																			    f'{model_class} does not exist. If you just want to test the pretrained model, you might want to'
																			    ' set `--only_pretrain_model` or `args.only_pretrain_model=True`.'    )
									else:
														__UpperCamelCase      :					int       		=TF_MODEL_MAPPING[config.__class__](lowerCamelCase__    )

									# encoder-decoder has vocab size saved differently
									__UpperCamelCase      :					Any       		=config.vocab_size if hasattr(lowerCamelCase__							,  'vocab_size'    ) else config.encoder.vocab_size
									__UpperCamelCase      :					Any       		=random_input_ids(lowerCamelCase__							,  lowerCamelCase__							,  lowerCamelCase__    )

									@run_with_tf_optimizations(self.args.eager_mode							,  self.args.use_xla    )
									def encoder_decoder_forward():
														return model(lowerCamelCase__							,  decoder_input_ids=lowerCamelCase__							,  training=lowerCamelCase__    )

									@run_with_tf_optimizations(self.args.eager_mode							,  self.args.use_xla    )
									def encoder_forward():
														return model(lowerCamelCase__							,  training=lowerCamelCase__    )

									__UpperCamelCase      :					Optional[int]       		=encoder_decoder_forward if config.is_encoder_decoder else encoder_forward

									return _inference


				def       __lowercase							( self							,  lowerCamelCase__							,  lowerCamelCase__							,  lowerCamelCase__    ):




									"""simple docstring"""

									__UpperCamelCase      :					str       		=self.config_dict[model_name]

									if self.args.eager_mode is not False:
														raise ValueError('Training cannot be done in eager mode. Please make sure that `args.eager_mode = False`.'    )

									if self.args.fpaa:
														raise NotImplementedError('Mixed precision is currently not supported.'    )

									__UpperCamelCase      :					List[str]       		=(
									    hasattr(lowerCamelCase__							,  'architectures'    )
									    and isinstance(config.architectures							,  lowerCamelCase__    )
									    and len(config.architectures    ) > 0
									)
									if not self.args.only_pretrain_model and has_model_class_in_config:
														try:
																			__UpperCamelCase      :					List[str]       		='TF' + config.architectures[0]  # prepend 'TF' for tensorflow model
																			__UpperCamelCase      :					Any       		=__import__('transformers'							,  fromlist=[model_class]    )
																			__UpperCamelCase      :					Tuple       		=getattr(lowerCamelCase__							,  lowerCamelCase__    )
																			__UpperCamelCase      :					str       		=model_cls(lowerCamelCase__    )
														except ImportError:
																			raise ImportError(
																			    f'{model_class} does not exist. If you just want to test the pretrained model, you might want to'
																			    ' set `--only_pretrain_model` or `args.only_pretrain_model=True`.'    )
									else:
														__UpperCamelCase      :					Any       		=TF_MODEL_WITH_LM_HEAD_MAPPING[config.__class__](lowerCamelCase__    )

									# encoder-decoder has vocab size saved differently
									__UpperCamelCase      :					str       		=config.vocab_size if hasattr(lowerCamelCase__							,  'vocab_size'    ) else config.encoder.vocab_size
									__UpperCamelCase      :					str       		=random_input_ids(lowerCamelCase__							,  lowerCamelCase__							,  lowerCamelCase__    )

									@run_with_tf_optimizations(self.args.eager_mode							,  self.args.use_xla    )
									def encoder_decoder_train():
														__UpperCamelCase      :					int       		=model(lowerCamelCase__							,  decoder_input_ids=lowerCamelCase__							,  labels=lowerCamelCase__							,  training=lowerCamelCase__    )[0]
														__UpperCamelCase      :					List[Any]       		=tf.gradients(lowerCamelCase__							,  model.trainable_variables    )
														return gradients

									@run_with_tf_optimizations(self.args.eager_mode							,  self.args.use_xla    )
									def encoder_train():
														__UpperCamelCase      :					Any       		=model(lowerCamelCase__							,  labels=lowerCamelCase__							,  training=lowerCamelCase__    )[0]
														__UpperCamelCase      :					List[str]       		=tf.gradients(lowerCamelCase__							,  model.trainable_variables    )
														return gradients

									__UpperCamelCase      :					int       		=encoder_decoder_train if config.is_encoder_decoder else encoder_train

									return _train


				def       __lowercase							( self							,  lowerCamelCase__    ):




									"""simple docstring"""

									with self.args.strategy.scope():
														try:
																			if self.args.is_tpu or self.args.use_xla:
																								# run additional 10 times to stabilize compilation for tpu
																								logger.info('Do inference on TPU. Running model 5 times to stabilize compilation'    )
																								timeit.repeat(lowerCamelCase__							,  repeat=1							,  number=5    )

																			# as written in https://docs.python.org/2/library/timeit.html#timeit.Timer.repeat, min should be taken rather than the average
																			__UpperCamelCase      :					Tuple       		=timeit.repeat(
																			    lowerCamelCase__							,  repeat=self.args.repeat							,  number=10							,  )

																			return min(lowerCamelCase__    ) / 10.0
														except ResourceExhaustedError as e:
																			self.print_fn(f'Doesn\'t fit on GPU. {e}'    )







				def       __lowercase							( self							,  lowerCamelCase__    ):




									"""simple docstring"""

									logger.info(
									    'Note that TensorFlow allocates more memory than '
									    'it might need to speed up computation. '
									    'The memory reported here corresponds to the memory '
									    'reported by `nvidia-smi`, which can vary depending '
									    'on total available memory on the GPU that is used.'    )
									with self.args.strategy.scope():
														try:
																			if self.args.trace_memory_line_by_line:
																								if not self.args.eager_mode:
																													raise ValueError(
																													    '`args.eager_mode` is set to `False`. Make sure to run model in eager mode to measure memory'
																													    ' consumption line by line.'    )
																								__UpperCamelCase      :					Tuple       		=start_memory_tracing('transformers'    )

																			if self.args.is_tpu:
																								# tpu
																								raise NotImplementedError(
																								    'Memory Benchmarking is currently not implemented for TPU. Please disable memory benchmarking'
																								    ' with `args.memory=False`'    )
																			elif self.args.is_gpu:
																								# gpu
																								if not is_pyanvml_available():
																													logger.warning(
																													    'py3nvml not installed, we won\'t log GPU memory usage. '
																													    'Install py3nvml (pip install py3nvml) to log information about GPU.'    )
																													__UpperCamelCase      :					Union[str, Any]       		='N/A'
																								else:
																													logger.info(
																													    'Measuring total GPU usage on GPU device. Make sure to not have additional processes'
																													    ' running on the same GPU.'    )
																													# init nvml
																													nvml.nvmlInit()
																													func()
																													__UpperCamelCase      :					str       		=nvml.nvmlDeviceGetHandleByIndex(self.args.device_idx    )
																													__UpperCamelCase      :					Tuple       		=nvml.nvmlDeviceGetMemoryInfo(lowerCamelCase__    )
																													__UpperCamelCase      :					str       		=meminfo.used
																													__UpperCamelCase      :					Any       		=Memory(lowerCamelCase__    )
																													# shutdown nvml
																													nvml.nvmlShutdown()
																			else:
																								# cpu
																								if self.args.trace_memory_line_by_line:
																													logger.info(
																													    'When enabling line by line tracing, the max peak memory for CPU is inaccurate in'
																													    ' TensorFlow.'    )
																													__UpperCamelCase      :					Optional[int]       		=None
																								else:
																													__UpperCamelCase      :					Dict       		=measure_peak_memory_cpu(lowerCamelCase__    )
																													__UpperCamelCase      :					Optional[Any]       		=Memory(lowerCamelCase__    ) if isinstance(lowerCamelCase__							,  lowerCamelCase__    ) else memory_bytes
																			if self.args.trace_memory_line_by_line:
																								__UpperCamelCase      :					List[str]       		=stop_memory_tracing(lowerCamelCase__    )
																								if memory is None:
																													__UpperCamelCase      :					int       		=summary.total
																			else:
																								__UpperCamelCase      :					Tuple       		=None

																			return memory, summary
														except ResourceExhaustedError as e:
																			self.print_fn(f'Doesn\'t fit on GPU. {e}'    )
																			return "N/A", None





import os
import re
import warnings
from shutil import copyfile
from typing import List, Optional, Tuple

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import is_sentencepiece_available, logging


if is_sentencepiece_available():
	from .tokenization_ta import TaTokenizer
else:
	lowerCAmelCase           =		None


lowerCAmelCase           =		logging.get_logger(__name__)

lowerCAmelCase           =		{'''vocab_file''': '''spiece.model''', '''tokenizer_file''': '''tokenizer.json'''}

lowerCAmelCase           =		{
    '''vocab_file''': {
        '''t5-small''': '''https://huggingface.co/t5-small/resolve/main/spiece.model''',
        '''t5-base''': '''https://huggingface.co/t5-base/resolve/main/spiece.model''',
        '''t5-large''': '''https://huggingface.co/t5-large/resolve/main/spiece.model''',
        '''t5-3b''': '''https://huggingface.co/t5-3b/resolve/main/spiece.model''',
        '''t5-11b''': '''https://huggingface.co/t5-11b/resolve/main/spiece.model''',
    },
    '''tokenizer_file''': {
        '''t5-small''': '''https://huggingface.co/t5-small/resolve/main/tokenizer.json''',
        '''t5-base''': '''https://huggingface.co/t5-base/resolve/main/tokenizer.json''',
        '''t5-large''': '''https://huggingface.co/t5-large/resolve/main/tokenizer.json''',
        '''t5-3b''': '''https://huggingface.co/t5-3b/resolve/main/tokenizer.json''',
        '''t5-11b''': '''https://huggingface.co/t5-11b/resolve/main/tokenizer.json''',
    },
}


# TODO(PVP) - this should be removed in Transformers v5
lowerCAmelCase           =		{
    '''t5-small''': 5_1_2,
    '''t5-base''': 5_1_2,
    '''t5-large''': 5_1_2,
    '''t5-3b''': 5_1_2,
    '''t5-11b''': 5_1_2,
}





class       A    (   A_ ):
							UpperCamelCase_   :							Dict       =VOCAB_FILES_NAMES
							UpperCamelCase_   :							Dict       =PRETRAINED_VOCAB_FILES_MAP
							UpperCamelCase_   :							List[Any]       =PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
							UpperCamelCase_   :							str       =['''input_ids''', '''attention_mask''']
							UpperCamelCase_   :							List[str]       =TaTokenizer

							UpperCamelCase_   :							List[int]       =[]


							def __init__(self      ,				lowerCAmelCase=None      ,				lowerCAmelCase=None      ,				lowerCAmelCase="</s>"      ,				lowerCAmelCase="<unk>"      ,				lowerCAmelCase="<pad>"      ,				lowerCAmelCase=1_0_0      ,				lowerCAmelCase=None      ,				**lowerCAmelCase      ,				):
									# Add extra_ids to the special token list
									if extra_ids > 0 and additional_special_tokens is None:
											__lowercase=			[f'<extra_id_{i}>' for i in range(lowerCAmelCase					)]
									elif extra_ids > 0 and additional_special_tokens is not None:
											# Check that we have the right number of extra special tokens
											__lowercase=			len(set(filter(lambda lowerCAmelCase					: bool('extra_id_' in str(lowerCAmelCase					)					)      ,				lowerCAmelCase					)					)					)
											if extra_tokens != extra_ids:
													raise ValueError(
													    f'Both extra_ids ({extra_ids}) and additional_special_tokens ({additional_special_tokens}) are'
													    ' provided to T5Tokenizer. In this case the additional_special_tokens must include the extra_ids'
													    ' tokens'					)

									super().__init__(
									    lowerCAmelCase      ,				tokenizer_file=lowerCAmelCase      ,				eos_token=lowerCAmelCase      ,				unk_token=lowerCAmelCase      ,				pad_token=lowerCAmelCase      ,				extra_ids=lowerCAmelCase      ,				additional_special_tokens=lowerCAmelCase      ,				**lowerCAmelCase      ,				)

									__lowercase=			vocab_file
									__lowercase=			False if not self.vocab_file else True
									__lowercase=			extra_ids


							@staticmethod
							def 			_A       (lowerCAmelCase      ,				lowerCAmelCase      ,				lowerCAmelCase					):
									if pretrained_model_name_or_path in TaTokenizerFast.max_model_input_sizes:
											__lowercase=			TaTokenizerFast.max_model_input_sizes[pretrained_model_name_or_path]
											if init_max_model_length is not None and init_max_model_length != max_model_length:
													return init_max_model_length
											elif init_max_model_length is None:
													warnings.warn(
													    'This tokenizer was incorrectly instantiated with a model max length of'
													    f' {deprecated_max_model_length} which will be corrected in Transformers v5.\nFor now, this'
													    ' behavior is kept to avoid breaking backwards compatibility when padding/encoding with'
													    ' `truncation is True`.\n- Be aware that you SHOULD NOT rely on'
													    f' {pretrained_model_name_or_path} automatically truncating your input to'
													    f' {deprecated_max_model_length} when padding/encoding.\n- If you want to encode/pad to sequences'
													    f' longer than {deprecated_max_model_length} you can either instantiate this tokenizer with'
													    ' `model_max_length` or pass `max_length` when encoding/padding.\n- To avoid this warning, please'
													    ' instantiate this tokenizer with `model_max_length` set to your preferred value.'      ,				lowerCAmelCase      ,				)

									return max_model_length


							def 			_A       (self      ,				lowerCAmelCase      ,				lowerCAmelCase = None					):
									if not self.can_save_slow_tokenizer:
											raise ValueError(
											    'Your fast tokenizer does not have the necessary information to save the vocabulary for a slow '
											    'tokenizer.'					)

									if not os.path.isdir(lowerCAmelCase					):
											logger.error(f'Vocabulary path ({save_directory}) should be a directory'					)
											return
									__lowercase=			os.path.join(
									    lowerCAmelCase      ,				(filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['vocab_file']					)

									if os.path.abspath(self.vocab_file					) != os.path.abspath(lowerCAmelCase					):
											copyfile(self.vocab_file      ,				lowerCAmelCase					)
											logger.info(f'Copy vocab file to {out_vocab_file}'					)

									return (out_vocab_file,)


							def 			_A       (self      ,				lowerCAmelCase      ,				lowerCAmelCase = None					):
									__lowercase=			token_ids_a + [self.eos_token_id]
									if token_ids_a is None:
											return self.prefix_tokens + token_ids_a
									else:
											__lowercase=			token_ids_a + [self.eos_token_id]
											return self.prefix_tokens + token_ids_a + token_ids_a


							def 			_A       (self      ,				lowerCAmelCase      ,				lowerCAmelCase = None					):
									__lowercase=			[self.eos_token_id]

									if token_ids_a is None:
											return len(token_ids_a + eos					) * [0]
									return len(token_ids_a + eos + token_ids_a + eos					) * [0]


							def 			_A       (self					):
									return list(
									    set(filter(lambda lowerCAmelCase					: bool(re.search(r'<extra_id_\d+>'      ,				lowerCAmelCase					)					) is not None      ,				self.additional_special_tokens					)					)					)







							def 			_A       (self					):
									return [self.convert_tokens_to_ids(lowerCAmelCase					) for token in self.get_sentinel_tokens()]






