






'''simple docstring'''
import warnings

from ...configuration_utils import PretrainedConfig
from ...utils import logging


a_ 		=       logging.get_logger(__name__)

a_ 		=       {
    'xlnet-base-cased': 'https://huggingface.co/xlnet-base-cased/resolve/main/config.json',
    'xlnet-large-cased': 'https://huggingface.co/xlnet-large-cased/resolve/main/config.json',
}






class    __SCREAMING_SNAKE_CASE					(	__SCREAMING_SNAKE_CASE ):
 snake_case_     							=							"""xlnet"""
 snake_case_     							=							["""mems"""]
 snake_case_     							=							{
     """n_token""": """vocab_size""",  # Backward compatibility
     """hidden_size""": """d_model""",
     """num_attention_heads""": """n_head""",
     """num_hidden_layers""": """n_layer""",
 }




 def __init__(					self		:							Optional[Any]    ,					__lowercase		:							Dict=3_20_00    ,					__lowercase		:							Any=10_24    ,					__lowercase		:							str=24    ,					__lowercase		:							Tuple=16    ,					__lowercase		:							List[str]=40_96    ,					__lowercase		:							List[str]="gelu"    ,					__lowercase		:							Union[str, Any]=True    ,					__lowercase		:							int="bi"    ,					__lowercase		:							Union[str, Any]=0.02    ,					__lowercase		:							Optional[Any]=1e-12    ,					__lowercase		:							Union[str, Any]=0.1    ,					__lowercase		:							Optional[Any]=5_12    ,					__lowercase		:							Optional[Any]=None    ,					__lowercase		:							Union[str, Any]=True    ,					__lowercase		:							Optional[int]=False    ,					__lowercase		:							int=False    ,					__lowercase		:							str=-1    ,					__lowercase		:							Union[str, Any]=False    ,					__lowercase		:							Dict="last"    ,					__lowercase		:							str=True    ,					__lowercase		:							Dict="tanh"    ,					__lowercase		:							Tuple=0.1    ,					__lowercase		:							Optional[int]=5    ,					__lowercase		:							Union[str, Any]=5    ,					__lowercase		:							Tuple=5    ,					__lowercase		:							str=1    ,					__lowercase		:							Tuple=2    ,					**__lowercase		:							Union[str, Any]    ,					)		->  Union[str, Any]:
       SCREAMING_SNAKE_CASE__   : Dict													=vocab_size
       SCREAMING_SNAKE_CASE__   : Any													=d_model
       SCREAMING_SNAKE_CASE__   : Tuple													=n_layer
       SCREAMING_SNAKE_CASE__   : Tuple													=n_head
       if d_model % n_head != 0:
             raise ValueError(F"'d_model % n_head' ({d_model % n_head}) should be equal to 0"					)
       if "d_head" in kwargs:
             if kwargs["d_head"] != d_model // n_head:
                   raise ValueError(
                       F"`d_head` ({kwargs['d_head']}) should be equal to `d_model // n_head` ({d_model // n_head})"					)
       SCREAMING_SNAKE_CASE__   : int													=d_model // n_head
       SCREAMING_SNAKE_CASE__   : Any													=ff_activation
       SCREAMING_SNAKE_CASE__   : str													=d_inner
       SCREAMING_SNAKE_CASE__   : List[str]													=untie_r
       SCREAMING_SNAKE_CASE__   : Tuple													=attn_type

       SCREAMING_SNAKE_CASE__   : Optional[Any]													=initializer_range
       SCREAMING_SNAKE_CASE__   : Any													=layer_norm_eps

       SCREAMING_SNAKE_CASE__   : str													=dropout
       SCREAMING_SNAKE_CASE__   : List[str]													=mem_len
       SCREAMING_SNAKE_CASE__   : Dict													=reuse_len
       SCREAMING_SNAKE_CASE__   : int													=bi_data
       SCREAMING_SNAKE_CASE__   : Optional[int]													=clamp_len
       SCREAMING_SNAKE_CASE__   : List[Any]													=same_length

       SCREAMING_SNAKE_CASE__   : Dict													=summary_type
       SCREAMING_SNAKE_CASE__   : int													=summary_use_proj
       SCREAMING_SNAKE_CASE__   : Dict													=summary_activation
       SCREAMING_SNAKE_CASE__   : str													=summary_last_dropout
       SCREAMING_SNAKE_CASE__   : Union[str, Any]													=start_n_top
       SCREAMING_SNAKE_CASE__   : List[Any]													=end_n_top

       SCREAMING_SNAKE_CASE__   : Union[str, Any]													=bos_token_id
       SCREAMING_SNAKE_CASE__   : Tuple													=pad_token_id
       SCREAMING_SNAKE_CASE__   : Union[str, Any]													=eos_token_id

       if "use_cache" in kwargs:
             warnings.warn(
                 '''The `use_cache` argument is deprecated and will be removed in a future version, use `use_mems_eval`'''
                 ''' instead.'''    ,					_SCREAMING_SNAKE_CASE    ,					)
             SCREAMING_SNAKE_CASE__   : Tuple													=kwargs["use_cache"]

       SCREAMING_SNAKE_CASE__   : Tuple													=use_mems_eval
       SCREAMING_SNAKE_CASE__   : Dict													=use_mems_train
       super().__init__(pad_token_id=_SCREAMING_SNAKE_CASE    ,					bos_token_id=_SCREAMING_SNAKE_CASE    ,					eos_token_id=_SCREAMING_SNAKE_CASE    ,					**_SCREAMING_SNAKE_CASE					)




 @property
 def    __magic_name__		(					self		:							Optional[Any]					)		->  Any:
       logger.info(F"The model {self.model_type} is one of the few models that has no sequence length limit."					)
       return -1




 @max_position_embeddings.setter
 def    __magic_name__		(					self		:							Union[str, Any]    ,					__lowercase		:							List[Any]					)		->  Any:
       raise NotImplementedError(
           F"The model {self.model_type} is one of the few models that has no sequence length limit."					)




"""simple docstring"""






import gc
import threading
import time

import psutil
import torch



class 			A__   :




	'''simple docstring'''





	def __init__(     self:		str)				->       Optional[Any]:

				"""simple docstring"""

				__lowerCAmelCase      :  Optional[Any]												=       psutil.Process()
				__lowerCAmelCase      :  str												=       False

	def 							_SCREAMING_SNAKE_CASE   (     self:		int)				->       Optional[int]:

				"""simple docstring"""

				__lowerCAmelCase      :  Optional[Any]												=       -1

				while True:
							__lowerCAmelCase      :  str												=       max(self.process.memory_info().rss       ,			self.cpu_memory_peak)

							# can't sleep or will not catch the peak right (this comment is here on purpose)
							if not self.peak_monitoring:
										break

	def 							_SCREAMING_SNAKE_CASE   (     self:		Tuple)				->       int:

				"""simple docstring"""

				__lowerCAmelCase      :  List[str]												=       True
				__lowerCAmelCase      :  str												=       threading.Thread(target=self.peak_monitor)
				__lowerCAmelCase      :  Tuple												=       True
				self.thread.start()







	def 							_SCREAMING_SNAKE_CASE   (     self:		Optional[Any])				->       List[str]:

				"""simple docstring"""

				__lowerCAmelCase      :  Tuple												=       False
				self.thread.join()
				return self.cpu_memory_peak






__snake_case							:   Tuple      =    PeakCPUMemory()


def 			_lowercase				(  )						->   str:
			# Time
			__lowerCAmelCase      :  str												=       {"time": time.time()}

			gc.collect()
			torch.cuda.empty_cache()

			# CPU mem
			__lowerCAmelCase      :  Optional[Any]												=       psutil.Process().memory_info().rss
			cpu_peak_tracker.start()

			# GPU mem
			for i in range(torch.cuda.device_count()	):
						__lowerCAmelCase      :  Union[str, Any]												=       torch.cuda.memory_allocated(__snake_case	)
			torch.cuda.reset_peak_memory_stats()

			return measures


def 			_lowercase				(  __snake_case	)						->   Optional[Any]:
			# Time
			__lowerCAmelCase      :  str												=       {"time": time.time() - start_measures["time"]}

			gc.collect()
			torch.cuda.empty_cache()

			# CPU mem
			__lowerCAmelCase      :  str												=       (psutil.Process().memory_info().rss - start_measures["cpu"]) / 2**20
			__lowerCAmelCase      :  List[str]												=       (cpu_peak_tracker.stop() - start_measures["cpu"]) / 2**20

			# GPU mem
			for i in range(torch.cuda.device_count()	):
						__lowerCAmelCase      :  Union[str, Any]												=       (torch.cuda.memory_allocated(__snake_case	) - start_measures[str(__snake_case	)]) / 2**20
						__lowerCAmelCase      :  Any												=       (torch.cuda.max_memory_allocated(__snake_case	) - start_measures[str(__snake_case	)]) / 2**20

			return measures





def 			_lowercase				(  __snake_case					,__snake_case	)						->   Dict:
			print(F"""{description}:"""	)
			print(F"""- Time: {measures['time']:.2f}s"""	)
			for i in range(torch.cuda.device_count()	):
						print(F"""- GPU {i} allocated: {measures[str(__snake_case	)]:.2f}MiB"""	)
						__lowerCAmelCase      :  Optional[Any]												=       measures[F"""{i}-peak"""]
						print(F"""- GPU {i} peak: {peak:.2f}MiB"""	)
			print(F"""- CPU RAM allocated: {measures['cpu']:.2f}MiB"""	)
			print(F"""- CPU RAM peak: {measures['cpu-peak']:.2f}MiB"""	)