

'''simple docstring'''
import warnings

from ...configuration_utils import PretrainedConfig
from ...utils import logging


lowerCamelCase :      List[Any]					 =     logging.get_logger(__name__)

lowerCamelCase :      Optional[int]					 =     {
    'xlnet-base-cased': 'https://huggingface.co/xlnet-base-cased/resolve/main/config.json',
    'xlnet-large-cased': 'https://huggingface.co/xlnet-large-cased/resolve/main/config.json',
}







class 							A__    (       _A  ):
  A__		 =				'xlnet'
  A__		 =				['mems']
  A__		 =				{
      'n_token': 'vocab_size',  # Backward compatibility
      'hidden_size': 'd_model',
      'num_attention_heads': 'n_head',
      'num_hidden_layers': 'n_layer',
  }





  def __init__(		self      :	int							,					_a      :	List[str]=3_2000							,					_a      :	Optional[int]=1024							,					_a      :	Union[str, Any]=24							,					_a      :	Optional[int]=16							,					_a      :	List[str]=4096							,					_a      :	Union[str, Any]="gelu"							,					_a      :	Tuple=True							,					_a      :	Tuple="bi"							,					_a      :	List[str]=0.02							,					_a      :	List[str]=1e-12							,					_a      :	Tuple=0.1							,					_a      :	Tuple=512							,					_a      :	str=None							,					_a      :	Dict=True							,					_a      :	Optional[int]=False							,					_a      :	List[str]=False							,					_a      :	Any=-1							,					_a      :	Dict=False							,					_a      :	int="last"							,					_a      :	Optional[Any]=True							,					_a      :	List[Any]="tanh"							,					_a      :	Tuple=0.1							,					_a      :	List[str]=5							,					_a      :	Tuple=5							,					_a      :	str=5							,					_a      :	Optional[int]=1							,					_a      :	List[str]=2							,					**_a      :	List[Any]							,					)	->    Any:





       '''simple docstring'''






       _SCREAMING_SNAKE_CASE							    =vocab_size
       _SCREAMING_SNAKE_CASE							    =d_model
       _SCREAMING_SNAKE_CASE							    =n_layer
       _SCREAMING_SNAKE_CASE							    =n_head
       if d_model % n_head != 0:
            raise ValueError(f"\'d_model % n_head\' ({d_model % n_head}) should be equal to 0" )
       if "d_head" in kwargs:
            if kwargs["d_head"] != d_model // n_head:
                 raise ValueError(
                     f"`d_head` ({kwargs['d_head']}) should be equal to `d_model // n_head` ({d_model // n_head})" )
       _SCREAMING_SNAKE_CASE							    =d_model // n_head
       _SCREAMING_SNAKE_CASE							    =ff_activation
       _SCREAMING_SNAKE_CASE							    =d_inner
       _SCREAMING_SNAKE_CASE							    =untie_r
       _SCREAMING_SNAKE_CASE							    =attn_type

       _SCREAMING_SNAKE_CASE							    =initializer_range
       _SCREAMING_SNAKE_CASE							    =layer_norm_eps

       _SCREAMING_SNAKE_CASE							    =dropout
       _SCREAMING_SNAKE_CASE							    =mem_len
       _SCREAMING_SNAKE_CASE							    =reuse_len
       _SCREAMING_SNAKE_CASE							    =bi_data
       _SCREAMING_SNAKE_CASE							    =clamp_len
       _SCREAMING_SNAKE_CASE							    =same_length

       _SCREAMING_SNAKE_CASE							    =summary_type
       _SCREAMING_SNAKE_CASE							    =summary_use_proj
       _SCREAMING_SNAKE_CASE							    =summary_activation
       _SCREAMING_SNAKE_CASE							    =summary_last_dropout
       _SCREAMING_SNAKE_CASE							    =start_n_top
       _SCREAMING_SNAKE_CASE							    =end_n_top

       _SCREAMING_SNAKE_CASE							    =bos_token_id
       _SCREAMING_SNAKE_CASE							    =pad_token_id
       _SCREAMING_SNAKE_CASE							    =eos_token_id

       if "use_cache" in kwargs:
            warnings.warn(
                'The `use_cache` argument is deprecated and will be removed in a future version, use `use_mems_eval`'
                ' instead.'							,					UpperCamelCase__							,					)
            _SCREAMING_SNAKE_CASE							    =kwargs['use_cache']

       _SCREAMING_SNAKE_CASE							    =use_mems_eval
       _SCREAMING_SNAKE_CASE							    =use_mems_train
       super().__init__(pad_token_id=UpperCamelCase__							,					bos_token_id=UpperCamelCase__							,					eos_token_id=UpperCamelCase__							,					**UpperCamelCase__ )





  @property
  def 						A  (		self      :	int )	->    Tuple:





       '''simple docstring'''






       logger.info(f"The model {self.model_type} is one of the few models that has no sequence length limit." )
       return -1





  @max_position_embeddings.setter
  def 						A  (		self      :	Any							,					_a      :	Optional[Any] )	->    Dict:





       '''simple docstring'''






       raise NotImplementedError(
           f"The model {self.model_type} is one of the few models that has no sequence length limit." )


import importlib
import sys
from argparse import REMAINDER, ArgumentParser
from pathlib import Path

import torch_xla.distributed.xla_multiprocessing as xmp

def a__    (       ):






  '''simple docstring'''





  __magic_name__				     =       ArgumentParser(
      description=(
          """PyTorch TPU distributed training launch helper utility that will spawn up multiple distributed processes"""
      )					)

  # Optional arguments for the launch helper
  parser.add_argument("""--num_cores""",     type=A_,     default=1,     help="""Number of TPU cores to use (1 or 8)."""					)

  # positional
  parser.add_argument(
      """training_script""",     type=A_,     help=(
          """The full path to the single TPU training """
          """program/script to be launched in parallel, """
          """followed by all the arguments for the """
          """training script"""
      ),     )

  # rest from the training program
  parser.add_argument("""training_script_args""",     nargs=A_					)

  return parser.parse_args()

def a__    (       ):






  '''simple docstring'''





  __magic_name__				     =       parse_args()

  # Import training_script as a module.
  __magic_name__				     =       Path(args.training_script					)
  sys.path.append(str(script_fpath.parent.resolve()					)					)
  __magic_name__				     =       script_fpath.stem
  __magic_name__				     =       importlib.import_module(A_					)

  # Patch sys.argv
  __magic_name__				     =       [args.training_script] + args.training_script_args + ["""--tpu_num_cores""", str(args.num_cores					)]

  xmp.spawn(mod._mp_fn,     args=(),     nprocs=args.num_cores					)


if __name__ == "__main__":
    main()

