




"""simple docstring"""




import operator as op


lowerCAmelCase							:			Dict     							=							"""scaler.pt"""
lowerCAmelCase							:			Tuple     							=							"""pytorch_model"""
lowerCAmelCase							:			Union[str, Any]     							=							"""random_states"""
lowerCAmelCase							:			Union[str, Any]     							=							"""optimizer"""
lowerCAmelCase							:			Dict     							=							"""scheduler"""
lowerCAmelCase							:			int     							=							"""pytorch_model.bin"""
lowerCAmelCase							:			str     							=							"""pytorch_model.bin.index.json"""
lowerCAmelCase							:			Union[str, Any]     							=							"""model.safetensors"""
lowerCAmelCase							:			List[Any]     							=							"""model.safetensors.index.json"""
lowerCAmelCase							:			List[Any]     							=							"""1.10.2"""
lowerCAmelCase							:			Any     							=							"""py38"""
lowerCAmelCase							:			Optional[int]     							=							"""4.17.0"""
lowerCAmelCase							:			str     							=							["""ml.p3.16xlarge""", """ml.p3dn.24xlarge""", """ml.p4dn.24xlarge"""]
lowerCAmelCase							:			Tuple     							=							["""FULL_SHARD""", """SHARD_GRAD_OP""", """NO_SHARD""", """HYBRID_SHARD""", """HYBRID_SHARD_ZERO2"""]
lowerCAmelCase							:			List[Any]     							=							["""TRANSFORMER_BASED_WRAP""", """SIZE_BASED_WRAP""", """NO_WRAP"""]
lowerCAmelCase							:			List[str]     							=							["""BACKWARD_PRE""", """BACKWARD_POST""", """NO_PREFETCH"""]
lowerCAmelCase							:			List[str]     							=							["""FULL_STATE_DICT""", """LOCAL_STATE_DICT""", """SHARDED_STATE_DICT"""]
lowerCAmelCase							:			Any     							=							"""2.0.1"""
lowerCAmelCase							:			List[Any]     							=							["""pdsh""", """standard""", """openmpi""", """mvapich"""]
lowerCAmelCase							:			Union[str, Any]     							=							["""default""", """reduce-overhead""", """max-autotune"""]

lowerCAmelCase							:			Optional[int]     							=							{""">""": op.gt, """>=""": op.ge, """==""": op.eq, """!=""": op.ne, """<=""": op.le, """<""": op.lt}

# These are the args for `torch.distributed.launch` for pytorch < 1.9
lowerCAmelCase							:			Union[str, Any]     							=							[
    """nnodes""",
    """nproc_per_node""",
    """rdzv_backend""",
    """rdzv_endpoint""",
    """rdzv_id""",
    """rdzv_conf""",
    """standalone""",
    """max_restarts""",
    """monitor_interval""",
    """start_method""",
    """role""",
    """module""",
    """m""",
    """no_python""",
    """run_path""",
    """log_dir""",
    """r""",
    """redirects""",
    """t""",
    """tee""",
    """node_rank""",
    """master_addr""",
    """master_port""",
]

lowerCAmelCase							:			List[str]     							=							["""DEEPSPEED""", """MULTI_GPU""", """FSDP""", """MEGATRON_LM"""]
lowerCAmelCase							:			Optional[Any]     							=							["""DEEPSPEED""", """MULTI_XPU""", """FSDP"""]









"""simple docstring"""




import os
from shutil import copyfile
from typing import List, Optional, Tuple

from ...tokenization_utils import AddedToken
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import is_sentencepiece_available, logging


if is_sentencepiece_available():
					from .tokenization_xlnet import XLNetTokenizer
else:
					lowerCAmelCase							:			int     							=							None


lowerCAmelCase							:			Tuple     							=							logging.get_logger(__name__)

lowerCAmelCase							:			Tuple     							=							{"""vocab_file""": """spiece.model""", """tokenizer_file""": """tokenizer.json"""}

lowerCAmelCase							:			Union[str, Any]     							=							{
    """vocab_file""": {
        """xlnet-base-cased""": """https://huggingface.co/xlnet-base-cased/resolve/main/spiece.model""",
        """xlnet-large-cased""": """https://huggingface.co/xlnet-large-cased/resolve/main/spiece.model""",
    },
    """tokenizer_file""": {
        """xlnet-base-cased""": """https://huggingface.co/xlnet-base-cased/resolve/main/tokenizer.json""",
        """xlnet-large-cased""": """https://huggingface.co/xlnet-large-cased/resolve/main/tokenizer.json""",
    },
}

lowerCAmelCase							:			Optional[int]     							=							{
    """xlnet-base-cased""": None,
    """xlnet-large-cased""": None,
}

lowerCAmelCase							:			Union[str, Any]     							=							"""▁"""

# Segments (not really needed)
lowerCAmelCase							:			str     							=							0
lowerCAmelCase							:			Optional[int]     							=							1
lowerCAmelCase							:			Tuple     							=							2
lowerCAmelCase							:			Optional[Any]     							=							3
lowerCAmelCase							:			List[Any]     							=							4






class     __magic_name__							(       UpperCAmelCase__				):






				'''simple docstring'''
				__UpperCamelCase			  =						VOCAB_FILES_NAMES
				__UpperCamelCase			  =						PRETRAINED_VOCAB_FILES_MAP
				__UpperCamelCase			  =						PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
				__UpperCamelCase			  =						"left"
				__UpperCamelCase			  =						XLNetTokenizer





				def __init__(       self		, _a=None		, _a=None		, _a=False		, _a=True		, _a=False		, _a="<s>"		, _a="</s>"		, _a="<unk>"		, _a="<sep>"		, _a="<pad>"		, _a="<cls>"		, _a="<mask>"		, _a=["<eop>", "<eod>"]		, **_a		, ):





							"""simple docstring"""
							# Mask token behave like a normal word, i.e. include the space before it
							lowerCamelCase			      =					AddedToken(_a		, lstrip=_a		, rstrip=_a ) if isinstance(_a		, _a ) else mask_token

							super().__init__(
							    vocab_file=_a		, tokenizer_file=_a		, do_lower_case=_a		, remove_space=_a		, keep_accents=_a		, bos_token=_a		, eos_token=_a		, unk_token=_a		, sep_token=_a		, pad_token=_a		, cls_token=_a		, mask_token=_a		, additional_special_tokens=_a		, **_a		, )

							lowerCamelCase			      =					3
							lowerCamelCase			      =					do_lower_case
							lowerCamelCase			      =					remove_space
							lowerCamelCase			      =					keep_accents
							lowerCamelCase			      =					vocab_file
							lowerCamelCase			      =					False if not self.vocab_file else True





				def    _lowerCAmelCase				(       self		, _a		, _a = None ):





							"""simple docstring"""
							lowerCamelCase			      =					[self.sep_token_id]
							lowerCamelCase			      =					[self.cls_token_id]
							if token_ids_a is None:
										return token_ids_a + sep + cls
							return token_ids_a + sep + token_ids_a + sep + cls





				def    _lowerCAmelCase				(       self		, _a		, _a = None ):





							"""simple docstring"""
							lowerCamelCase			      =					[self.sep_token_id]
							lowerCamelCase			      =					[2]

							if token_ids_a is None:
										return len(token_ids_a + sep ) * [0] + cls_segment_id
							return len(token_ids_a + sep ) * [0] + len(token_ids_a + sep ) * [1] + cls_segment_id





				def    _lowerCAmelCase				(       self		, _a		, _a = None ):





							"""simple docstring"""
							if not self.can_save_slow_tokenizer:
										raise ValueError(
										    """Your fast tokenizer does not have the necessary information to save the vocabulary for a slow """
										    """tokenizer.""" )

							if not os.path.isdir(_a ):
										logger.error(f'Vocabulary path ({save_directory}) should be a directory' )
										return
							lowerCamelCase			      =					os.path.join(
							    _a		, (filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""vocab_file"""] )

							if os.path.abspath(self.vocab_file ) != os.path.abspath(_a ):
										copyfile(self.vocab_file		, _a )

							return (out_vocab_file,)



