




'''simple docstring'''




import argparse
import json
import os
import sys
import tempfile
import unittest
from argparse import Namespace
from dataclasses import dataclass, field
from enum import Enum
from pathlib import Path
from typing import List, Literal, Optional

import yaml

from transformers import HfArgumentParser, TrainingArguments
from transformers.hf_argparser import make_choice_type_function, string_to_bool


# Since Python 3.10, we can use the builtin `|` operator for Union types
# See PEP 604: https://peps.python.org/pep-0604
a_      :		str							      =       sys.version_info >= (3, 10)





def      a_							(      __snake_case							:			Union[str, Any]=None   ,       __snake_case							:			Union[str, Any]=None   )							->						int:



       """simple docstring"""
       return field(default_factory=lambda: default   ,       metadata=__snake_case   )


@dataclass
class   __UpperCamelCase       :
       lowercase	:    int
       lowercase	:    float
       lowercase	:    str
       lowercase	:    bool





@dataclass
class   __UpperCamelCase       :
       lowercase	:    int      						=42
       lowercase	:    str      						=field(default='toto'			,					metadata={'help': 'help message'}				)





@dataclass
class   __UpperCamelCase       :
       lowercase	:    bool      						=False
       lowercase	:    bool      						=True
       lowercase	:    Optional[bool]      						=None





class   __UpperCamelCase       ( lowerCamelCase__				):
       lowercase	:    Union[str, Any]      						='titi'
       lowercase	:    Union[str, Any]      						='toto'





class   __UpperCamelCase       ( lowerCamelCase__				):
       lowercase	:    Optional[int]      						='titi'
       lowercase	:    Dict      						='toto'
       lowercase	:    int      						=42





@dataclass
class   __UpperCamelCase       :
       lowercase	:    BasicEnum      						="toto"






       def      lowercase__     (							self  ):



              """simple docstring"""







              lowerCamelCase_							=BasicEnum(self.foo  )





@dataclass
class   __UpperCamelCase       :
       lowercase	:    MixedTypeEnum      						="toto"






       def      lowercase__     (							self  ):



              """simple docstring"""







              lowerCamelCase_							=MixedTypeEnum(self.foo  )





@dataclass
class   __UpperCamelCase       :
       lowercase	:    Optional[int]      						=None
       lowercase	:    Optional[float]      						=field(default=lowerCamelCase__			,					metadata={'help': 'help message'}				)
       lowercase	:    Optional[str]      						=None
       lowercase	:    Optional[List[str]]      						=list_field(default=[]				)
       lowercase	:    Optional[List[int]]      						=list_field(default=[]				)





@dataclass
class   __UpperCamelCase       :
       lowercase	:    List[int]      						=list_field(default=[]				)
       lowercase	:    List[int]      						=list_field(default=[1, 2, 3]				)
       lowercase	:    List[str]      						=list_field(default=['Hallo', 'Bonjour', 'Hello']				)
       lowercase	:    List[float]      						=list_field(default=[0.1, 0.2, 0.3]				)





@dataclass
class   __UpperCamelCase       :
       lowercase	:    List[int]      						=field()
       lowercase	:    str      						=field()
       lowercase	:    BasicEnum      						=field()






       def      lowercase__     (							self  ):



              """simple docstring"""







              lowerCamelCase_							=BasicEnum(self.required_enum  )





@dataclass
class   __UpperCamelCase       :
       lowercase	:    int
       lowercase	:    "BasicEnum"      						=field()
       lowercase	:    "Optional[bool]"      						=None
       lowercase	:    "str"      						=field(default='toto'			,					metadata={'help': 'help message'}				)
       lowercase	:    "List[str]"      						=list_field(default=['Hallo', 'Bonjour', 'Hello']				)


if is_python_no_less_than_3_10:


     @dataclass
     class   __UpperCamelCase       :
            lowercase	:    bool      						=False
            lowercase	:    bool      						=True
            lowercase	:    bool | None      						=None





     @dataclass
     class   __UpperCamelCase       :
            lowercase	:    int | None      						=None
            lowercase	:    float | None      						=field(default=lowerCamelCase__			,					metadata={'help': 'help message'}				)
            lowercase	:    str | None      						=None
            lowercase	:    list[str] | None      						=list_field(default=[]				)
            lowercase	:    list[int] | None      						=list_field(default=[]				)


class   __UpperCamelCase       ( unittest.TestCase				):






       def      lowercase__     (							self,		lowerCAmelCase,		lowerCAmelCase  ):



              """simple docstring"""







              self.assertEqual(len(a._actions  ),		len(b._actions  )  )
              for x, y in zip(a._actions,		b._actions  ):
                     lowerCamelCase_							={k: v for k, v in vars(lowerCAmelCase  ).items() if k != '''container'''}
                     lowerCamelCase_							={k: v for k, v in vars(lowerCAmelCase  ).items() if k != '''container'''}

                     # Choices with mixed type have custom function as "type"
                     # So we need to compare results directly for equality
                     if xx.get('''choices''',		lowerCAmelCase  ) and yy.get('''choices''',		lowerCAmelCase  ):
                            for expected_choice in yy["choices"] + xx["choices"]:
                                   self.assertEqual(xx['''type'''](lowerCAmelCase  ),		yy['''type'''](lowerCAmelCase  )  )
                            del xx["type"], yy["type"]

                     self.assertEqual(lowerCAmelCase,		lowerCAmelCase  )






       def      lowercase__     (							self  ):



              """simple docstring"""







              lowerCamelCase_							=HfArgumentParser(lowerCAmelCase  )

              lowerCamelCase_							=argparse.ArgumentParser()
              expected.add_argument('''--foo''',		type=lowerCAmelCase,		required=lowerCAmelCase  )
              expected.add_argument('''--bar''',		type=lowerCAmelCase,		required=lowerCAmelCase  )
              expected.add_argument('''--baz''',		type=lowerCAmelCase,		required=lowerCAmelCase  )
              expected.add_argument('''--flag''',		type=lowerCAmelCase,		default=lowerCAmelCase,		const=lowerCAmelCase,		nargs='''?'''  )
              self.argparsersEqual(lowerCAmelCase,		lowerCAmelCase  )

              lowerCamelCase_							=['''--foo''', '''1''', '''--baz''', '''quux''', '''--bar''', '''0.5''']
              ((lowerCamelCase_),      )							=parser.parse_args_into_dataclasses(lowerCAmelCase,		look_for_args_file=lowerCAmelCase  )
              self.assertFalse(example.flag  )






       def      lowercase__     (							self  ):



              """simple docstring"""







              lowerCamelCase_							=HfArgumentParser(lowerCAmelCase  )

              lowerCamelCase_							=argparse.ArgumentParser()
              expected.add_argument('''--foo''',		default=42,		type=lowerCAmelCase  )
              expected.add_argument('''--baz''',		default='''toto''',		type=lowerCAmelCase,		help='''help message'''  )
              self.argparsersEqual(lowerCAmelCase,		lowerCAmelCase  )






       def      lowercase__     (							self  ):



              """simple docstring"""







              lowerCamelCase_							=argparse.ArgumentParser()
              expected.add_argument('''--foo''',		type=lowerCAmelCase,		default=lowerCAmelCase,		const=lowerCAmelCase,		nargs='''?'''  )
              expected.add_argument('''--baz''',		type=lowerCAmelCase,		default=lowerCAmelCase,		const=lowerCAmelCase,		nargs='''?'''  )
              # A boolean no_* argument always has to come after its "default: True" regular counter-part
              # and its default must be set to False
              expected.add_argument('''--no_baz''',		action='''store_false''',		default=lowerCAmelCase,		dest='''baz'''  )
              expected.add_argument('''--opt''',		type=lowerCAmelCase,		default=lowerCAmelCase  )

              lowerCamelCase_							=[WithDefaultBoolExample]
              if is_python_no_less_than_3_10:
                     dataclass_types.append(lowerCAmelCase  )

              for dataclass_type in dataclass_types:
                     lowerCamelCase_							=HfArgumentParser(lowerCAmelCase  )
                     self.argparsersEqual(lowerCAmelCase,		lowerCAmelCase  )

                     lowerCamelCase_							=parser.parse_args([]  )
                     self.assertEqual(lowerCAmelCase,		Namespace(foo=lowerCAmelCase,		baz=lowerCAmelCase,		opt=lowerCAmelCase  )  )

                     lowerCamelCase_							=parser.parse_args(['''--foo''', '''--no_baz''']  )
                     self.assertEqual(lowerCAmelCase,		Namespace(foo=lowerCAmelCase,		baz=lowerCAmelCase,		opt=lowerCAmelCase  )  )

                     lowerCamelCase_							=parser.parse_args(['''--foo''', '''--baz''']  )
                     self.assertEqual(lowerCAmelCase,		Namespace(foo=lowerCAmelCase,		baz=lowerCAmelCase,		opt=lowerCAmelCase  )  )

                     lowerCamelCase_							=parser.parse_args(['''--foo''', '''True''', '''--baz''', '''True''', '''--opt''', '''True''']  )
                     self.assertEqual(lowerCAmelCase,		Namespace(foo=lowerCAmelCase,		baz=lowerCAmelCase,		opt=lowerCAmelCase  )  )

                     lowerCamelCase_							=parser.parse_args(['''--foo''', '''False''', '''--baz''', '''False''', '''--opt''', '''False''']  )
                     self.assertEqual(lowerCAmelCase,		Namespace(foo=lowerCAmelCase,		baz=lowerCAmelCase,		opt=lowerCAmelCase  )  )






       def      lowercase__     (							self  ):



              """simple docstring"""







              lowerCamelCase_							=HfArgumentParser(lowerCAmelCase  )

              lowerCamelCase_							=argparse.ArgumentParser()
              expected.add_argument(
                  '''--foo''',		default='''toto''',		choices=['''titi''', '''toto''', 42],		type=make_choice_type_function(['''titi''', '''toto''', 42]  ),		)
              self.argparsersEqual(lowerCAmelCase,		lowerCAmelCase  )

              lowerCamelCase_							=parser.parse_args([]  )
              self.assertEqual(args.foo,		'''toto'''  )
              lowerCamelCase_							=parser.parse_args_into_dataclasses([]  )[0]
              self.assertEqual(enum_ex.foo,		MixedTypeEnum.toto  )

              lowerCamelCase_							=parser.parse_args(['''--foo''', '''titi''']  )
              self.assertEqual(args.foo,		'''titi'''  )
              lowerCamelCase_							=parser.parse_args_into_dataclasses(['''--foo''', '''titi''']  )[0]
              self.assertEqual(enum_ex.foo,		MixedTypeEnum.titi  )

              lowerCamelCase_							=parser.parse_args(['''--foo''', '''42''']  )
              self.assertEqual(args.foo,		42  )
              lowerCamelCase_							=parser.parse_args_into_dataclasses(['''--foo''', '''42''']  )[0]
              self.assertEqual(enum_ex.foo,		MixedTypeEnum.fourtytwo  )






       def      lowercase__     (							self  ):



              """simple docstring"""







              @dataclass
              class   __UpperCamelCase       :
                     lowercase	:    Literal["titi", "toto", 42]      						="toto"





              lowerCamelCase_							=HfArgumentParser(lowerCAmelCase  )

              lowerCamelCase_							=argparse.ArgumentParser()
              expected.add_argument(
                  '''--foo''',		default='''toto''',		choices=('''titi''', '''toto''', 42),		type=make_choice_type_function(['''titi''', '''toto''', 42]  ),		)
              self.argparsersEqual(lowerCAmelCase,		lowerCAmelCase  )

              lowerCamelCase_							=parser.parse_args([]  )
              self.assertEqual(args.foo,		'''toto'''  )

              lowerCamelCase_							=parser.parse_args(['''--foo''', '''titi''']  )
              self.assertEqual(args.foo,		'''titi'''  )

              lowerCamelCase_							=parser.parse_args(['''--foo''', '''42''']  )
              self.assertEqual(args.foo,		42  )






       def      lowercase__     (							self  ):



              """simple docstring"""







              lowerCamelCase_							=HfArgumentParser(lowerCAmelCase  )

              lowerCamelCase_							=argparse.ArgumentParser()
              expected.add_argument('''--foo_int''',		nargs='''+''',		default=[],		type=lowerCAmelCase  )
              expected.add_argument('''--bar_int''',		nargs='''+''',		default=[1, 2, 3],		type=lowerCAmelCase  )
              expected.add_argument('''--foo_str''',		nargs='''+''',		default=['''Hallo''', '''Bonjour''', '''Hello'''],		type=lowerCAmelCase  )
              expected.add_argument('''--foo_float''',		nargs='''+''',		default=[0.1, 0.2, 0.3],		type=lowerCAmelCase  )

              self.argparsersEqual(lowerCAmelCase,		lowerCAmelCase  )

              lowerCamelCase_							=parser.parse_args([]  )
              self.assertEqual(
                  lowerCAmelCase,		Namespace(foo_int=[],		bar_int=[1, 2, 3],		foo_str=['''Hallo''', '''Bonjour''', '''Hello'''],		foo_float=[0.1, 0.2, 0.3]  ),		)

              lowerCamelCase_							=parser.parse_args('''--foo_int 1 --bar_int 2 3 --foo_str a b c --foo_float 0.1 0.7'''.split()  )
              self.assertEqual(lowerCAmelCase,		Namespace(foo_int=[1],		bar_int=[2, 3],		foo_str=['''a''', '''b''', '''c'''],		foo_float=[0.1, 0.7]  )  )






       def      lowercase__     (							self  ):



              """simple docstring"""







              lowerCamelCase_							=argparse.ArgumentParser()
              expected.add_argument('''--foo''',		default=lowerCAmelCase,		type=lowerCAmelCase  )
              expected.add_argument('''--bar''',		default=lowerCAmelCase,		type=lowerCAmelCase,		help='''help message'''  )
              expected.add_argument('''--baz''',		default=lowerCAmelCase,		type=lowerCAmelCase  )
              expected.add_argument('''--ces''',		nargs='''+''',		default=[],		type=lowerCAmelCase  )
              expected.add_argument('''--des''',		nargs='''+''',		default=[],		type=lowerCAmelCase  )

              lowerCamelCase_							=[OptionalExample]
              if is_python_no_less_than_3_10:
                     dataclass_types.append(lowerCAmelCase  )

              for dataclass_type in dataclass_types:
                     lowerCamelCase_							=HfArgumentParser(lowerCAmelCase  )

                     self.argparsersEqual(lowerCAmelCase,		lowerCAmelCase  )

                     lowerCamelCase_							=parser.parse_args([]  )
                     self.assertEqual(lowerCAmelCase,		Namespace(foo=lowerCAmelCase,		bar=lowerCAmelCase,		baz=lowerCAmelCase,		ces=[],		des=[]  )  )

                     lowerCamelCase_							=parser.parse_args('''--foo 12 --bar 3.14 --baz 42 --ces a b c --des 1 2 3'''.split()  )
                     self.assertEqual(lowerCAmelCase,		Namespace(foo=12,		bar=3.1_4,		baz='''42''',		ces=['''a''', '''b''', '''c'''],		des=[1, 2, 3]  )  )






       def      lowercase__     (							self  ):



              """simple docstring"""







              lowerCamelCase_							=HfArgumentParser(lowerCAmelCase  )

              lowerCamelCase_							=argparse.ArgumentParser()
              expected.add_argument('''--required_list''',		nargs='''+''',		type=lowerCAmelCase,		required=lowerCAmelCase  )
              expected.add_argument('''--required_str''',		type=lowerCAmelCase,		required=lowerCAmelCase  )
              expected.add_argument(
                  '''--required_enum''',		type=make_choice_type_function(['''titi''', '''toto''']  ),		choices=['''titi''', '''toto'''],		required=lowerCAmelCase,		)
              self.argparsersEqual(lowerCAmelCase,		lowerCAmelCase  )






       def      lowercase__     (							self  ):



              """simple docstring"""







              lowerCamelCase_							=HfArgumentParser(lowerCAmelCase  )

              lowerCamelCase_							=argparse.ArgumentParser()
              expected.add_argument('''--foo''',		type=lowerCAmelCase,		required=lowerCAmelCase  )
              expected.add_argument(
                  '''--required_enum''',		type=make_choice_type_function(['''titi''', '''toto''']  ),		choices=['''titi''', '''toto'''],		required=lowerCAmelCase,		)
              expected.add_argument('''--opt''',		type=lowerCAmelCase,		default=lowerCAmelCase  )
              expected.add_argument('''--baz''',		default='''toto''',		type=lowerCAmelCase,		help='''help message'''  )
              expected.add_argument('''--foo_str''',		nargs='''+''',		default=['''Hallo''', '''Bonjour''', '''Hello'''],		type=lowerCAmelCase  )
              self.argparsersEqual(lowerCAmelCase,		lowerCAmelCase  )






       def      lowercase__     (							self  ):



              """simple docstring"""







              lowerCamelCase_							=HfArgumentParser(lowerCAmelCase  )

              lowerCamelCase_							={
                  '''foo''': 12,
                  '''bar''': 3.1_4,
                  '''baz''': '''42''',
                  '''flag''': True,
              }

              lowerCamelCase_							=parser.parse_dict(lowerCAmelCase  )[0]
              lowerCamelCase_							=BasicExample(**lowerCAmelCase  )
              self.assertEqual(lowerCAmelCase,		lowerCAmelCase  )






       def      lowercase__     (							self  ):



              """simple docstring"""







              lowerCamelCase_							=HfArgumentParser(lowerCAmelCase  )

              lowerCamelCase_							={
                  '''foo''': 12,
                  '''bar''': 3.1_4,
                  '''baz''': '''42''',
                  '''flag''': True,
                  '''extra''': 42,
              }

              self.assertRaises(lowerCAmelCase,		parser.parse_dict,		lowerCAmelCase,		allow_extra_keys=lowerCAmelCase  )






       def      lowercase__     (							self  ):



              """simple docstring"""







              lowerCamelCase_							=HfArgumentParser(lowerCAmelCase  )

              lowerCamelCase_							={
                  '''foo''': 12,
                  '''bar''': 3.1_4,
                  '''baz''': '''42''',
                  '''flag''': True,
              }
              with tempfile.TemporaryDirectory() as tmp_dir:
                     lowerCamelCase_							=os.path.join(lowerCAmelCase,		'''temp_json'''  )
                     os.mkdir(lowerCAmelCase  )
                     with open(temp_local_path + '''.json''',		'''w+'''  ) as f:
                            json.dump(lowerCAmelCase,		lowerCAmelCase  )
                     lowerCamelCase_							=parser.parse_yaml_file(Path(temp_local_path + '''.json'''  )  )[0]

              lowerCamelCase_							=BasicExample(**lowerCAmelCase  )
              self.assertEqual(lowerCAmelCase,		lowerCAmelCase  )






       def      lowercase__     (							self  ):



              """simple docstring"""







              lowerCamelCase_							=HfArgumentParser(lowerCAmelCase  )

              lowerCamelCase_							={
                  '''foo''': 12,
                  '''bar''': 3.1_4,
                  '''baz''': '''42''',
                  '''flag''': True,
              }
              with tempfile.TemporaryDirectory() as tmp_dir:
                     lowerCamelCase_							=os.path.join(lowerCAmelCase,		'''temp_yaml'''  )
                     os.mkdir(lowerCAmelCase  )
                     with open(temp_local_path + '''.yaml''',		'''w+'''  ) as f:
                            yaml.dump(lowerCAmelCase,		lowerCAmelCase  )
                     lowerCamelCase_							=parser.parse_yaml_file(Path(temp_local_path + '''.yaml'''  )  )[0]
              lowerCamelCase_							=BasicExample(**lowerCAmelCase  )
              self.assertEqual(lowerCAmelCase,		lowerCAmelCase  )






       def      lowercase__     (							self  ):



              """simple docstring"""







              lowerCamelCase_							=HfArgumentParser(lowerCAmelCase  )
              self.assertIsNotNone(lowerCAmelCase  )










"""simple docstring"""




import os
from shutil import copyfile
from typing import List, Optional, Tuple

from ...tokenization_utils import AddedToken
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import is_sentencepiece_available, logging


if is_sentencepiece_available():
					from .tokenization_xlnet import XLNetTokenizer
else:
					lowerCAmelCase							:			int     							=							None


lowerCAmelCase							:			Tuple     							=							logging.get_logger(__name__)

lowerCAmelCase							:			Tuple     							=							{"""vocab_file""": """spiece.model""", """tokenizer_file""": """tokenizer.json"""}

lowerCAmelCase							:			Union[str, Any]     							=							{
    """vocab_file""": {
        """xlnet-base-cased""": """https://huggingface.co/xlnet-base-cased/resolve/main/spiece.model""",
        """xlnet-large-cased""": """https://huggingface.co/xlnet-large-cased/resolve/main/spiece.model""",
    },
    """tokenizer_file""": {
        """xlnet-base-cased""": """https://huggingface.co/xlnet-base-cased/resolve/main/tokenizer.json""",
        """xlnet-large-cased""": """https://huggingface.co/xlnet-large-cased/resolve/main/tokenizer.json""",
    },
}

lowerCAmelCase							:			Optional[int]     							=							{
    """xlnet-base-cased""": None,
    """xlnet-large-cased""": None,
}

lowerCAmelCase							:			Union[str, Any]     							=							"""▁"""

# Segments (not really needed)
lowerCAmelCase							:			str     							=							0
lowerCAmelCase							:			Optional[int]     							=							1
lowerCAmelCase							:			Tuple     							=							2
lowerCAmelCase							:			Optional[Any]     							=							3
lowerCAmelCase							:			List[Any]     							=							4






class     __magic_name__							(       UpperCAmelCase__				):






				'''simple docstring'''
				__UpperCamelCase			  =						VOCAB_FILES_NAMES
				__UpperCamelCase			  =						PRETRAINED_VOCAB_FILES_MAP
				__UpperCamelCase			  =						PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
				__UpperCamelCase			  =						"left"
				__UpperCamelCase			  =						XLNetTokenizer





				def __init__(       self		, _a=None		, _a=None		, _a=False		, _a=True		, _a=False		, _a="<s>"		, _a="</s>"		, _a="<unk>"		, _a="<sep>"		, _a="<pad>"		, _a="<cls>"		, _a="<mask>"		, _a=["<eop>", "<eod>"]		, **_a		, ):





							"""simple docstring"""
							# Mask token behave like a normal word, i.e. include the space before it
							lowerCamelCase			      =					AddedToken(_a		, lstrip=_a		, rstrip=_a ) if isinstance(_a		, _a ) else mask_token

							super().__init__(
							    vocab_file=_a		, tokenizer_file=_a		, do_lower_case=_a		, remove_space=_a		, keep_accents=_a		, bos_token=_a		, eos_token=_a		, unk_token=_a		, sep_token=_a		, pad_token=_a		, cls_token=_a		, mask_token=_a		, additional_special_tokens=_a		, **_a		, )

							lowerCamelCase			      =					3
							lowerCamelCase			      =					do_lower_case
							lowerCamelCase			      =					remove_space
							lowerCamelCase			      =					keep_accents
							lowerCamelCase			      =					vocab_file
							lowerCamelCase			      =					False if not self.vocab_file else True





				def    _lowerCAmelCase				(       self		, _a		, _a = None ):





							"""simple docstring"""
							lowerCamelCase			      =					[self.sep_token_id]
							lowerCamelCase			      =					[self.cls_token_id]
							if token_ids_a is None:
										return token_ids_a + sep + cls
							return token_ids_a + sep + token_ids_a + sep + cls





				def    _lowerCAmelCase				(       self		, _a		, _a = None ):





							"""simple docstring"""
							lowerCamelCase			      =					[self.sep_token_id]
							lowerCamelCase			      =					[2]

							if token_ids_a is None:
										return len(token_ids_a + sep ) * [0] + cls_segment_id
							return len(token_ids_a + sep ) * [0] + len(token_ids_a + sep ) * [1] + cls_segment_id





				def    _lowerCAmelCase				(       self		, _a		, _a = None ):





							"""simple docstring"""
							if not self.can_save_slow_tokenizer:
										raise ValueError(
										    """Your fast tokenizer does not have the necessary information to save the vocabulary for a slow """
										    """tokenizer.""" )

							if not os.path.isdir(_a ):
										logger.error(f'Vocabulary path ({save_directory}) should be a directory' )
										return
							lowerCamelCase			      =					os.path.join(
							    _a		, (filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""vocab_file"""] )

							if os.path.abspath(self.vocab_file ) != os.path.abspath(_a ):
										copyfile(self.vocab_file		, _a )

							return (out_vocab_file,)



