



"""simple docstring"""






import warnings

from ...configuration_utils import PretrainedConfig
from ...utils import logging


__snake_case							:   Optional[Any]      =    logging.get_logger(__name__)

__snake_case							:   Optional[Any]      =    {
    'xlnet-base-cased': 'https://huggingface.co/xlnet-base-cased/resolve/main/config.json',
    'xlnet-large-cased': 'https://huggingface.co/xlnet-large-cased/resolve/main/config.json',
}



class 			A__   (  __SCREAMING_SNAKE_CASE							):




	'''simple docstring'''





	SCREAMING_SNAKE_CASE          =	'xlnet'
	SCREAMING_SNAKE_CASE          =	['mems']
	SCREAMING_SNAKE_CASE          =	{
	    'n_token': 'vocab_size',  # Backward compatibility
	    'hidden_size': 'd_model',
	    'num_attention_heads': 'n_head',
	    'num_hidden_layers': 'n_layer',
	}

	def __init__(     self:		Optional[Any]       ,			_SCREAMING_SNAKE_CASE:		Dict=3_2000       ,			_SCREAMING_SNAKE_CASE:		Any=1024       ,			_SCREAMING_SNAKE_CASE:		str=24       ,			_SCREAMING_SNAKE_CASE:		Tuple=16       ,			_SCREAMING_SNAKE_CASE:		List[str]=4096       ,			_SCREAMING_SNAKE_CASE:		List[str]="gelu"       ,			_SCREAMING_SNAKE_CASE:		Union[str, Any]=True       ,			_SCREAMING_SNAKE_CASE:		int="bi"       ,			_SCREAMING_SNAKE_CASE:		Union[str, Any]=0.02       ,			_SCREAMING_SNAKE_CASE:		Optional[Any]=1e-12       ,			_SCREAMING_SNAKE_CASE:		Union[str, Any]=0.1       ,			_SCREAMING_SNAKE_CASE:		Optional[Any]=512       ,			_SCREAMING_SNAKE_CASE:		Optional[Any]=None       ,			_SCREAMING_SNAKE_CASE:		Union[str, Any]=True       ,			_SCREAMING_SNAKE_CASE:		Optional[int]=False       ,			_SCREAMING_SNAKE_CASE:		int=False       ,			_SCREAMING_SNAKE_CASE:		str=-1       ,			_SCREAMING_SNAKE_CASE:		Union[str, Any]=False       ,			_SCREAMING_SNAKE_CASE:		Dict="last"       ,			_SCREAMING_SNAKE_CASE:		str=True       ,			_SCREAMING_SNAKE_CASE:		Dict="tanh"       ,			_SCREAMING_SNAKE_CASE:		Tuple=0.1       ,			_SCREAMING_SNAKE_CASE:		Optional[int]=5       ,			_SCREAMING_SNAKE_CASE:		Union[str, Any]=5       ,			_SCREAMING_SNAKE_CASE:		Tuple=5       ,			_SCREAMING_SNAKE_CASE:		str=1       ,			_SCREAMING_SNAKE_CASE:		Tuple=2       ,			**_SCREAMING_SNAKE_CASE:		Union[str, Any]       ,			)				->       Union[str, Any]:

				"""simple docstring"""

				__lowerCAmelCase      :  Dict												=       vocab_size
				__lowerCAmelCase      :  Any												=       d_model
				__lowerCAmelCase      :  Tuple												=       n_layer
				__lowerCAmelCase      :  Tuple												=       n_head
				if d_model % n_head != 0:
							raise ValueError(F"""'d_model % n_head' ({d_model % n_head}) should be equal to 0""")
				if "d_head" in kwargs:
							if kwargs["d_head"] != d_model // n_head:
										raise ValueError(
										    F"""`d_head` ({kwargs['d_head']}) should be equal to `d_model // n_head` ({d_model // n_head})""")
				__lowerCAmelCase      :  int												=       d_model // n_head
				__lowerCAmelCase      :  Any												=       ff_activation
				__lowerCAmelCase      :  str												=       d_inner
				__lowerCAmelCase      :  List[str]												=       untie_r
				__lowerCAmelCase      :  Tuple												=       attn_type

				__lowerCAmelCase      :  Optional[Any]												=       initializer_range
				__lowerCAmelCase      :  Any												=       layer_norm_eps

				__lowerCAmelCase      :  str												=       dropout
				__lowerCAmelCase      :  List[str]												=       mem_len
				__lowerCAmelCase      :  Dict												=       reuse_len
				__lowerCAmelCase      :  int												=       bi_data
				__lowerCAmelCase      :  Optional[int]												=       clamp_len
				__lowerCAmelCase      :  List[Any]												=       same_length

				__lowerCAmelCase      :  Dict												=       summary_type
				__lowerCAmelCase      :  int												=       summary_use_proj
				__lowerCAmelCase      :  Dict												=       summary_activation
				__lowerCAmelCase      :  str												=       summary_last_dropout
				__lowerCAmelCase      :  Union[str, Any]												=       start_n_top
				__lowerCAmelCase      :  List[Any]												=       end_n_top

				__lowerCAmelCase      :  Union[str, Any]												=       bos_token_id
				__lowerCAmelCase      :  Tuple												=       pad_token_id
				__lowerCAmelCase      :  Union[str, Any]												=       eos_token_id

				if "use_cache" in kwargs:
							warnings.warn(
							    "The `use_cache` argument is deprecated and will be removed in a future version, use `use_mems_eval`"
							    " instead."       ,			_SCREAMING_SNAKE_CASE       ,			)
							__lowerCAmelCase      :  Tuple												=       kwargs["use_cache"]

				__lowerCAmelCase      :  Tuple												=       use_mems_eval
				__lowerCAmelCase      :  Dict												=       use_mems_train
				super().__init__(pad_token_id=_SCREAMING_SNAKE_CASE       ,			bos_token_id=_SCREAMING_SNAKE_CASE       ,			eos_token_id=_SCREAMING_SNAKE_CASE       ,			**_SCREAMING_SNAKE_CASE)

	@property
	def 							_SCREAMING_SNAKE_CASE   (     self:		Optional[Any])				->       Any:

				"""simple docstring"""

				logger.info(F"""The model {self.model_type} is one of the few models that has no sequence length limit.""")
				return -1







	@max_position_embeddings.setter
	def 							_SCREAMING_SNAKE_CASE   (     self:		Union[str, Any]       ,			_SCREAMING_SNAKE_CASE:		List[Any])				->       Any:

				"""simple docstring"""

				raise NotImplementedError(
				    F"""The model {self.model_type} is one of the few models that has no sequence length limit.""")




"""simple docstring"""






import os
import unittest

from transformers import FunnelTokenizer, FunnelTokenizerFast
from transformers.models.funnel.tokenization_funnel import VOCAB_FILES_NAMES
from transformers.testing_utils import require_tokenizers

from ...test_tokenization_common import TokenizerTesterMixin



@require_tokenizers
class 			A__   (  __SCREAMING_SNAKE_CASE  ,		unittest.TestCase							):




	'''simple docstring'''





	SCREAMING_SNAKE_CASE          =	FunnelTokenizer
	SCREAMING_SNAKE_CASE          =	FunnelTokenizerFast
	SCREAMING_SNAKE_CASE          =	True
	SCREAMING_SNAKE_CASE          =	True

	def 							_SCREAMING_SNAKE_CASE   (     self:		Tuple)				->       Optional[int]:

				"""simple docstring"""

				super().setUp()

				__lowerCAmelCase      :  str												=       [
				    "<unk>",
				    "<cls>",
				    "<sep>",
				    "want",
				    "##want",
				    "##ed",
				    "wa",
				    "un",
				    "runn",
				    "##ing",
				    ",",
				    "low",
				    "lowest",
				]
				__lowerCAmelCase      :  int												=       os.path.join(self.tmpdirname       ,			VOCAB_FILES_NAMES["vocab_file"])
				with open(self.vocab_file       ,			"w"       ,			encoding="utf-8") as vocab_writer:
							vocab_writer.write("".join([x + "\n" for x in vocab_tokens]))

	def 							_SCREAMING_SNAKE_CASE   (     self:		Optional[Any]       ,			**_SCREAMING_SNAKE_CASE:		Union[str, Any])				->       int:

				"""simple docstring"""

				return FunnelTokenizer.from_pretrained(self.tmpdirname       ,			**_SCREAMING_SNAKE_CASE)

	def 							_SCREAMING_SNAKE_CASE   (     self:		Any       ,			**_SCREAMING_SNAKE_CASE:		Any)				->       str:

				"""simple docstring"""

				return FunnelTokenizerFast.from_pretrained(self.tmpdirname       ,			**_SCREAMING_SNAKE_CASE)

	def 							_SCREAMING_SNAKE_CASE   (     self:		Tuple       ,			_SCREAMING_SNAKE_CASE:		str)				->       Any:

				"""simple docstring"""

				__lowerCAmelCase      :  Union[str, Any]												=       "UNwant\u00E9d,running"
				__lowerCAmelCase      :  str												=       "unwanted, running"
				return input_text, output_text

	def 							_SCREAMING_SNAKE_CASE   (     self:		Optional[int])				->       List[str]:

				"""simple docstring"""

				__lowerCAmelCase      :  Any												=       self.tokenizer_class(self.vocab_file)

				__lowerCAmelCase      :  Any												=       tokenizer.tokenize("UNwant\u00E9d,running")
				self.assertListEqual(_SCREAMING_SNAKE_CASE       ,			["un", "##want", "##ed", ",", "runn", "##ing"])
				self.assertListEqual(tokenizer.convert_tokens_to_ids(_SCREAMING_SNAKE_CASE)       ,			[7, 4, 5, 10, 8, 9])







	def 							_SCREAMING_SNAKE_CASE   (     self:		List[str])				->       List[str]:

				"""simple docstring"""

				__lowerCAmelCase      :  Optional[Any]												=       self.get_tokenizers(do_lower_case=_SCREAMING_SNAKE_CASE)
				for tokenizer in tokenizers:
							__lowerCAmelCase      :  List[str]												=       tokenizer("UNwant\u00E9d,running")
							__lowerCAmelCase      :  Optional[int]												=       len(inputs["input_ids"]) - 1
							self.assertListEqual(inputs["token_type_ids"]       ,			[2] + [0] * sentence_len)

							__lowerCAmelCase      :  List[str]												=       tokenizer("UNwant\u00E9d,running"       ,			"UNwant\u00E9d,running")
							self.assertListEqual(inputs["token_type_ids"]       ,			[2] + [0] * sentence_len + [1] * sentence_len)