



"""simple docstring"""







import torch
from torch import nn
class        __A       (	nn.Module    ):







		'''simple docstring'''





		def __init__(	self						:  Optional[int]			,_snake_case						:  Any			,_snake_case						:  str			,_snake_case						:  List[Any]			,_snake_case						:  str			,_snake_case						:  Optional[Any]=1			,_snake_case						:  List[str]=False						)					->		Optional[Any]:






				"""simple docstring"""



				super().__init__()

				lowercase__   :    Optional[Any]				=			n_token
				lowercase__   :    List[str]				=			d_embed
				lowercase__   :    int				=			d_proj

				lowercase__   :    Union[str, Any]				=			cutoffs + [n_token]
				lowercase__   :    Optional[Any]				=			[0] + self.cutoffs
				lowercase__   :    Optional[Any]				=			div_val

				lowercase__   :    Dict				=			self.cutoffs[0]
				lowercase__   :    str				=			len(self.cutoffs						) - 1
				lowercase__   :    Optional[Any]				=			self.shortlist_size + self.n_clusters

				if self.n_clusters > 0:
						lowercase__   :    List[Any]				=			nn.Parameter(torch.zeros(self.n_clusters			,self.d_embed						)						)
						lowercase__   :    Any				=			nn.Parameter(torch.zeros(self.n_clusters						)						)

				lowercase__   :    Dict				=			nn.ModuleList()
				lowercase__   :    Optional[Any]				=			nn.ParameterList()

				if div_val == 1:
						for i in range(len(self.cutoffs						)						):
								if d_proj != d_embed:
										self.out_projs.append(nn.Parameter(torch.FloatTensor(_snake_case			,_snake_case						)						)						)
								else:
										self.out_projs.append(_snake_case						)

						self.out_layers.append(nn.Linear(_snake_case			,_snake_case						)						)
				else:
						for i in range(len(self.cutoffs						)						):
								lowercase__		,			lowercase__   :    Tuple				=			self.cutoff_ends[i], self.cutoff_ends[i + 1]
								lowercase__   :    List[str]				=			d_embed // (div_val**i)

								self.out_projs.append(nn.Parameter(torch.FloatTensor(_snake_case			,_snake_case						)						)						)

								self.out_layers.append(nn.Linear(_snake_case			,r_idx - l_idx						)						)

				lowercase__   :    Union[str, Any]				=			keep_order







		def 					UpperCAmelCase    (	self						:  List[Any]			,_snake_case						:  int			,_snake_case						:  Union[str, Any]			,_snake_case						:  int			,_snake_case						:  Optional[Any]						)					->		Tuple:






				"""simple docstring"""



				if proj is None:
						lowercase__   :    List[Any]				=			nn.functional.linear(_snake_case			,_snake_case			,bias=_snake_case						)
				else:
						# if CUDA_MAJOR <= 9 and CUDA_MINOR <= 1:
						lowercase__   :    List[str]				=			nn.functional.linear(_snake_case			,proj.t().contiguous()						)
						lowercase__   :    Tuple				=			nn.functional.linear(_snake_case			,_snake_case			,bias=_snake_case						)
						# else:
						#     logit = torch.einsum('bd,de,ev->bv', (hidden, proj, weight.t()))
						#     if bias is not None:
						#         logit = logit + bias

				return logit







		def 					UpperCAmelCase    (	self						:  Dict			,_snake_case						:  List[str]			,_snake_case						:  Dict=None			,_snake_case						:  Dict=False						)					->		Optional[int]:






				"""simple docstring"""



				if labels is not None:
						# Shift so that tokens < n predict n
						lowercase__   :    List[str]				=			hidden[..., :-1, :].contiguous()
						lowercase__   :    List[str]				=			labels[..., 1:].contiguous()
						lowercase__   :    str				=			hidden.view(-1			,hidden.size(-1						)						)
						lowercase__   :    int				=			labels.view(-1						)
						if hidden.size(0						) != labels.size(0						):
								raise RuntimeError('''Input and labels should have the same size in the batch dimension.'''						)
				else:
						lowercase__   :    str				=			hidden.view(-1			,hidden.size(-1						)						)

				if self.n_clusters == 0:
						lowercase__   :    int				=			self._compute_logit(_snake_case			,self.out_layers[0].weight			,self.out_layers[0].bias			,self.out_projs[0]						)
						if labels is not None:
								lowercase__   :    Dict				=			labels != -100
								lowercase__   :    Union[str, Any]				=			torch.zeros_like(_snake_case			,dtype=hidden.dtype			,device=hidden.device						)
								lowercase__   :    List[str]				=			(
								    -nn.functional.log_softmax(_snake_case			,dim=-1						)[mask].gather(1			,labels[mask].unsqueeze(1						)						).squeeze(1						)
								)
						else:
								lowercase__   :    str				=			nn.functional.log_softmax(_snake_case			,dim=-1						)
				else:
						# construct weights and biases
						lowercase__		,			lowercase__   :    Dict				=			[], []
						for i in range(len(self.cutoffs						)						):
								if self.div_val == 1:
										lowercase__		,			lowercase__   :    Optional[Any]				=			self.cutoff_ends[i], self.cutoff_ends[i + 1]
										lowercase__   :    List[str]				=			self.out_layers[0].weight[l_idx:r_idx]
										lowercase__   :    int				=			self.out_layers[0].bias[l_idx:r_idx]
								else:
										lowercase__   :    Union[str, Any]				=			self.out_layers[i].weight
										lowercase__   :    List[str]				=			self.out_layers[i].bias

								if i == 0:
										lowercase__   :    int				=			torch.cat([weight_i, self.cluster_weight]			,dim=0						)
										lowercase__   :    Tuple				=			torch.cat([bias_i, self.cluster_bias]			,dim=0						)

								weights.append(_snake_case						)
								biases.append(_snake_case						)

						lowercase__		,			lowercase__		,			lowercase__   :    Optional[Any]				=			weights[0], biases[0], self.out_projs[0]

						lowercase__   :    Optional[int]				=			self._compute_logit(_snake_case			,_snake_case			,_snake_case			,_snake_case						)
						lowercase__   :    Tuple				=			nn.functional.log_softmax(_snake_case			,dim=1						)

						if labels is None:
								lowercase__   :    Any				=			hidden.new_empty((head_logit.size(0						), self.n_token)						)
						else:
								lowercase__   :    List[Any]				=			torch.zeros_like(_snake_case			,dtype=hidden.dtype			,device=hidden.device						)

						lowercase__   :    Any				=			0
						lowercase__   :    Optional[int]				=			[0] + self.cutoffs
						for i in range(len(_snake_case						) - 1						):
								lowercase__		,			lowercase__   :    Any				=			cutoff_values[i], cutoff_values[i + 1]

								if labels is not None:
										lowercase__   :    Dict				=			(labels >= l_idx) & (labels < r_idx)
										lowercase__   :    Optional[int]				=			mask_i.nonzero().squeeze()

										if indices_i.numel() == 0:
												continue

										lowercase__   :    Optional[int]				=			labels.index_select(0			,_snake_case						) - l_idx
										lowercase__   :    Tuple				=			head_logprob.index_select(0			,_snake_case						)
										lowercase__   :    List[Any]				=			hidden.index_select(0			,_snake_case						)
								else:
										lowercase__   :    int				=			hidden

								if i == 0:
										if labels is not None:
												lowercase__   :    str				=			head_logprob_i.gather(1			,target_i[:, None]						).squeeze(1						)
										else:
												lowercase__   :    Dict				=			head_logprob[:, : self.cutoffs[0]]
								else:
										lowercase__		,			lowercase__		,			lowercase__   :    Optional[Any]				=			weights[i], biases[i], self.out_projs[i]

										lowercase__   :    Optional[Any]				=			self._compute_logit(_snake_case			,_snake_case			,_snake_case			,_snake_case						)
										lowercase__   :    Union[str, Any]				=			nn.functional.log_softmax(_snake_case			,dim=1						)
										lowercase__   :    Optional[int]				=			self.cutoffs[0] + i - 1  # No probability for the head cluster
										if labels is not None:
												lowercase__   :    Dict				=			head_logprob_i[:, cluster_prob_idx] + tail_logprob_i.gather(
												    1			,target_i[:, None]						).squeeze(1						)
										else:
												lowercase__   :    List[str]				=			head_logprob[:, cluster_prob_idx, None] + tail_logprob_i
												lowercase__   :    Optional[Any]				=			logprob_i

								if labels is not None:
										if (hasattr(self			,'''keep_order'''						) and self.keep_order) or keep_order:
												out.index_copy_(0			,_snake_case			,-logprob_i						)
										else:
												out[offset : offset + logprob_i.size(0						)].copy_(-logprob_i						)
										offset += logprob_i.size(0						)

				return out







		def 					UpperCAmelCase    (	self						:  Optional[Any]			,_snake_case						:  List[str]						)					->		int:






				"""simple docstring"""



				if self.n_clusters == 0:
						lowercase__   :    List[Any]				=			self._compute_logit(_snake_case			,self.out_layers[0].weight			,self.out_layers[0].bias			,self.out_projs[0]						)
						return nn.functional.log_softmax(_snake_case			,dim=-1						)
				else:
						# construct weights and biases
						lowercase__		,			lowercase__   :    Optional[Any]				=			[], []
						for i in range(len(self.cutoffs						)						):
								if self.div_val == 1:
										lowercase__		,			lowercase__   :    List[str]				=			self.cutoff_ends[i], self.cutoff_ends[i + 1]
										lowercase__   :    List[Any]				=			self.out_layers[0].weight[l_idx:r_idx]
										lowercase__   :    List[Any]				=			self.out_layers[0].bias[l_idx:r_idx]
								else:
										lowercase__   :    Optional[int]				=			self.out_layers[i].weight
										lowercase__   :    int				=			self.out_layers[i].bias

								if i == 0:
										lowercase__   :    str				=			torch.cat([weight_i, self.cluster_weight]			,dim=0						)
										lowercase__   :    Dict				=			torch.cat([bias_i, self.cluster_bias]			,dim=0						)

								weights.append(_snake_case						)
								biases.append(_snake_case						)

						lowercase__		,			lowercase__		,			lowercase__   :    List[str]				=			weights[0], biases[0], self.out_projs[0]
						lowercase__   :    Optional[Any]				=			self._compute_logit(_snake_case			,_snake_case			,_snake_case			,_snake_case						)

						lowercase__   :    Optional[Any]				=			hidden.new_empty((head_logit.size(0						), self.n_token)						)
						lowercase__   :    List[Any]				=			nn.functional.log_softmax(_snake_case			,dim=1						)

						lowercase__   :    Optional[Any]				=			[0] + self.cutoffs
						for i in range(len(_snake_case						) - 1						):
								lowercase__		,			lowercase__   :    Union[str, Any]				=			cutoff_values[i], cutoff_values[i + 1]

								if i == 0:
										lowercase__   :    Dict				=			head_logprob[:, : self.cutoffs[0]]
								else:
										lowercase__		,			lowercase__		,			lowercase__   :    List[str]				=			weights[i], biases[i], self.out_projs[i]

										lowercase__   :    Any				=			self._compute_logit(_snake_case			,_snake_case			,_snake_case			,_snake_case						)
										lowercase__   :    Any				=			nn.functional.log_softmax(_snake_case			,dim=1						)

										lowercase__   :    Any				=			head_logprob[:, -i] + tail_logprob_i
										lowercase__   :    str				=			logprob_i

						return out


"""simple docstring"""





import unittest

from transformers import SPIECE_UNDERLINE, XLNetTokenizer, XLNetTokenizerFast
from transformers.testing_utils import get_tests_dir, require_sentencepiece, require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin


__magic_name__       =			get_tests_dir("fixtures/test_sentencepiece.model")
@require_sentencepiece
@require_tokenizers
class 						SCREAMING_SNAKE_CASE_ (						__a     ,				unittest.TestCase ):




		"""simple docstring"""





		__lowercase							:  Optional[Any]						   =		XLNetTokenizer
		__lowercase							:  List[str]						   =		XLNetTokenizerFast
		__lowercase							:  List[Any]						   =		True
		__lowercase							:  int						   =		True




		def 	snake_case_     (     self):
				super().setUp()

				# We have a SentencePiece fixture for testing
				__SCREAMING_SNAKE_CASE            = XLNetTokenizer(lowerCAmelCase__							,  keep_accents=lowerCAmelCase__)
				tokenizer.sanitize_special_tokens()
				tokenizer.save_pretrained(self.tmpdirname)




		def 	snake_case_     (     self):
				__SCREAMING_SNAKE_CASE            = """<s>"""
				__SCREAMING_SNAKE_CASE            = 1

				self.assertEqual(self.get_tokenizer()._convert_token_to_id(lowerCAmelCase__)							,  lowerCAmelCase__)
				self.assertEqual(self.get_tokenizer()._convert_id_to_token(lowerCAmelCase__)							,  lowerCAmelCase__)




		def 	snake_case_     (     self):
				__SCREAMING_SNAKE_CASE            = list(self.get_tokenizer().get_vocab().keys())

				self.assertEqual(vocab_keys[0]							,  """<unk>""")
				self.assertEqual(vocab_keys[1]							,  """<s>""")
				self.assertEqual(vocab_keys[-1]							,  """<eod>""")
				self.assertEqual(len(lowerCAmelCase__)							,  1_0_0_6)




		def 	snake_case_     (     self):
				self.assertEqual(self.get_tokenizer().vocab_size							,  1_0_0_0)




		def 	snake_case_     (     self):
				__SCREAMING_SNAKE_CASE            = XLNetTokenizer(lowerCAmelCase__							,  keep_accents=lowerCAmelCase__)

				__SCREAMING_SNAKE_CASE            = tokenizer.tokenize("""This is a test""")
				self.assertListEqual(lowerCAmelCase__							,  ["""▁This""", """▁is""", """▁a""", """▁t""", """est"""])

				self.assertListEqual(tokenizer.convert_tokens_to_ids(lowerCAmelCase__)							,  [2_8_5, 4_6, 1_0, 1_7_0, 3_8_2])

				__SCREAMING_SNAKE_CASE            = tokenizer.tokenize("""I was born in 92000, and this is falsé.""")
				self.assertListEqual(
				    lowerCAmelCase__							,  [
				        SPIECE_UNDERLINE + """I""",
				        SPIECE_UNDERLINE + """was""",
				        SPIECE_UNDERLINE + """b""",
				        """or""",
				        """n""",
				        SPIECE_UNDERLINE + """in""",
				        SPIECE_UNDERLINE + """""",
				        """9""",
				        """2""",
				        """0""",
				        """0""",
				        """0""",
				        """,""",
				        SPIECE_UNDERLINE + """and""",
				        SPIECE_UNDERLINE + """this""",
				        SPIECE_UNDERLINE + """is""",
				        SPIECE_UNDERLINE + """f""",
				        """al""",
				        """s""",
				        """é""",
				        """.""",
				    ]							,  )
				__SCREAMING_SNAKE_CASE            = tokenizer.convert_tokens_to_ids(lowerCAmelCase__)
				self.assertListEqual(lowerCAmelCase__							,  [8, 2_1, 8_4, 5_5, 2_4, 1_9, 7, 0, 6_0_2, 3_4_7, 3_4_7, 3_4_7, 3, 1_2, 6_6, 4_6, 7_2, 8_0, 6, 0, 4])

				__SCREAMING_SNAKE_CASE            = tokenizer.convert_ids_to_tokens(lowerCAmelCase__)
				self.assertListEqual(
				    lowerCAmelCase__							,  [
				        SPIECE_UNDERLINE + """I""",
				        SPIECE_UNDERLINE + """was""",
				        SPIECE_UNDERLINE + """b""",
				        """or""",
				        """n""",
				        SPIECE_UNDERLINE + """in""",
				        SPIECE_UNDERLINE + """""",
				        """<unk>""",
				        """2""",
				        """0""",
				        """0""",
				        """0""",
				        """,""",
				        SPIECE_UNDERLINE + """and""",
				        SPIECE_UNDERLINE + """this""",
				        SPIECE_UNDERLINE + """is""",
				        SPIECE_UNDERLINE + """f""",
				        """al""",
				        """s""",
				        """<unk>""",
				        """.""",
				    ]							,  )




		def 	snake_case_     (     self):
				__SCREAMING_SNAKE_CASE            = XLNetTokenizer(lowerCAmelCase__							,  do_lower_case=lowerCAmelCase__)
				__SCREAMING_SNAKE_CASE            = tokenizer.tokenize("""I was born in 92000, and this is falsé.""")
				self.assertListEqual(
				    lowerCAmelCase__							,  [
				        SPIECE_UNDERLINE + """""",
				        """i""",
				        SPIECE_UNDERLINE + """was""",
				        SPIECE_UNDERLINE + """b""",
				        """or""",
				        """n""",
				        SPIECE_UNDERLINE + """in""",
				        SPIECE_UNDERLINE + """""",
				        """9""",
				        """2""",
				        """0""",
				        """0""",
				        """0""",
				        """,""",
				        SPIECE_UNDERLINE + """and""",
				        SPIECE_UNDERLINE + """this""",
				        SPIECE_UNDERLINE + """is""",
				        SPIECE_UNDERLINE + """f""",
				        """al""",
				        """se""",
				        """.""",
				    ]							,  )
				self.assertListEqual(tokenizer.tokenize("""H\u00E9llo""")							,  ["""▁he""", """ll""", """o"""])




		def 	snake_case_     (     self):
				__SCREAMING_SNAKE_CASE            = XLNetTokenizer(lowerCAmelCase__							,  do_lower_case=lowerCAmelCase__)
				__SCREAMING_SNAKE_CASE            = tokenizer.tokenize("""I was born in 92000, and this is falsé.""")
				self.assertListEqual(
				    lowerCAmelCase__							,  [
				        SPIECE_UNDERLINE + """I""",
				        SPIECE_UNDERLINE + """was""",
				        SPIECE_UNDERLINE + """b""",
				        """or""",
				        """n""",
				        SPIECE_UNDERLINE + """in""",
				        SPIECE_UNDERLINE + """""",
				        """9""",
				        """2""",
				        """0""",
				        """0""",
				        """0""",
				        """,""",
				        SPIECE_UNDERLINE + """and""",
				        SPIECE_UNDERLINE + """this""",
				        SPIECE_UNDERLINE + """is""",
				        SPIECE_UNDERLINE + """f""",
				        """al""",
				        """se""",
				        """.""",
				    ]							,  )




		@slow
		def 	snake_case_     (     self):
				__SCREAMING_SNAKE_CASE            = XLNetTokenizer.from_pretrained("""xlnet-base-cased""")

				__SCREAMING_SNAKE_CASE            = tokenizer.encode("""sequence builders"""							,  add_special_tokens=lowerCAmelCase__)
				__SCREAMING_SNAKE_CASE            = tokenizer.encode("""multi-sequence build"""							,  add_special_tokens=lowerCAmelCase__)

				__SCREAMING_SNAKE_CASE            = tokenizer.build_inputs_with_special_tokens(lowerCAmelCase__)
				__SCREAMING_SNAKE_CASE            = tokenizer.build_inputs_with_special_tokens(lowerCAmelCase__							,  lowerCAmelCase__)

				assert encoded_sentence == text + [4, 3]
				assert encoded_pair == text + [4] + text_a + [4, 3]




		@slow
		def 	snake_case_     (     self):
				# fmt: off
				__SCREAMING_SNAKE_CASE            = {"""input_ids""": [[1_7, 2_1_4_4_2, 2_7_0, 1_7, 1_0, 1_4_6_4_5, 3_1_8, 3_4, 1_7, 4_5_4_6, 3_1_4_5, 7_8_7, 1_3, 7_7_5_2, 2_2_0_1_8, 2_3, 2_1, 1_7, 4_5_4_6, 3_1_4_5, 7_8_7, 1_3, 3_3_5_2, 1_4_4_3_1, 1_3, 5_5_0_0, 1_1, 1_1_7_6, 5_8_0, 1_3, 1_6_8_1_9, 4_7_9_7, 2_3, 1_7, 1_0, 1_7_1_3_5, 6_5_8, 1_9, 4_5_7, 7_9_3_2, 1_3, 1_8_4, 1_9, 3_1_5_4, 1_7_1_3_5, 6_4_6_8, 1_9, 1_4_0_4, 1_2_2_6_9, 1_9, 4_2_2_9, 5_3_5_6, 1_6_2_6_4, 4_6, 1_9, 1_7, 2_0_5_4_5, 1_0_3_9_5, 9, 9, 9, 1_1, 2_8, 6_4_2_1, 9_5_3_1, 2_0_7_2_9, 1_7, 1_0, 3_5_3, 1_7_0_2_2, 1_1, 2_1, 6_4_2_1, 9_5_3_1, 1_6_9_4_9, 1_7, 1_0, 1_1_5_0_9, 7_5_3, 1_1, 3_3, 9_5, 2_4_2_1, 7_3_8_5, 9_5_6, 1_4_4_3_1, 2_6_2_6, 2_5, 8_4_2, 7_3_8_5, 4_8_3_6, 2_1, 1_4_2_9, 2_2_7_2, 9_8_5_5, 3_1_2_0, 1_6_1, 2_4_7_3_8, 1_9, 1_3_2_0_3, 6_5_8, 2_1_8, 7_8_7, 2_1, 4_3_0, 1_8_4_8_2, 8_4_7, 2_6_3_7, 9, 4, 3], [5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 3_2_2, 2_2_1_7_8, 2_7, 1_0_6_4, 2_2, 9_5_6, 1_3, 1_1_1_0_1, 1_4_2_9, 5_8_5_4, 2_4_3_1_3, 1_8_9_5_3, 4_0, 4_2_2, 2_4_3_6_6, 6_8, 1_7_5_8, 3_7, 1_0_4_8_3, 1_4_2_5_7, 3_1, 2_0_7, 2_6_3, 2_1, 2_0_3, 3_7_7_3, 2_5, 7_1, 9_7_3_5, 9, 4, 3], [5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 3_2, 2_0_4_9, 3_4_4_2, 1_7, 1_3_8_9_4, 3_3_8_0, 2_3, 9_5, 1_8, 1_7_6_3_4, 2_2_8_8, 9, 4, 3]], """token_type_ids""": [[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2], [3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2], [3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2]], """attention_mask""": [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]]}  # noqa: E501
				# fmt: on

				self.tokenizer_integration_test_util(
				    expected_encoding=lowerCAmelCase__							,  model_name="""xlnet-base-cased"""							,  revision="""c841166438c31ec7ca9a106dee7bb312b73ae511"""							,  )




