
"""simple docstring"""





import argparse
import os

import evaluate
import torch
from datasets import load_dataset
from torch.optim import AdamW
from torch.utils.data import DataLoader
from transformers import AutoModelForSequenceClassification, AutoTokenizer, get_linear_schedule_with_warmup, set_seed

from accelerate import Accelerator, DistributedType


########################################################################
# This is a fully working simple example to use Accelerate,
# specifically showcasing the experiment tracking capability,
# and builds off the `nlp_example.py` script.
#
# This example trains a Bert base model on GLUE MRPC
# in any of the following settings (with the same script):
#   - single CPU or single GPU
#   - multi GPUS (using PyTorch distributed mode)
#   - (multi) TPUs
#   - fp16 (mixed-precision) or fp32 (normal precision)
#
# To help focus on the differences in the code, building `DataLoaders`
# was refactored into its own function.
# New additions from the base script can be found quickly by
# looking for the # New Code # tags
#
# To run it in each of these various modes, follow the instructions
# in the readme for examples:
# https://github.com/huggingface/accelerate/tree/main/examples
#
########################################################################

__magic_name__       =			16
__magic_name__       =			32



def 						_lowerCAmelCase				(				UpperCamelCase_  ,				UpperCamelCase_ = 16				):
		__SCREAMING_SNAKE_CASE            = AutoTokenizer.from_pretrained("""bert-base-cased"""				)
		__SCREAMING_SNAKE_CASE            = load_dataset("""glue"""  ,				"""mrpc"""				)

		def tokenize_function(UpperCamelCase_				):
				# max_length=None => use the model max length (it's actually the default)
				__SCREAMING_SNAKE_CASE            = tokenizer(examples["""sentence1"""]  ,				examples["""sentence2"""]  ,				truncation=UpperCamelCase_  ,				max_length=UpperCamelCase_				)
				return outputs

		# Apply the method we just defined to all the examples in all the splits of the dataset
		# starting with the main process first:
		with accelerator.main_process_first():
				__SCREAMING_SNAKE_CASE            = datasets.map(
				    UpperCamelCase_  ,				batched=UpperCamelCase_  ,				remove_columns=["""idx""", """sentence1""", """sentence2"""]  ,				)

		# We also rename the 'label' column to 'labels' which is the expected name for labels by the models of the
		# transformers library
		__SCREAMING_SNAKE_CASE            = tokenized_datasets.rename_column("""label"""  ,				"""labels"""				)

		def collate_fn(UpperCamelCase_				):
				# On TPU it's best to pad everything to the same length or training will be very slow.
				__SCREAMING_SNAKE_CASE            = 128 if accelerator.distributed_type == DistributedType.TPU else None
				# When using mixed precision we want round multiples of 8/16
				if accelerator.mixed_precision == "fp8":
						__SCREAMING_SNAKE_CASE            = 16
				elif accelerator.mixed_precision != "no":
						__SCREAMING_SNAKE_CASE            = 8
				else:
						__SCREAMING_SNAKE_CASE            = None

				return tokenizer.pad(
				    UpperCamelCase_  ,				padding="""longest"""  ,				max_length=UpperCamelCase_  ,				pad_to_multiple_of=UpperCamelCase_  ,				return_tensors="""pt"""  ,				)

		# Instantiate dataloaders.
		__SCREAMING_SNAKE_CASE            = DataLoader(
		    tokenized_datasets["""train"""]  ,				shuffle=UpperCamelCase_  ,				collate_fn=UpperCamelCase_  ,				batch_size=UpperCamelCase_				)
		__SCREAMING_SNAKE_CASE            = DataLoader(
		    tokenized_datasets["""validation"""]  ,				shuffle=UpperCamelCase_  ,				collate_fn=UpperCamelCase_  ,				batch_size=UpperCamelCase_				)

		return train_dataloader, eval_dataloader


# For testing only
if os.environ.get("TESTING_MOCKED_DATALOADERS", None) == "1":
							from accelerate.test_utils.training import mocked_dataloaders

							__magic_name__       =			mocked_dataloaders  # noqa: F811



def 						_lowerCAmelCase				(				UpperCamelCase_  ,				UpperCamelCase_				):
		# For testing only
		if os.environ.get("""TESTING_MOCKED_DATALOADERS"""  ,				UpperCamelCase_				) == "1":
				__SCREAMING_SNAKE_CASE            = 2
		# Initialize Accelerator

		# New Code #
		# We pass in "all" to `log_with` to grab all available trackers in the environment
		# Note: If using a custom `Tracker` class, should be passed in here such as:
		# >>> log_with = ["all", MyCustomTrackerClassInstance()]
		if args.with_tracking:
				__SCREAMING_SNAKE_CASE            = Accelerator(
				    cpu=args.cpu  ,				mixed_precision=args.mixed_precision  ,				log_with="""all"""  ,				project_dir=args.project_dir				)
		else:
				__SCREAMING_SNAKE_CASE            = Accelerator(cpu=args.cpu  ,				mixed_precision=args.mixed_precision				)
		# Sample hyper-parameters for learning rate, batch size, seed and a few other HPs
		__SCREAMING_SNAKE_CASE            = config["""lr"""]
		__SCREAMING_SNAKE_CASE            = int(config["""num_epochs"""]				)
		__SCREAMING_SNAKE_CASE            = int(config["""seed"""]				)
		__SCREAMING_SNAKE_CASE            = int(config["""batch_size"""]				)
		set_seed(UpperCamelCase_				)

		__SCREAMING_SNAKE_CASE       ,__SCREAMING_SNAKE_CASE            = get_dataloaders(UpperCamelCase_  ,				UpperCamelCase_				)
		__SCREAMING_SNAKE_CASE            = evaluate.load("""glue"""  ,				"""mrpc"""				)

		# If the batch size is too big we use gradient accumulation
		__SCREAMING_SNAKE_CASE            = 1
		if batch_size > MAX_GPU_BATCH_SIZE and accelerator.distributed_type != DistributedType.TPU:
				__SCREAMING_SNAKE_CASE            = batch_size // MAX_GPU_BATCH_SIZE
				__SCREAMING_SNAKE_CASE            = MAX_GPU_BATCH_SIZE

		# Instantiate the model (we build the model here so that the seed also control new weights initialization)
		__SCREAMING_SNAKE_CASE            = AutoModelForSequenceClassification.from_pretrained("""bert-base-cased"""  ,				return_dict=UpperCamelCase_				)

		# We could avoid this line since the accelerator is set with `device_placement=True` (default value).
		# Note that if you are placing tensors on devices manually, this line absolutely needs to be before the optimizer
		# creation otherwise training will not work on TPU (`accelerate` will kindly throw an error to make us aware of that).
		__SCREAMING_SNAKE_CASE            = model.to(accelerator.device				)

		# Instantiate optimizer
		__SCREAMING_SNAKE_CASE            = AdamW(params=model.parameters()  ,				lr=UpperCamelCase_				)

		# Instantiate scheduler
		__SCREAMING_SNAKE_CASE            = get_linear_schedule_with_warmup(
		    optimizer=UpperCamelCase_  ,				num_warmup_steps=100  ,				num_training_steps=(len(UpperCamelCase_				) * num_epochs) // gradient_accumulation_steps  ,				)

		# Prepare everything
		# There is no specific order to remember, we just need to unpack the objects in the same order we gave them to the
		# prepare method.
		__SCREAMING_SNAKE_CASE       ,__SCREAMING_SNAKE_CASE       ,__SCREAMING_SNAKE_CASE       ,__SCREAMING_SNAKE_CASE       ,__SCREAMING_SNAKE_CASE            = accelerator.prepare(
		    UpperCamelCase_  ,				UpperCamelCase_  ,				UpperCamelCase_  ,				UpperCamelCase_  ,				UpperCamelCase_				)

		# New Code #
		# We need to initialize the trackers we use. Overall configurations can also be stored
		if args.with_tracking:
				__SCREAMING_SNAKE_CASE            = os.path.split(UpperCamelCase_				)[-1].split("""."""				)[0]
				accelerator.init_trackers(UpperCamelCase_  ,				UpperCamelCase_				)

		# Now we train the model
		for epoch in range(UpperCamelCase_				):
				model.train()
				# New Code #
				# For our tracking example, we will log the total loss of each epoch
				if args.with_tracking:
						__SCREAMING_SNAKE_CASE            = 0
				for step, batch in enumerate(UpperCamelCase_				):
						# We could avoid this line since we set the accelerator with `device_placement=True`.
						batch.to(accelerator.device				)
						__SCREAMING_SNAKE_CASE            = model(**UpperCamelCase_				)
						__SCREAMING_SNAKE_CASE            = outputs.loss
						# New Code #
						if args.with_tracking:
								total_loss += loss.detach().float()
						__SCREAMING_SNAKE_CASE            = loss / gradient_accumulation_steps
						accelerator.backward(UpperCamelCase_				)
						if step % gradient_accumulation_steps == 0:
								optimizer.step()
								lr_scheduler.step()
								optimizer.zero_grad()

				model.eval()
				for step, batch in enumerate(UpperCamelCase_				):
						# We could avoid this line since we set the accelerator with `device_placement=True` (the default).
						batch.to(accelerator.device				)
						with torch.no_grad():
								__SCREAMING_SNAKE_CASE            = model(**UpperCamelCase_				)
						__SCREAMING_SNAKE_CASE            = outputs.logits.argmax(dim=-1				)
						__SCREAMING_SNAKE_CASE       ,__SCREAMING_SNAKE_CASE            = accelerator.gather_for_metrics((predictions, batch["""labels"""])				)
						metric.add_batch(
						    predictions=UpperCamelCase_  ,				references=UpperCamelCase_  ,				)

				__SCREAMING_SNAKE_CASE            = metric.compute()
				# Use accelerator.print to print only on the main process.
				accelerator.print(f"epoch {epoch}:"  ,				UpperCamelCase_				)

				# New Code #
				# To actually log, we call `Accelerator.log`
				# The values passed can be of `str`, `int`, `float` or `dict` of `str` to `float`/`int`
				if args.with_tracking:
						accelerator.log(
						    {
						        """accuracy""": eval_metric["""accuracy"""],
						        """f1""": eval_metric["""f1"""],
						        """train_loss""": total_loss.item() / len(UpperCamelCase_				),
						        """epoch""": epoch,
						    }  ,				step=UpperCamelCase_  ,				)

    # New Code #
    # When a run is finished, you should call `accelerator.end_training()`
    # to close all of the open trackers
		if args.with_tracking:
				accelerator.end_training()



def 						_lowerCAmelCase				(				):
		__SCREAMING_SNAKE_CASE            = argparse.ArgumentParser(description="""Simple example of training script."""				)
		parser.add_argument(
		    """--mixed_precision"""  ,				type=UpperCamelCase_  ,				default=UpperCamelCase_  ,				choices=["""no""", """fp16""", """bf16""", """fp8"""]  ,				help="""Whether to use mixed precision. Choose"""
		    """between fp16 and bf16 (bfloat16). Bf16 requires PyTorch >= 1.10."""
		    """and an Nvidia Ampere GPU."""  ,				)
		parser.add_argument("""--cpu"""  ,				action="""store_true"""  ,				help="""If passed, will train on the CPU."""				)
		parser.add_argument(
		    """--with_tracking"""  ,				action="""store_true"""  ,				help="""Whether to load in all available experiment trackers from the environment and use them for logging."""  ,				)
		parser.add_argument(
		    """--project_dir"""  ,				type=UpperCamelCase_  ,				default="""logs"""  ,				help="""Location on where to store experiment tracking logs` and relevent project information"""  ,				)
		__SCREAMING_SNAKE_CASE            = parser.parse_args()
		__SCREAMING_SNAKE_CASE            = {"""lr""": 2e-5, """num_epochs""": 3, """seed""": 42, """batch_size""": 16}
		training_function(UpperCamelCase_  ,				UpperCamelCase_				)


if __name__ == "__main__":
							main()






"""simple docstring"""





import unittest

from transformers import SPIECE_UNDERLINE, XLNetTokenizer, XLNetTokenizerFast
from transformers.testing_utils import get_tests_dir, require_sentencepiece, require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin


__magic_name__       =			get_tests_dir("fixtures/test_sentencepiece.model")
@require_sentencepiece
@require_tokenizers
class 						SCREAMING_SNAKE_CASE_ (						__a     ,				unittest.TestCase ):




		"""simple docstring"""





		__lowercase							:  Optional[Any]						   =		XLNetTokenizer
		__lowercase							:  List[str]						   =		XLNetTokenizerFast
		__lowercase							:  List[Any]						   =		True
		__lowercase							:  int						   =		True




		def 	snake_case_     (     self):
				super().setUp()

				# We have a SentencePiece fixture for testing
				__SCREAMING_SNAKE_CASE            = XLNetTokenizer(lowerCAmelCase__							,  keep_accents=lowerCAmelCase__)
				tokenizer.sanitize_special_tokens()
				tokenizer.save_pretrained(self.tmpdirname)




		def 	snake_case_     (     self):
				__SCREAMING_SNAKE_CASE            = """<s>"""
				__SCREAMING_SNAKE_CASE            = 1

				self.assertEqual(self.get_tokenizer()._convert_token_to_id(lowerCAmelCase__)							,  lowerCAmelCase__)
				self.assertEqual(self.get_tokenizer()._convert_id_to_token(lowerCAmelCase__)							,  lowerCAmelCase__)




		def 	snake_case_     (     self):
				__SCREAMING_SNAKE_CASE            = list(self.get_tokenizer().get_vocab().keys())

				self.assertEqual(vocab_keys[0]							,  """<unk>""")
				self.assertEqual(vocab_keys[1]							,  """<s>""")
				self.assertEqual(vocab_keys[-1]							,  """<eod>""")
				self.assertEqual(len(lowerCAmelCase__)							,  1_0_0_6)




		def 	snake_case_     (     self):
				self.assertEqual(self.get_tokenizer().vocab_size							,  1_0_0_0)




		def 	snake_case_     (     self):
				__SCREAMING_SNAKE_CASE            = XLNetTokenizer(lowerCAmelCase__							,  keep_accents=lowerCAmelCase__)

				__SCREAMING_SNAKE_CASE            = tokenizer.tokenize("""This is a test""")
				self.assertListEqual(lowerCAmelCase__							,  ["""▁This""", """▁is""", """▁a""", """▁t""", """est"""])

				self.assertListEqual(tokenizer.convert_tokens_to_ids(lowerCAmelCase__)							,  [2_8_5, 4_6, 1_0, 1_7_0, 3_8_2])

				__SCREAMING_SNAKE_CASE            = tokenizer.tokenize("""I was born in 92000, and this is falsé.""")
				self.assertListEqual(
				    lowerCAmelCase__							,  [
				        SPIECE_UNDERLINE + """I""",
				        SPIECE_UNDERLINE + """was""",
				        SPIECE_UNDERLINE + """b""",
				        """or""",
				        """n""",
				        SPIECE_UNDERLINE + """in""",
				        SPIECE_UNDERLINE + """""",
				        """9""",
				        """2""",
				        """0""",
				        """0""",
				        """0""",
				        """,""",
				        SPIECE_UNDERLINE + """and""",
				        SPIECE_UNDERLINE + """this""",
				        SPIECE_UNDERLINE + """is""",
				        SPIECE_UNDERLINE + """f""",
				        """al""",
				        """s""",
				        """é""",
				        """.""",
				    ]							,  )
				__SCREAMING_SNAKE_CASE            = tokenizer.convert_tokens_to_ids(lowerCAmelCase__)
				self.assertListEqual(lowerCAmelCase__							,  [8, 2_1, 8_4, 5_5, 2_4, 1_9, 7, 0, 6_0_2, 3_4_7, 3_4_7, 3_4_7, 3, 1_2, 6_6, 4_6, 7_2, 8_0, 6, 0, 4])

				__SCREAMING_SNAKE_CASE            = tokenizer.convert_ids_to_tokens(lowerCAmelCase__)
				self.assertListEqual(
				    lowerCAmelCase__							,  [
				        SPIECE_UNDERLINE + """I""",
				        SPIECE_UNDERLINE + """was""",
				        SPIECE_UNDERLINE + """b""",
				        """or""",
				        """n""",
				        SPIECE_UNDERLINE + """in""",
				        SPIECE_UNDERLINE + """""",
				        """<unk>""",
				        """2""",
				        """0""",
				        """0""",
				        """0""",
				        """,""",
				        SPIECE_UNDERLINE + """and""",
				        SPIECE_UNDERLINE + """this""",
				        SPIECE_UNDERLINE + """is""",
				        SPIECE_UNDERLINE + """f""",
				        """al""",
				        """s""",
				        """<unk>""",
				        """.""",
				    ]							,  )




		def 	snake_case_     (     self):
				__SCREAMING_SNAKE_CASE            = XLNetTokenizer(lowerCAmelCase__							,  do_lower_case=lowerCAmelCase__)
				__SCREAMING_SNAKE_CASE            = tokenizer.tokenize("""I was born in 92000, and this is falsé.""")
				self.assertListEqual(
				    lowerCAmelCase__							,  [
				        SPIECE_UNDERLINE + """""",
				        """i""",
				        SPIECE_UNDERLINE + """was""",
				        SPIECE_UNDERLINE + """b""",
				        """or""",
				        """n""",
				        SPIECE_UNDERLINE + """in""",
				        SPIECE_UNDERLINE + """""",
				        """9""",
				        """2""",
				        """0""",
				        """0""",
				        """0""",
				        """,""",
				        SPIECE_UNDERLINE + """and""",
				        SPIECE_UNDERLINE + """this""",
				        SPIECE_UNDERLINE + """is""",
				        SPIECE_UNDERLINE + """f""",
				        """al""",
				        """se""",
				        """.""",
				    ]							,  )
				self.assertListEqual(tokenizer.tokenize("""H\u00E9llo""")							,  ["""▁he""", """ll""", """o"""])




		def 	snake_case_     (     self):
				__SCREAMING_SNAKE_CASE            = XLNetTokenizer(lowerCAmelCase__							,  do_lower_case=lowerCAmelCase__)
				__SCREAMING_SNAKE_CASE            = tokenizer.tokenize("""I was born in 92000, and this is falsé.""")
				self.assertListEqual(
				    lowerCAmelCase__							,  [
				        SPIECE_UNDERLINE + """I""",
				        SPIECE_UNDERLINE + """was""",
				        SPIECE_UNDERLINE + """b""",
				        """or""",
				        """n""",
				        SPIECE_UNDERLINE + """in""",
				        SPIECE_UNDERLINE + """""",
				        """9""",
				        """2""",
				        """0""",
				        """0""",
				        """0""",
				        """,""",
				        SPIECE_UNDERLINE + """and""",
				        SPIECE_UNDERLINE + """this""",
				        SPIECE_UNDERLINE + """is""",
				        SPIECE_UNDERLINE + """f""",
				        """al""",
				        """se""",
				        """.""",
				    ]							,  )




		@slow
		def 	snake_case_     (     self):
				__SCREAMING_SNAKE_CASE            = XLNetTokenizer.from_pretrained("""xlnet-base-cased""")

				__SCREAMING_SNAKE_CASE            = tokenizer.encode("""sequence builders"""							,  add_special_tokens=lowerCAmelCase__)
				__SCREAMING_SNAKE_CASE            = tokenizer.encode("""multi-sequence build"""							,  add_special_tokens=lowerCAmelCase__)

				__SCREAMING_SNAKE_CASE            = tokenizer.build_inputs_with_special_tokens(lowerCAmelCase__)
				__SCREAMING_SNAKE_CASE            = tokenizer.build_inputs_with_special_tokens(lowerCAmelCase__							,  lowerCAmelCase__)

				assert encoded_sentence == text + [4, 3]
				assert encoded_pair == text + [4] + text_a + [4, 3]




		@slow
		def 	snake_case_     (     self):
				# fmt: off
				__SCREAMING_SNAKE_CASE            = {"""input_ids""": [[1_7, 2_1_4_4_2, 2_7_0, 1_7, 1_0, 1_4_6_4_5, 3_1_8, 3_4, 1_7, 4_5_4_6, 3_1_4_5, 7_8_7, 1_3, 7_7_5_2, 2_2_0_1_8, 2_3, 2_1, 1_7, 4_5_4_6, 3_1_4_5, 7_8_7, 1_3, 3_3_5_2, 1_4_4_3_1, 1_3, 5_5_0_0, 1_1, 1_1_7_6, 5_8_0, 1_3, 1_6_8_1_9, 4_7_9_7, 2_3, 1_7, 1_0, 1_7_1_3_5, 6_5_8, 1_9, 4_5_7, 7_9_3_2, 1_3, 1_8_4, 1_9, 3_1_5_4, 1_7_1_3_5, 6_4_6_8, 1_9, 1_4_0_4, 1_2_2_6_9, 1_9, 4_2_2_9, 5_3_5_6, 1_6_2_6_4, 4_6, 1_9, 1_7, 2_0_5_4_5, 1_0_3_9_5, 9, 9, 9, 1_1, 2_8, 6_4_2_1, 9_5_3_1, 2_0_7_2_9, 1_7, 1_0, 3_5_3, 1_7_0_2_2, 1_1, 2_1, 6_4_2_1, 9_5_3_1, 1_6_9_4_9, 1_7, 1_0, 1_1_5_0_9, 7_5_3, 1_1, 3_3, 9_5, 2_4_2_1, 7_3_8_5, 9_5_6, 1_4_4_3_1, 2_6_2_6, 2_5, 8_4_2, 7_3_8_5, 4_8_3_6, 2_1, 1_4_2_9, 2_2_7_2, 9_8_5_5, 3_1_2_0, 1_6_1, 2_4_7_3_8, 1_9, 1_3_2_0_3, 6_5_8, 2_1_8, 7_8_7, 2_1, 4_3_0, 1_8_4_8_2, 8_4_7, 2_6_3_7, 9, 4, 3], [5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 3_2_2, 2_2_1_7_8, 2_7, 1_0_6_4, 2_2, 9_5_6, 1_3, 1_1_1_0_1, 1_4_2_9, 5_8_5_4, 2_4_3_1_3, 1_8_9_5_3, 4_0, 4_2_2, 2_4_3_6_6, 6_8, 1_7_5_8, 3_7, 1_0_4_8_3, 1_4_2_5_7, 3_1, 2_0_7, 2_6_3, 2_1, 2_0_3, 3_7_7_3, 2_5, 7_1, 9_7_3_5, 9, 4, 3], [5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 3_2, 2_0_4_9, 3_4_4_2, 1_7, 1_3_8_9_4, 3_3_8_0, 2_3, 9_5, 1_8, 1_7_6_3_4, 2_2_8_8, 9, 4, 3]], """token_type_ids""": [[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2], [3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2], [3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2]], """attention_mask""": [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]]}  # noqa: E501
				# fmt: on

				self.tokenizer_integration_test_util(
				    expected_encoding=lowerCAmelCase__							,  model_name="""xlnet-base-cased"""							,  revision="""c841166438c31ec7ca9a106dee7bb312b73ae511"""							,  )




