



'''simple docstring'''
import argparse
import json
import os

import evaluate
import torch
from datasets import load_dataset
from torch.optim import AdamW
from torch.utils.data import DataLoader
from transformers import AutoModelForSequenceClassification, AutoTokenizer, get_linear_schedule_with_warmup, set_seed

from accelerate import Accelerator, DistributedType
from accelerate.utils.deepspeed import DummyOptim, DummyScheduler


snake_case_ :	int         =  16
snake_case_ :	int         =  32






def A__    (     UpperCAmelCase_  ,  UpperCAmelCase_ = 1_6  ,  UpperCAmelCase_ = "bert-base-cased"							):
						_UpperCamelCase					:		List[Any] 				=   AutoTokenizer.from_pretrained(UpperCAmelCase_							)
						_UpperCamelCase					:		List[Any] 				=   load_dataset('glue'  ,  'mrpc'							)

						def tokenize_function(UpperCAmelCase_							):
												# max_length=None => use the model max length (it's actually the default)
												_UpperCamelCase					:		Dict 				=   tokenizer(examples['sentence1']  ,  examples['sentence2']  ,  truncation=UpperCAmelCase_  ,  max_length=UpperCAmelCase_							)
												return outputs

						# Apply the method we just defined to all the examples in all the splits of the dataset
						_UpperCamelCase					:		int 				=   datasets.map(
						    UpperCAmelCase_  ,  batched=UpperCAmelCase_  ,  remove_columns=['idx', 'sentence1', 'sentence2']  ,  load_from_cache_file=UpperCAmelCase_							)

						# We also rename the 'label' column to 'labels' which is the expected name for labels by the models of the
						# transformers library
						_UpperCamelCase					:		Optional[Any] 				=   tokenized_datasets.rename_column('label'  ,  'labels'							)

						def collate_fn(UpperCAmelCase_							):
												# On TPU it's best to pad everything to the same length or training will be very slow.
												if accelerator.distributed_type == DistributedType.TPU:
																		return tokenizer.pad(UpperCAmelCase_  ,  padding='max_length'  ,  max_length=1_2_8  ,  return_tensors='pt'							)
												return tokenizer.pad(UpperCAmelCase_  ,  padding='longest'  ,  return_tensors='pt'							)

						# Instantiate dataloaders.
						_UpperCamelCase					:		Any 				=   DataLoader(
						    tokenized_datasets['train']  ,  shuffle=UpperCAmelCase_  ,  collate_fn=UpperCAmelCase_  ,  batch_size=UpperCAmelCase_							)
						_UpperCamelCase					:		int 				=   DataLoader(
						    tokenized_datasets['validation']  ,  shuffle=UpperCAmelCase_  ,  collate_fn=UpperCAmelCase_  ,  batch_size=UpperCAmelCase_							)

						return train_dataloader, eval_dataloader






def A__    (     UpperCAmelCase_  ,  UpperCAmelCase_							):
						# Initialize accelerator
						_UpperCamelCase					:		List[Any] 				=   Accelerator()

						# Sample hyper-parameters for learning rate, batch size, seed and a few other HPs
						_UpperCamelCase					:		Union[str, Any] 				=   config['lr']
						_UpperCamelCase					:		Optional[Any] 				=   int(config['num_epochs']							)
						_UpperCamelCase					:		str 				=   int(config['seed']							)
						_UpperCamelCase					:		List[Any] 				=   int(config['batch_size']							)
						_UpperCamelCase					:		int 				=   args.model_name_or_path

						set_seed(UpperCAmelCase_							)
						_UpperCamelCase				,			_UpperCamelCase					:		Dict 				=   get_dataloaders(UpperCAmelCase_  ,  UpperCAmelCase_  ,  UpperCAmelCase_							)

						# Instantiate the model (we build the model here so that the seed also control new weights initialization)
						_UpperCamelCase					:		str 				=   AutoModelForSequenceClassification.from_pretrained(UpperCAmelCase_  ,  return_dict=UpperCAmelCase_							)

						# Instantiate optimizer
						_UpperCamelCase					:		Tuple 				=   (
						    AdamW
						    if accelerator.state.deepspeed_plugin is None
						    or 'optimizer' not in accelerator.state.deepspeed_plugin.deepspeed_config
						    else DummyOptim
						)
						_UpperCamelCase					:		Union[str, Any] 				=   optimizer_cls(params=model.parameters()  ,  lr=UpperCAmelCase_							)

						if accelerator.state.deepspeed_plugin is not None:
												_UpperCamelCase					:		int 				=   accelerator.state.deepspeed_plugin.deepspeed_config[
												    'gradient_accumulation_steps'
												]
						else:
												_UpperCamelCase					:		List[Any] 				=   1
						_UpperCamelCase					:		str 				=   (len(UpperCAmelCase_							) * num_epochs) // gradient_accumulation_steps

						# Instantiate scheduler
						if (
						    accelerator.state.deepspeed_plugin is None
						    or "scheduler" not in accelerator.state.deepspeed_plugin.deepspeed_config
						):
												_UpperCamelCase					:		Union[str, Any] 				=   get_linear_schedule_with_warmup(
												    optimizer=UpperCAmelCase_  ,  num_warmup_steps=0  ,  num_training_steps=UpperCAmelCase_  ,  )
						else:
												_UpperCamelCase					:		str 				=   DummyScheduler(UpperCAmelCase_  ,  total_num_steps=UpperCAmelCase_  ,  warmup_num_steps=0							)

						# Prepare everything
						# There is no specific order to remember, we just need to unpack the objects in the same order we gave them to the
						# prepare method.
						_UpperCamelCase				,			_UpperCamelCase				,			_UpperCamelCase				,			_UpperCamelCase				,			_UpperCamelCase					:		Dict 				=   accelerator.prepare(
						    UpperCAmelCase_  ,  UpperCAmelCase_  ,  UpperCAmelCase_  ,  UpperCAmelCase_  ,  UpperCAmelCase_							)

						# We need to keep track of how many total steps we have iterated over
						_UpperCamelCase					:		str 				=   0
						# We also need to keep track of the stating epoch so files are named properly
						_UpperCamelCase					:		int 				=   0

						# Now we train the model
						_UpperCamelCase					:		Any 				=   evaluate.load('glue'  ,  'mrpc'							)
						_UpperCamelCase					:		Union[str, Any] 				=   0
						_UpperCamelCase					:		str 				=   {}
						for epoch in range(UpperCAmelCase_  ,  UpperCAmelCase_							):
												model.train()
												for step, batch in enumerate(UpperCAmelCase_							):
																		_UpperCamelCase					:		Dict 				=   model(**UpperCAmelCase_							)
																		_UpperCamelCase					:		Dict 				=   outputs.loss
																		_UpperCamelCase					:		List[str] 				=   loss / gradient_accumulation_steps
																		accelerator.backward(UpperCAmelCase_							)
																		if step % gradient_accumulation_steps == 0:
																								optimizer.step()
																								lr_scheduler.step()
																								optimizer.zero_grad()

																		overall_step += 1

												model.eval()
												_UpperCamelCase					:		int 				=   0
												for step, batch in enumerate(UpperCAmelCase_							):
																		# We could avoid this line since we set the accelerator with `device_placement=True`.
																		batch.to(accelerator.device							)
																		with torch.no_grad():
																								_UpperCamelCase					:		Any 				=   model(**UpperCAmelCase_							)
																		_UpperCamelCase					:		int 				=   outputs.logits.argmax(dim=-1							)
																		# It is slightly faster to call this once, than multiple times
																		_UpperCamelCase				,			_UpperCamelCase					:		List[str] 				=   accelerator.gather(
																		    (predictions, batch['labels'])							)  # If we are in a multiprocess environment, the last batch has duplicates
																		if accelerator.use_distributed:
																								if step == len(UpperCAmelCase_							) - 1:
																														_UpperCamelCase					:		Dict 				=   predictions[: len(eval_dataloader.dataset							) - samples_seen]
																														_UpperCamelCase					:		Union[str, Any] 				=   references[: len(eval_dataloader.dataset							) - samples_seen]
																								else:
																														samples_seen += references.shape[0]
																		metric.add_batch(
																		    predictions=UpperCAmelCase_  ,  references=UpperCAmelCase_  ,  )

												_UpperCamelCase					:		Any 				=   metric.compute()
												# Use accelerator.print to print only on the main process.
												accelerator.print(f'epoch {epoch}:'  ,  UpperCAmelCase_							)
												_UpperCamelCase					:		Optional[Any] 				=   eval_metric['accuracy']

												if best_performance < eval_metric["accuracy"]:
																		_UpperCamelCase					:		Dict 				=   eval_metric['accuracy']

						if args.performance_lower_bound is not None:
												assert (
												    args.performance_lower_bound <= best_performance
												), f'Best performance metric {best_performance} is lower than the lower bound {args.performance_lower_bound}'

						accelerator.wait_for_everyone()
						if accelerator.is_main_process:
												with open(os.path.join(args.output_dir  ,  'all_results.json'							)  ,  'w'							) as f:
																		json.dump(UpperCAmelCase_  ,  UpperCAmelCase_							)






def A__    (     ):
						_UpperCamelCase					:		Union[str, Any] 				=   argparse.ArgumentParser(description='Simple example of training script tracking peak GPU memory usage.'							)
						parser.add_argument(
						    '--model_name_or_path'  ,  type=UpperCAmelCase_  ,  default='bert-base-cased'  ,  help='Path to pretrained model or model identifier from huggingface.co/models.'  ,  required=UpperCAmelCase_  ,  )
						parser.add_argument(
						    '--output_dir'  ,  type=UpperCAmelCase_  ,  default='.'  ,  help='Optional save directory where all checkpoint folders will be stored. Default is the current working directory.'  ,  )
						parser.add_argument(
						    '--performance_lower_bound'  ,  type=UpperCAmelCase_  ,  default=UpperCAmelCase_  ,  help='Optional lower bound for the performance metric. If set, the training will throw error when the performance metric drops below this value.'  ,  )
						parser.add_argument(
						    '--num_epochs'  ,  type=UpperCAmelCase_  ,  default=3  ,  help='Number of train epochs.'  ,  )
						_UpperCamelCase					:		List[Any] 				=   parser.parse_args()
						_UpperCamelCase					:		Optional[int] 				=   {'lr': 2E-5, 'num_epochs': args.num_epochs, 'seed': 4_2, 'batch_size': 1_6}
						training_function(UpperCAmelCase_  ,  UpperCAmelCase_							)


if __name__ == "__main__":
					main()









import unittest

from transformers import SPIECE_UNDERLINE
from transformers.models.speechta import SpeechTaTokenizer
from transformers.testing_utils import get_tests_dir, require_sentencepiece, require_tokenizers, slow
from transformers.tokenization_utils import AddedToken

from ...test_tokenization_common import TokenizerTesterMixin


lowerCAmelCase_    =				get_tests_dir('''fixtures/test_sentencepiece_bpe_char.model''')



@require_sentencepiece
@require_tokenizers
class 					__lowerCAmelCase       (			_a,     unittest.TestCase     ):
			lowerCamelCase_     :     Optional[int]       			=      SpeechTaTokenizer
			lowerCamelCase_     :     int       			=      False
			lowerCamelCase_     :     Dict       			=      True






			def 			lowerCamelCase  (self    )     ->   Optional[Any]:


							'''simple docstring'''



							super().setUp()

							# We have a SentencePiece fixture for testing
							snake_case_						:    Tuple				=  SpeechTaTokenizer(__magic_name__    )

							snake_case_						:    Any				=  AddedToken('''<mask>'''					,							lstrip=__magic_name__					,							rstrip=__magic_name__    )
							snake_case_						:    int				=  mask_token
							tokenizer.add_special_tokens({'''mask_token''': mask_token}    )
							tokenizer.add_tokens(['''<ctc_blank>''']    )

							tokenizer.save_pretrained(self.tmpdirname    )






			def 			lowerCamelCase  (self					,							__magic_name__    )     ->   Dict:


							'''simple docstring'''



							snake_case_						:    Dict				=  '''this is a test'''
							snake_case_						:    int				=  '''this is a test'''
							return input_text, output_text






			def 			lowerCamelCase  (self					,							__magic_name__					,							__magic_name__=False					,							__magic_name__=20					,							__magic_name__=5    )     ->   List[Any]:


							'''simple docstring'''



							snake_case_			,							snake_case_						:    int				=  self.get_input_output_texts(__magic_name__    )
							snake_case_						:    Optional[Any]				=  tokenizer.encode(__magic_name__					,							add_special_tokens=__magic_name__    )
							snake_case_						:    Any				=  tokenizer.decode(__magic_name__					,							clean_up_tokenization_spaces=__magic_name__    )
							return text, ids






			def 			lowerCamelCase  (self    )     ->   Optional[int]:


							'''simple docstring'''



							snake_case_						:    List[str]				=  '''<pad>'''
							snake_case_						:    Any				=  1

							self.assertEqual(self.get_tokenizer()._convert_token_to_id(__magic_name__    )					,							__magic_name__    )
							self.assertEqual(self.get_tokenizer()._convert_id_to_token(__magic_name__    )					,							__magic_name__    )






			def 			lowerCamelCase  (self    )     ->   Any:


							'''simple docstring'''



							snake_case_						:    Any				=  list(self.get_tokenizer().get_vocab().keys()    )

							self.assertEqual(vocab_keys[0]					,							'''<s>'''    )
							self.assertEqual(vocab_keys[1]					,							'''<pad>'''    )
							self.assertEqual(vocab_keys[-4]					,							'''œ'''    )
							self.assertEqual(vocab_keys[-2]					,							'''<mask>'''    )
							self.assertEqual(vocab_keys[-1]					,							'''<ctc_blank>'''    )
							self.assertEqual(len(__magic_name__    )					,							81    )






			def 			lowerCamelCase  (self    )     ->   Dict:


							'''simple docstring'''



							self.assertEqual(self.get_tokenizer().vocab_size					,							79    )






			def 			lowerCamelCase  (self    )     ->   Tuple:


							'''simple docstring'''



							snake_case_						:    int				=  self.get_tokenizers(do_lower_case=__magic_name__    )
							for tokenizer in tokenizers:
											with self.subTest(F'''{tokenizer.__class__.__name__}'''    ):
															snake_case_						:    int				=  tokenizer.vocab_size
															snake_case_						:    Optional[Any]				=  len(__magic_name__    )

															self.assertNotEqual(__magic_name__					,							0    )

															# We usually have added tokens from the start in tests because our vocab fixtures are
															# smaller than the original vocabs - let's not assert this
															# self.assertEqual(vocab_size, all_size)

															snake_case_						:    List[Any]				=  ['''aaaaa bbbbbb''', '''cccccccccdddddddd''']
															snake_case_						:    List[Any]				=  tokenizer.add_tokens(__magic_name__    )
															snake_case_						:    Dict				=  tokenizer.vocab_size
															snake_case_						:    Optional[Any]				=  len(__magic_name__    )

															self.assertNotEqual(__magic_name__					,							0    )
															self.assertEqual(__magic_name__					,							__magic_name__    )
															self.assertEqual(__magic_name__					,							len(__magic_name__    )    )
															self.assertEqual(__magic_name__					,							all_size + len(__magic_name__    )    )

															snake_case_						:    Union[str, Any]				=  tokenizer.encode('''aaaaa bbbbbb low cccccccccdddddddd l'''					,							add_special_tokens=__magic_name__    )

															self.assertGreaterEqual(len(__magic_name__    )					,							4    )
															self.assertGreater(tokens[0]					,							tokenizer.vocab_size - 1    )
															self.assertGreater(tokens[-3]					,							tokenizer.vocab_size - 1    )

															snake_case_						:    Union[str, Any]				=  {'''eos_token''': '''>>>>|||<||<<|<<''', '''pad_token''': '''<<<<<|||>|>>>>|>'''}
															snake_case_						:    List[str]				=  tokenizer.add_special_tokens(__magic_name__    )
															snake_case_						:    Dict				=  tokenizer.vocab_size
															snake_case_						:    Dict				=  len(__magic_name__    )

															self.assertNotEqual(__magic_name__					,							0    )
															self.assertEqual(__magic_name__					,							__magic_name__    )
															self.assertEqual(__magic_name__					,							len(__magic_name__    )    )
															self.assertEqual(__magic_name__					,							all_size_a + len(__magic_name__    )    )

															snake_case_						:    Tuple				=  tokenizer.encode(
															    '''>>>>|||<||<<|<< aaaaabbbbbb low cccccccccdddddddd <<<<<|||>|>>>>|> l'''					,							add_special_tokens=__magic_name__    )

															self.assertGreaterEqual(len(__magic_name__    )					,							6    )
															self.assertGreater(tokens[0]					,							tokenizer.vocab_size - 1    )
															self.assertGreater(tokens[0]					,							tokens[1]    )
															self.assertGreater(tokens[-3]					,							tokenizer.vocab_size - 1    )
															self.assertGreater(tokens[-3]					,							tokens[-4]    )
															self.assertEqual(tokens[0]					,							tokenizer.eos_token_id    )
															self.assertEqual(tokens[-3]					,							tokenizer.pad_token_id    )






			def 			lowerCamelCase  (self    )     ->   Union[str, Any]:


							'''simple docstring'''



							pass






			def 			lowerCamelCase  (self    )     ->   List[str]:


							'''simple docstring'''



							pass






			def 			lowerCamelCase  (self    )     ->   int:


							'''simple docstring'''



							snake_case_						:    Dict				=  self.get_tokenizer()

							snake_case_						:    Optional[Any]				=  tokenizer.tokenize('''This is a test'''    )
							# fmt: off
							self.assertListEqual(__magic_name__					,							[SPIECE_UNDERLINE, '''T''', '''h''', '''i''', '''s''', SPIECE_UNDERLINE, '''i''', '''s''', SPIECE_UNDERLINE, '''a''', SPIECE_UNDERLINE, '''t''', '''e''', '''s''', '''t''']    )
							# fmt: on

							self.assertListEqual(
							    tokenizer.convert_tokens_to_ids(__magic_name__    )					,							[4, 32, 11, 10, 12, 4, 10, 12, 4, 7, 4, 6, 5, 12, 6]					,							)

							snake_case_						:    List[Any]				=  tokenizer.tokenize('''I was born in 92000, and this is falsé.'''    )
							self.assertListEqual(
							    __magic_name__					,							[SPIECE_UNDERLINE, '''I''', SPIECE_UNDERLINE, '''w''', '''a''', '''s''', SPIECE_UNDERLINE, '''b''', '''o''', '''r''', '''n''', SPIECE_UNDERLINE, '''i''', '''n''', SPIECE_UNDERLINE, '''92000''', ''',''', SPIECE_UNDERLINE, '''a''', '''n''', '''d''', SPIECE_UNDERLINE, '''t''', '''h''', '''i''', '''s''', SPIECE_UNDERLINE, '''i''', '''s''', SPIECE_UNDERLINE, '''f''', '''a''', '''l''', '''s''', '''é''', '''.''']    )

							snake_case_						:    List[str]				=  tokenizer.convert_tokens_to_ids(__magic_name__    )
							# fmt: off
							self.assertListEqual(__magic_name__					,							[4, 30, 4, 20, 7, 12, 4, 25, 8, 13, 9, 4, 10, 9, 4, 3, 23, 4, 7, 9, 14, 4, 6, 11, 10, 12, 4, 10, 12, 4, 19, 7, 15, 12, 73, 26]    )
							# fmt: on

							snake_case_						:    int				=  tokenizer.convert_ids_to_tokens(__magic_name__    )
							self.assertListEqual(
							    __magic_name__					,							[SPIECE_UNDERLINE, '''I''', SPIECE_UNDERLINE, '''w''', '''a''', '''s''', SPIECE_UNDERLINE, '''b''', '''o''', '''r''', '''n''', SPIECE_UNDERLINE, '''i''', '''n''', SPIECE_UNDERLINE, '''<unk>''', ''',''', SPIECE_UNDERLINE, '''a''', '''n''', '''d''', SPIECE_UNDERLINE, '''t''', '''h''', '''i''', '''s''', SPIECE_UNDERLINE, '''i''', '''s''', SPIECE_UNDERLINE, '''f''', '''a''', '''l''', '''s''', '''é''', '''.''']    )






			@slow
			def 			lowerCamelCase  (self    )     ->   Tuple:


							'''simple docstring'''



							snake_case_						:    Tuple				=  [
							    '''Transformers (formerly known as pytorch-transformers and pytorch-pretrained-bert) provides '''
							    '''general-purpose architectures (BERT, GPT, RoBERTa, XLM, DistilBert, XLNet...) for Natural '''
							    '''Language Understanding (NLU) and Natural Language Generation (NLG) with over thirty-two pretrained '''
							    '''models in one hundred plus languages and deep interoperability between Jax, PyTorch and TensorFlow.''',
							    '''BERT is designed to pre-train deep bidirectional representations from unlabeled text by jointly '''
							    '''conditioning on both left and right context in all layers.''',
							    '''The quick brown fox jumps over the lazy dog.''',
							]

							# fmt: off
							snake_case_						:    List[Any]				=  {
							    '''input_ids''': [
							        [4, 32, 13, 7, 9, 12, 19, 8, 13, 18, 5, 13, 12, 4, 64, 19, 8, 13, 18, 5, 13, 15, 22, 4, 28, 9, 8, 20, 9, 4, 7, 12, 4, 24, 22, 6, 8, 13, 17, 11, 39, 6, 13, 7, 9, 12, 19, 8, 13, 18, 5, 13, 12, 4, 7, 9, 14, 4, 24, 22, 6, 8, 13, 17, 11, 39, 24, 13, 5, 6, 13, 7, 10, 9, 5, 14, 39, 25, 5, 13, 6, 63, 4, 24, 13, 8, 27, 10, 14, 5, 12, 4, 21, 5, 9, 5, 13, 7, 15, 39, 24, 16, 13, 24, 8, 12, 5, 4, 7, 13, 17, 11, 10, 6, 5, 17, 6, 16, 13, 5, 12, 4, 64, 40, 47, 54, 32, 23, 4, 53, 49, 32, 23, 4, 54, 8, 40, 47, 54, 32, 7, 23, 4, 69, 52, 43, 23, 4, 51, 10, 12, 6, 10, 15, 40, 5, 13, 6, 23, 4, 69, 52, 48, 5, 6, 26, 26, 26, 63, 4, 19, 8, 13, 4, 48, 7, 6, 16, 13, 7, 15, 4, 52, 7, 9, 21, 16, 7, 21, 5, 4, 61, 9, 14, 5, 13, 12, 6, 7, 9, 14, 10, 9, 21, 4, 64, 48, 52, 61, 63, 4, 7, 9, 14, 4, 48, 7, 6, 16, 13, 7, 15, 4, 52, 7, 9, 21, 16, 7, 21, 5, 4, 53, 5, 9, 5, 13, 7, 6, 10, 8, 9, 4, 64, 48, 52, 53, 63, 4, 20, 10, 6, 11, 4, 8, 27, 5, 13, 4, 6, 11, 10, 13, 6, 22, 39, 6, 20, 8, 4, 24, 13, 5, 6, 13, 7, 10, 9, 5, 14, 4, 18, 8, 14, 5, 15, 12, 4, 10, 9, 4, 8, 9, 5, 4, 11, 16, 9, 14, 13, 5, 14, 4, 24, 15, 16, 12, 4, 15, 7, 9, 21, 16, 7, 21, 5, 12, 4, 7, 9, 14, 4, 14, 5, 5, 24, 4, 10, 9, 6, 5, 13, 8, 24, 5, 13, 7, 25, 10, 15, 10, 6, 22, 4, 25, 5, 6, 20, 5, 5, 9, 4, 58, 7, 37, 23, 4, 49, 22, 32, 8, 13, 17, 11, 4, 7, 9, 14, 4, 32, 5, 9, 12, 8, 13, 55, 15, 8, 20, 26, 2],
							        [4, 40, 47, 54, 32, 4, 10, 12, 4, 14, 5, 12, 10, 21, 9, 5, 14, 4, 6, 8, 4, 24, 13, 5, 39, 6, 13, 7, 10, 9, 4, 14, 5, 5, 24, 4, 25, 10, 14, 10, 13, 5, 17, 6, 10, 8, 9, 7, 15, 4, 13, 5, 24, 13, 5, 12, 5, 9, 6, 7, 6, 10, 8, 9, 12, 4, 19, 13, 8, 18, 4, 16, 9, 15, 7, 25, 5, 15, 5, 14, 4, 6, 5, 37, 6, 4, 25, 22, 4, 46, 8, 10, 9, 6, 15, 22, 4, 17, 8, 9, 14, 10, 6, 10, 8, 9, 10, 9, 21, 4, 8, 9, 4, 25, 8, 6, 11, 4, 15, 5, 19, 6, 4, 7, 9, 14, 4, 13, 10, 21, 11, 6, 4, 17, 8, 9, 6, 5, 37, 6, 4, 10, 9, 4, 7, 15, 15, 4, 15, 7, 22, 5, 13, 12, 26, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1],
							        [4, 32, 11, 5, 4, 45, 16, 10, 17, 28, 4, 25, 13, 8, 20, 9, 4, 19, 8, 37, 4, 46, 16, 18, 24, 12, 4, 8, 27, 5, 13, 4, 6, 11, 5, 4, 15, 7, 57, 22, 4, 14, 8, 21, 26, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1],
							    ],
							    '''attention_mask''': [
							        [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1],
							        [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
							        [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
							    ]
							}
							# fmt: on

							self.tokenizer_integration_test_util(
							    expected_encoding=__magic_name__					,							model_name='''microsoft/speecht5_asr'''					,							revision='''c5ef64c71905caeccde0e4462ef3f9077224c524'''					,							sequences=__magic_name__					,							)
