





import unittest

from transformers import SPIECE_UNDERLINE
from transformers.models.speechta import SpeechTaTokenizer
from transformers.testing_utils import get_tests_dir, require_sentencepiece, require_tokenizers, slow
from transformers.tokenization_utils import AddedToken

from ...test_tokenization_common import TokenizerTesterMixin


snake_case								=							get_tests_dir("""fixtures/test_sentencepiece_bpe_char.model""")





@require_sentencepiece
@require_tokenizers
class 					A_					(   UpperCAmelCase       ,						unittest.TestCase						):







       """simple docstring"""





       SCREAMING_SNAKE_CASE_   :			Optional[Any]  			=	SpeechTaTokenizer
       SCREAMING_SNAKE_CASE_   :			int  			=	False
       SCREAMING_SNAKE_CASE_   :			List[Any]  			=	True







       def  __UpperCAmelCase			(     self						:						Any	)    ->       Optional[int]:
         super().setUp()

         # We have a SentencePiece fixture for testing
         _lowercase						    =			SpeechTaTokenizer(__A	)

         _lowercase						    =			AddedToken('<mask>'    ,lstrip=__A    ,rstrip=__A	)
         _lowercase						    =			mask_token
         tokenizer.add_special_tokens({'mask_token': mask_token}	)
         tokenizer.add_tokens(['<ctc_blank>']	)

         tokenizer.save_pretrained(self.tmpdirname	)







       def  __UpperCAmelCase			(     self						:						Dict    ,__A						:						Any	)    ->       Any:
         _lowercase						    =			'this is a test'
         _lowercase						    =			'this is a test'
         return input_text, output_text







       def  __UpperCAmelCase			(     self						:						int    ,__A						:						Optional[Any]    ,__A						:						Dict=False    ,__A						:						Tuple=20    ,__A						:						List[Any]=5	)    ->       int:
         _lowercase    ,		_lowercase						    =			self.get_input_output_texts(__A	)
         _lowercase						    =			tokenizer.encode(__A    ,add_special_tokens=__A	)
         _lowercase						    =			tokenizer.decode(__A    ,clean_up_tokenization_spaces=__A	)
         return text, ids







       def  __UpperCAmelCase			(     self						:						Tuple	)    ->       Any:
         _lowercase						    =			'<pad>'
         _lowercase						    =			1

         self.assertEqual(self.get_tokenizer()._convert_token_to_id(__A	)    ,__A	)
         self.assertEqual(self.get_tokenizer()._convert_id_to_token(__A	)    ,__A	)







       def  __UpperCAmelCase			(     self						:						Dict	)    ->       Tuple:
         _lowercase						    =			list(self.get_tokenizer().get_vocab().keys()	)

         self.assertEqual(vocab_keys[0]    ,'<s>'	)
         self.assertEqual(vocab_keys[1]    ,'<pad>'	)
         self.assertEqual(vocab_keys[-4]    ,'œ'	)
         self.assertEqual(vocab_keys[-2]    ,'<mask>'	)
         self.assertEqual(vocab_keys[-1]    ,'<ctc_blank>'	)
         self.assertEqual(len(__A	)    ,81	)







       def  __UpperCAmelCase			(     self						:						Union[str, Any]	)    ->       Any:
         self.assertEqual(self.get_tokenizer().vocab_size    ,79	)







       def  __UpperCAmelCase			(     self						:						Tuple	)    ->       Dict:
         _lowercase						    =			self.get_tokenizers(do_lower_case=__A	)
         for tokenizer in tokenizers:
           with self.subTest(F"""{tokenizer.__class__.__name__}"""	):
             _lowercase						    =			tokenizer.vocab_size
             _lowercase						    =			len(__A	)

             self.assertNotEqual(__A    ,0	)

             # We usually have added tokens from the start in tests because our vocab fixtures are
             # smaller than the original vocabs - let's not assert this
             # self.assertEqual(vocab_size, all_size)

             _lowercase						    =			['aaaaa bbbbbb', 'cccccccccdddddddd']
             _lowercase						    =			tokenizer.add_tokens(__A	)
             _lowercase						    =			tokenizer.vocab_size
             _lowercase						    =			len(__A	)

             self.assertNotEqual(__A    ,0	)
             self.assertEqual(__A    ,__A	)
             self.assertEqual(__A    ,len(__A	)	)
             self.assertEqual(__A    ,all_size + len(__A	)	)

             _lowercase						    =			tokenizer.encode('aaaaa bbbbbb low cccccccccdddddddd l'    ,add_special_tokens=__A	)

             self.assertGreaterEqual(len(__A	)    ,4	)
             self.assertGreater(tokens[0]    ,tokenizer.vocab_size - 1	)
             self.assertGreater(tokens[-3]    ,tokenizer.vocab_size - 1	)

             _lowercase						    =			{'eos_token': '>>>>|||<||<<|<<', 'pad_token': '<<<<<|||>|>>>>|>'}
             _lowercase						    =			tokenizer.add_special_tokens(__A	)
             _lowercase						    =			tokenizer.vocab_size
             _lowercase						    =			len(__A	)

             self.assertNotEqual(__A    ,0	)
             self.assertEqual(__A    ,__A	)
             self.assertEqual(__A    ,len(__A	)	)
             self.assertEqual(__A    ,all_size_a + len(__A	)	)

             _lowercase						    =			tokenizer.encode(
                 '>>>>|||<||<<|<< aaaaabbbbbb low cccccccccdddddddd <<<<<|||>|>>>>|> l'    ,add_special_tokens=__A	)

             self.assertGreaterEqual(len(__A	)    ,6	)
             self.assertGreater(tokens[0]    ,tokenizer.vocab_size - 1	)
             self.assertGreater(tokens[0]    ,tokens[1]	)
             self.assertGreater(tokens[-3]    ,tokenizer.vocab_size - 1	)
             self.assertGreater(tokens[-3]    ,tokens[-4]	)
             self.assertEqual(tokens[0]    ,tokenizer.eos_token_id	)
             self.assertEqual(tokens[-3]    ,tokenizer.pad_token_id	)







       def  __UpperCAmelCase			(     self						:						Dict	)    ->       int:
         pass







       def  __UpperCAmelCase			(     self						:						List[str]	)    ->       int:
         pass







       def  __UpperCAmelCase			(     self						:						Any	)    ->       Any:
         _lowercase						    =			self.get_tokenizer()

         _lowercase						    =			tokenizer.tokenize('This is a test'	)
         # fmt: off
         self.assertListEqual(__A    ,[SPIECE_UNDERLINE, 'T', 'h', 'i', 's', SPIECE_UNDERLINE, 'i', 's', SPIECE_UNDERLINE, 'a', SPIECE_UNDERLINE, 't', 'e', 's', 't']	)
         # fmt: on

         self.assertListEqual(
             tokenizer.convert_tokens_to_ids(__A	)    ,[4, 32, 11, 10, 12, 4, 10, 12, 4, 7, 4, 6, 5, 12, 6]    ,)

         _lowercase						    =			tokenizer.tokenize('I was born in 92000, and this is falsé.'	)
         self.assertListEqual(
             __A    ,[SPIECE_UNDERLINE, 'I', SPIECE_UNDERLINE, 'w', 'a', 's', SPIECE_UNDERLINE, 'b', 'o', 'r', 'n', SPIECE_UNDERLINE, 'i', 'n', SPIECE_UNDERLINE, '92000', ',', SPIECE_UNDERLINE, 'a', 'n', 'd', SPIECE_UNDERLINE, 't', 'h', 'i', 's', SPIECE_UNDERLINE, 'i', 's', SPIECE_UNDERLINE, 'f', 'a', 'l', 's', 'é', '.']	)

         _lowercase						    =			tokenizer.convert_tokens_to_ids(__A	)
         # fmt: off
         self.assertListEqual(__A    ,[4, 30, 4, 20, 7, 12, 4, 25, 8, 13, 9, 4, 10, 9, 4, 3, 23, 4, 7, 9, 14, 4, 6, 11, 10, 12, 4, 10, 12, 4, 19, 7, 15, 12, 73, 26]	)
         # fmt: on

         _lowercase						    =			tokenizer.convert_ids_to_tokens(__A	)
         self.assertListEqual(
             __A    ,[SPIECE_UNDERLINE, 'I', SPIECE_UNDERLINE, 'w', 'a', 's', SPIECE_UNDERLINE, 'b', 'o', 'r', 'n', SPIECE_UNDERLINE, 'i', 'n', SPIECE_UNDERLINE, '<unk>', ',', SPIECE_UNDERLINE, 'a', 'n', 'd', SPIECE_UNDERLINE, 't', 'h', 'i', 's', SPIECE_UNDERLINE, 'i', 's', SPIECE_UNDERLINE, 'f', 'a', 'l', 's', 'é', '.']	)







       @slow
       def  __UpperCAmelCase			(     self						:						Tuple	)    ->       Optional[int]:
         # Use custom sequence because this tokenizer does not handle numbers.
         _lowercase						    =			[
             'Transformers (formerly known as pytorch-transformers and pytorch-pretrained-bert) provides '
             'general-purpose architectures (BERT, GPT, RoBERTa, XLM, DistilBert, XLNet...) for Natural '
             'Language Understanding (NLU) and Natural Language Generation (NLG) with over thirty-two pretrained '
             'models in one hundred plus languages and deep interoperability between Jax, PyTorch and TensorFlow.',
             'BERT is designed to pre-train deep bidirectional representations from unlabeled text by jointly '
             'conditioning on both left and right context in all layers.',
             'The quick brown fox jumps over the lazy dog.',
         ]

         # fmt: off
         _lowercase						    =			{
             'input_ids': [
                 [4, 32, 13, 7, 9, 12, 19, 8, 13, 18, 5, 13, 12, 4, 64, 19, 8, 13, 18, 5, 13, 15, 22, 4, 28, 9, 8, 20, 9, 4, 7, 12, 4, 24, 22, 6, 8, 13, 17, 11, 39, 6, 13, 7, 9, 12, 19, 8, 13, 18, 5, 13, 12, 4, 7, 9, 14, 4, 24, 22, 6, 8, 13, 17, 11, 39, 24, 13, 5, 6, 13, 7, 10, 9, 5, 14, 39, 25, 5, 13, 6, 63, 4, 24, 13, 8, 27, 10, 14, 5, 12, 4, 21, 5, 9, 5, 13, 7, 15, 39, 24, 16, 13, 24, 8, 12, 5, 4, 7, 13, 17, 11, 10, 6, 5, 17, 6, 16, 13, 5, 12, 4, 64, 40, 47, 54, 32, 23, 4, 53, 49, 32, 23, 4, 54, 8, 40, 47, 54, 32, 7, 23, 4, 69, 52, 43, 23, 4, 51, 10, 12, 6, 10, 15, 40, 5, 13, 6, 23, 4, 69, 52, 48, 5, 6, 26, 26, 26, 63, 4, 19, 8, 13, 4, 48, 7, 6, 16, 13, 7, 15, 4, 52, 7, 9, 21, 16, 7, 21, 5, 4, 61, 9, 14, 5, 13, 12, 6, 7, 9, 14, 10, 9, 21, 4, 64, 48, 52, 61, 63, 4, 7, 9, 14, 4, 48, 7, 6, 16, 13, 7, 15, 4, 52, 7, 9, 21, 16, 7, 21, 5, 4, 53, 5, 9, 5, 13, 7, 6, 10, 8, 9, 4, 64, 48, 52, 53, 63, 4, 20, 10, 6, 11, 4, 8, 27, 5, 13, 4, 6, 11, 10, 13, 6, 22, 39, 6, 20, 8, 4, 24, 13, 5, 6, 13, 7, 10, 9, 5, 14, 4, 18, 8, 14, 5, 15, 12, 4, 10, 9, 4, 8, 9, 5, 4, 11, 16, 9, 14, 13, 5, 14, 4, 24, 15, 16, 12, 4, 15, 7, 9, 21, 16, 7, 21, 5, 12, 4, 7, 9, 14, 4, 14, 5, 5, 24, 4, 10, 9, 6, 5, 13, 8, 24, 5, 13, 7, 25, 10, 15, 10, 6, 22, 4, 25, 5, 6, 20, 5, 5, 9, 4, 58, 7, 37, 23, 4, 49, 22, 32, 8, 13, 17, 11, 4, 7, 9, 14, 4, 32, 5, 9, 12, 8, 13, 55, 15, 8, 20, 26, 2],
                 [4, 40, 47, 54, 32, 4, 10, 12, 4, 14, 5, 12, 10, 21, 9, 5, 14, 4, 6, 8, 4, 24, 13, 5, 39, 6, 13, 7, 10, 9, 4, 14, 5, 5, 24, 4, 25, 10, 14, 10, 13, 5, 17, 6, 10, 8, 9, 7, 15, 4, 13, 5, 24, 13, 5, 12, 5, 9, 6, 7, 6, 10, 8, 9, 12, 4, 19, 13, 8, 18, 4, 16, 9, 15, 7, 25, 5, 15, 5, 14, 4, 6, 5, 37, 6, 4, 25, 22, 4, 46, 8, 10, 9, 6, 15, 22, 4, 17, 8, 9, 14, 10, 6, 10, 8, 9, 10, 9, 21, 4, 8, 9, 4, 25, 8, 6, 11, 4, 15, 5, 19, 6, 4, 7, 9, 14, 4, 13, 10, 21, 11, 6, 4, 17, 8, 9, 6, 5, 37, 6, 4, 10, 9, 4, 7, 15, 15, 4, 15, 7, 22, 5, 13, 12, 26, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1],
                 [4, 32, 11, 5, 4, 45, 16, 10, 17, 28, 4, 25, 13, 8, 20, 9, 4, 19, 8, 37, 4, 46, 16, 18, 24, 12, 4, 8, 27, 5, 13, 4, 6, 11, 5, 4, 15, 7, 57, 22, 4, 14, 8, 21, 26, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1],
             ],
             'attention_mask': [
                 [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1],
                 [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
                 [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
             ]
         }
         # fmt: on

         self.tokenizer_integration_test_util(
             expected_encoding=__A    ,model_name='microsoft/speecht5_asr'    ,revision='c5ef64c71905caeccde0e4462ef3f9077224c524'    ,sequences=__A    ,)
import tempfile
import unittest
from pathlib import Path
from shutil import copyfile

from transformers import BatchEncoding, MarianTokenizer
from transformers.testing_utils import get_tests_dir, require_sentencepiece, slow
from transformers.utils import is_sentencepiece_available, is_tf_available, is_torch_available


if is_sentencepiece_available():
 from transformers.models.marian.tokenization_marian import VOCAB_FILES_NAMES, save_json

from ...test_tokenization_common import TokenizerTesterMixin


a  :      List[Any]            =    get_tests_dir("""fixtures/test_sentencepiece.model""")

a  :      Union[str, Any]            =    {"""target_lang""": """fi""", """source_lang""": """en"""}
a  :      Optional[int]            =    """>>zh<<"""
a  :      Optional[int]            =    """Helsinki-NLP/"""

if is_torch_available():
 a  :      str            =    """pt"""
elif is_tf_available():
 a  :      List[Any]            =    """tf"""
else:
 a  :      str            =    """jax"""

@require_sentencepiece
class 					_lowercase    (     UpperCAmelCase__ ,  unittest.TestCase			):






    '''simple docstring'''
    SCREAMING_SNAKE_CASE:				str          =    MarianTokenizer
    SCREAMING_SNAKE_CASE:				Tuple          =    False
    SCREAMING_SNAKE_CASE:				Optional[int]          =    True




    def      _a    (    self ):
       super().setUp()
       lowerCAmelCase_:							Optional[Any]       =	["</s>", "<unk>", "▁This", "▁is", "▁a", "▁t", "est", "\u0120", "<pad>"]
       lowerCAmelCase_:							Optional[Any]       =	dict(zip(lowerCamelCase__       ,						range(len(lowerCamelCase__ ) ) ) )
       lowerCAmelCase_:							Any       =	Path(self.tmpdirname )
       save_json(lowerCamelCase__       ,						save_dir / VOCAB_FILES_NAMES["vocab"] )
       save_json(lowerCamelCase__       ,						save_dir / VOCAB_FILES_NAMES["tokenizer_config_file"] )
       if not (save_dir / VOCAB_FILES_NAMES["source_spm"]).exists():
          copyfile(lowerCamelCase__       ,						save_dir / VOCAB_FILES_NAMES["source_spm"] )
          copyfile(lowerCamelCase__       ,						save_dir / VOCAB_FILES_NAMES["target_spm"] )

       lowerCAmelCase_:							Any       =	MarianTokenizer.from_pretrained(self.tmpdirname )
       tokenizer.save_pretrained(self.tmpdirname )




    def      _a    (    self       ,						**lowerCamelCase__ ):
       return MarianTokenizer.from_pretrained(self.tmpdirname       ,						**lowerCamelCase__ )




    def      _a    (    self       ,						lowerCamelCase__ ):
       return (
           "This is a test",
           "This is a test",
       )




    def      _a    (    self ):
       lowerCAmelCase_:							Optional[Any]       =	"</s>"
       lowerCAmelCase_:							Tuple       =	0

       self.assertEqual(self.get_tokenizer()._convert_token_to_id(lowerCamelCase__ )       ,						lowerCamelCase__ )
       self.assertEqual(self.get_tokenizer()._convert_id_to_token(lowerCamelCase__ )       ,						lowerCamelCase__ )




    def      _a    (    self ):
       lowerCAmelCase_:							Dict       =	list(self.get_tokenizer().get_vocab().keys() )

       self.assertEqual(vocab_keys[0]       ,						"</s>" )
       self.assertEqual(vocab_keys[1]       ,						"<unk>" )
       self.assertEqual(vocab_keys[-1]       ,						"<pad>" )
       self.assertEqual(len(lowerCamelCase__ )       ,						9 )




    def      _a    (    self ):
       self.assertEqual(self.get_tokenizer().vocab_size       ,						9 )




    def      _a    (    self ):
       lowerCAmelCase_:							Union[str, Any]       =	MarianTokenizer.from_pretrained(F'''{ORG_NAME}opus-mt-en-de''' )
       lowerCAmelCase_:							List[str]       =	en_de_tokenizer(["I am a small frog"]       ,						return_tensors=lowerCamelCase__ )
       self.assertIsInstance(lowerCamelCase__       ,						lowerCamelCase__ )
       lowerCAmelCase_:							List[Any]       =	[38, 121, 14, 697, 38_848, 0]
       self.assertListEqual(lowerCamelCase__       ,						batch.input_ids[0] )

       lowerCAmelCase_:							int       =	tempfile.mkdtemp()
       en_de_tokenizer.save_pretrained(lowerCamelCase__ )
       lowerCAmelCase_:							List[Any]       =	[x.name for x in Path(lowerCamelCase__ ).glob("*" )]
       self.assertIn("source.spm"       ,						lowerCamelCase__ )
       MarianTokenizer.from_pretrained(lowerCamelCase__ )




    def      _a    (    self ):
       lowerCAmelCase_:							Union[str, Any]       =	self.get_tokenizer()

       lowerCAmelCase_:							Tuple       =	tok(
           ["I am a small frog" * 1_000, "I am a small frog"]       ,						padding=lowerCamelCase__       ,						truncation=lowerCamelCase__       ,						return_tensors=lowerCamelCase__ )
       self.assertIsInstance(lowerCamelCase__       ,						lowerCamelCase__ )
       self.assertEqual(batch.input_ids.shape       ,						(2, 512) )




    def      _a    (    self ):
       lowerCAmelCase_:							str       =	self.get_tokenizer()
       lowerCAmelCase_:							List[Any]       =	tok(["I am a tiny frog", "I am a small frog"]       ,						padding=lowerCamelCase__       ,						return_tensors=lowerCamelCase__ )
       self.assertIsInstance(lowerCamelCase__       ,						lowerCamelCase__ )
       self.assertEqual(batch_smaller.input_ids.shape       ,						(2, 10) )




    @slow
    def      _a    (    self ):
       # fmt: off
       lowerCAmelCase_:							int       =	{"input_ids": [[43_495, 462, 20, 42_164, 1_369, 52, 464, 132, 1_703, 492, 13, 7_491, 38_999, 6, 8, 464, 132, 1_703, 492, 13, 4_669, 37_867, 13, 7_525, 27, 1_593, 988, 13, 33_972, 7_029, 6, 20, 8_251, 383, 2, 270, 5_866, 3_788, 2, 2_353, 8_251, 12_338, 2, 13_958, 387, 2, 3_629, 6_953, 188, 2_900, 2, 13_958, 8_011, 11_501, 23, 8_460, 4_073, 34_009, 20, 435, 11_439, 27, 8, 8_460, 4_073, 6_004, 20, 9_988, 375, 27, 33, 266, 1_945, 1_076, 1_350, 37_867, 3_288, 5, 577, 1_076, 4_374, 8, 5_082, 5, 26_453, 257, 556, 403, 2, 242, 132, 383, 316, 492, 8, 10_767, 6, 316, 304, 4_239, 3, 0], [148, 15_722, 19, 1_839, 12, 1_350, 13, 22_327, 5_082, 5_418, 47_567, 35_938, 59, 318, 19_552, 108, 2_183, 54, 14_976, 4_835, 32, 547, 1_114, 8, 315, 2_417, 5, 92, 19_088, 3, 0, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100], [36, 6_395, 12_570, 39_147, 11_597, 6, 266, 4, 45_405, 7_296, 3, 0, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100]], "attention_mask": [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]]}  # noqa: E501
       # fmt: on

       self.tokenizer_integration_test_util(
           expected_encoding=lowerCamelCase__       ,						model_name="Helsinki-NLP/opus-mt-en-de"       ,						revision="1a8c2263da11e68e50938f97e10cd57820bd504c"       ,						decode_kwargs={"use_source_tokenizer": True}       ,						)




    def      _a    (    self ):
       lowerCAmelCase_:							Dict       =	MarianTokenizer.from_pretrained("hf-internal-testing/test-marian-two-vocabs" )

       lowerCAmelCase_:							Any       =	"Tämä on testi"
       lowerCAmelCase_:							Optional[int]       =	"This is a test"

       lowerCAmelCase_:							List[str]       =	[76, 7, 2_047, 2]
       lowerCAmelCase_:							str       =	[69, 12, 11, 940, 2]

       lowerCAmelCase_:							Dict       =	tokenizer(lowerCamelCase__ ).input_ids
       self.assertListEqual(lowerCamelCase__       ,						lowerCamelCase__ )

       lowerCAmelCase_:							List[Any]       =	tokenizer(text_target=lowerCamelCase__ ).input_ids
       self.assertListEqual(lowerCamelCase__       ,						lowerCamelCase__ )

       lowerCAmelCase_:							Any       =	tokenizer.decode(lowerCamelCase__       ,						skip_special_tokens=lowerCamelCase__ )
       self.assertEqual(lowerCamelCase__       ,						lowerCamelCase__ )