


'''simple docstring'''







import unittest

from transformers import SPIECE_UNDERLINE
from transformers.models.speechta import SpeechTaTokenizer
from transformers.testing_utils import get_tests_dir, require_sentencepiece, require_tokenizers, slow
from transformers.tokenization_utils import AddedToken

from ...test_tokenization_common import TokenizerTesterMixin


A__:    Optional[int]					    =       get_tests_dir('''fixtures/test_sentencepiece_bpe_char.model''')







@require_sentencepiece
@require_tokenizers
class A__       (						UpperCAmelCase__ ,     unittest.TestCase      ):
      __UpperCamelCase       :  Any	       =							SpeechTaTokenizer
      __UpperCamelCase       :  List[str]	       =							False
      __UpperCamelCase       :  Optional[int]	       =							True






      def __UpperCAmelCase       ( self			:Union[str, Any]	)       ->					Tuple:




             '''simple docstring'''
             super().setUp()

             # We have a SentencePiece fixture for testing
             _a    : Optional[Any]     					=SpeechTaTokenizer(SCREAMING_SNAKE_CASE	)

             _a    : List[str]     					=AddedToken("""<mask>"""     ,						lstrip=SCREAMING_SNAKE_CASE     ,						rstrip=SCREAMING_SNAKE_CASE	)
             _a    : str     					=mask_token
             tokenizer.add_special_tokens({"""mask_token""": mask_token}	)
             tokenizer.add_tokens(["""<ctc_blank>"""]	)

             tokenizer.save_pretrained(self.tmpdirname	)






      def __UpperCAmelCase       ( self			:Optional[int]     ,						SCREAMING_SNAKE_CASE			:Any	)       ->					Optional[Any]:




             '''simple docstring'''
             _a    : Tuple     					="""this is a test"""
             _a    : List[str]     					="""this is a test"""
             return input_text, output_text






      def __UpperCAmelCase       ( self			:str     ,						SCREAMING_SNAKE_CASE			:Tuple     ,						SCREAMING_SNAKE_CASE			:Dict=False     ,						SCREAMING_SNAKE_CASE			:Tuple=2_0     ,						SCREAMING_SNAKE_CASE			:Any=5	)       ->					Any:




             '''simple docstring'''
             _a					,    _a    : Any     					=self.get_input_output_texts(SCREAMING_SNAKE_CASE	)
             _a    : str     					=tokenizer.encode(SCREAMING_SNAKE_CASE     ,						add_special_tokens=SCREAMING_SNAKE_CASE	)
             _a    : Tuple     					=tokenizer.decode(SCREAMING_SNAKE_CASE     ,						clean_up_tokenization_spaces=SCREAMING_SNAKE_CASE	)
             return text, ids






      def __UpperCAmelCase       ( self			:List[Any]	)       ->					str:




             '''simple docstring'''
             _a    : Tuple     					="""<pad>"""
             _a    : List[Any]     					=1

             self.assertEqual(self.get_tokenizer()._convert_token_to_id(SCREAMING_SNAKE_CASE	)     ,						SCREAMING_SNAKE_CASE	)
             self.assertEqual(self.get_tokenizer()._convert_id_to_token(SCREAMING_SNAKE_CASE	)     ,						SCREAMING_SNAKE_CASE	)






      def __UpperCAmelCase       ( self			:int	)       ->					Optional[Any]:




             '''simple docstring'''
             _a    : Tuple     					=list(self.get_tokenizer().get_vocab().keys()	)

             self.assertEqual(vocab_keys[0]     ,						"""<s>"""	)
             self.assertEqual(vocab_keys[1]     ,						"""<pad>"""	)
             self.assertEqual(vocab_keys[-4]     ,						"""œ"""	)
             self.assertEqual(vocab_keys[-2]     ,						"""<mask>"""	)
             self.assertEqual(vocab_keys[-1]     ,						"""<ctc_blank>"""	)
             self.assertEqual(len(SCREAMING_SNAKE_CASE	)     ,						8_1	)






      def __UpperCAmelCase       ( self			:List[Any]	)       ->					int:




             '''simple docstring'''
             self.assertEqual(self.get_tokenizer().vocab_size     ,						7_9	)






      def __UpperCAmelCase       ( self			:List[str]	)       ->					Optional[int]:




             '''simple docstring'''
             _a    : int     					=self.get_tokenizers(do_lower_case=SCREAMING_SNAKE_CASE	)
             for tokenizer in tokenizers:
                    with self.subTest(f"{tokenizer.__class__.__name__}"	):
                           _a    : List[str]     					=tokenizer.vocab_size
                           _a    : Dict     					=len(SCREAMING_SNAKE_CASE	)

                           self.assertNotEqual(SCREAMING_SNAKE_CASE     ,						0	)

                           # We usually have added tokens from the start in tests because our vocab fixtures are
                           # smaller than the original vocabs - let's not assert this
                           # self.assertEqual(vocab_size, all_size)

                           _a    : str     					=["""aaaaa bbbbbb""", """cccccccccdddddddd"""]
                           _a    : int     					=tokenizer.add_tokens(SCREAMING_SNAKE_CASE	)
                           _a    : Optional[int]     					=tokenizer.vocab_size
                           _a    : Optional[Any]     					=len(SCREAMING_SNAKE_CASE	)

                           self.assertNotEqual(SCREAMING_SNAKE_CASE     ,						0	)
                           self.assertEqual(SCREAMING_SNAKE_CASE     ,						SCREAMING_SNAKE_CASE	)
                           self.assertEqual(SCREAMING_SNAKE_CASE     ,						len(SCREAMING_SNAKE_CASE	)	)
                           self.assertEqual(SCREAMING_SNAKE_CASE     ,						all_size + len(SCREAMING_SNAKE_CASE	)	)

                           _a    : Optional[Any]     					=tokenizer.encode("""aaaaa bbbbbb low cccccccccdddddddd l"""     ,						add_special_tokens=SCREAMING_SNAKE_CASE	)

                           self.assertGreaterEqual(len(SCREAMING_SNAKE_CASE	)     ,						4	)
                           self.assertGreater(tokens[0]     ,						tokenizer.vocab_size - 1	)
                           self.assertGreater(tokens[-3]     ,						tokenizer.vocab_size - 1	)

                           _a    : int     					={"""eos_token""": """>>>>|||<||<<|<<""", """pad_token""": """<<<<<|||>|>>>>|>"""}
                           _a    : Union[str, Any]     					=tokenizer.add_special_tokens(SCREAMING_SNAKE_CASE	)
                           _a    : Any     					=tokenizer.vocab_size
                           _a    : Optional[Any]     					=len(SCREAMING_SNAKE_CASE	)

                           self.assertNotEqual(SCREAMING_SNAKE_CASE     ,						0	)
                           self.assertEqual(SCREAMING_SNAKE_CASE     ,						SCREAMING_SNAKE_CASE	)
                           self.assertEqual(SCREAMING_SNAKE_CASE     ,						len(SCREAMING_SNAKE_CASE	)	)
                           self.assertEqual(SCREAMING_SNAKE_CASE     ,						all_size_a + len(SCREAMING_SNAKE_CASE	)	)

                           _a    : Union[str, Any]     					=tokenizer.encode(
                               """>>>>|||<||<<|<< aaaaabbbbbb low cccccccccdddddddd <<<<<|||>|>>>>|> l"""     ,						add_special_tokens=SCREAMING_SNAKE_CASE	)

                           self.assertGreaterEqual(len(SCREAMING_SNAKE_CASE	)     ,						6	)
                           self.assertGreater(tokens[0]     ,						tokenizer.vocab_size - 1	)
                           self.assertGreater(tokens[0]     ,						tokens[1]	)
                           self.assertGreater(tokens[-3]     ,						tokenizer.vocab_size - 1	)
                           self.assertGreater(tokens[-3]     ,						tokens[-4]	)
                           self.assertEqual(tokens[0]     ,						tokenizer.eos_token_id	)
                           self.assertEqual(tokens[-3]     ,						tokenizer.pad_token_id	)






      def __UpperCAmelCase       ( self			:int	)       ->					Any:




             '''simple docstring'''
             pass






      def __UpperCAmelCase       ( self			:Tuple	)       ->					List[Any]:




             '''simple docstring'''
             pass






      def __UpperCAmelCase       ( self			:List[str]	)       ->					Optional[Any]:




             '''simple docstring'''
             _a    : Optional[int]     					=self.get_tokenizer()

             _a    : Union[str, Any]     					=tokenizer.tokenize("""This is a test"""	)
             # fmt: off
             self.assertListEqual(SCREAMING_SNAKE_CASE     ,						[SPIECE_UNDERLINE, """T""", """h""", """i""", """s""", SPIECE_UNDERLINE, """i""", """s""", SPIECE_UNDERLINE, """a""", SPIECE_UNDERLINE, """t""", """e""", """s""", """t"""]	)
             # fmt: on

             self.assertListEqual(
                 tokenizer.convert_tokens_to_ids(SCREAMING_SNAKE_CASE	)     ,						[4, 3_2, 1_1, 1_0, 1_2, 4, 1_0, 1_2, 4, 7, 4, 6, 5, 1_2, 6]     ,						)

             _a    : Optional[int]     					=tokenizer.tokenize("""I was born in 92000, and this is falsé."""	)
             self.assertListEqual(
                 SCREAMING_SNAKE_CASE     ,						[SPIECE_UNDERLINE, """I""", SPIECE_UNDERLINE, """w""", """a""", """s""", SPIECE_UNDERLINE, """b""", """o""", """r""", """n""", SPIECE_UNDERLINE, """i""", """n""", SPIECE_UNDERLINE, """92000""", """,""", SPIECE_UNDERLINE, """a""", """n""", """d""", SPIECE_UNDERLINE, """t""", """h""", """i""", """s""", SPIECE_UNDERLINE, """i""", """s""", SPIECE_UNDERLINE, """f""", """a""", """l""", """s""", """é""", """."""]	)

             _a    : List[str]     					=tokenizer.convert_tokens_to_ids(SCREAMING_SNAKE_CASE	)
             # fmt: off
             self.assertListEqual(SCREAMING_SNAKE_CASE     ,						[4, 3_0, 4, 2_0, 7, 1_2, 4, 2_5, 8, 1_3, 9, 4, 1_0, 9, 4, 3, 2_3, 4, 7, 9, 1_4, 4, 6, 1_1, 1_0, 1_2, 4, 1_0, 1_2, 4, 1_9, 7, 1_5, 1_2, 7_3, 2_6]	)
             # fmt: on

             _a    : Any     					=tokenizer.convert_ids_to_tokens(SCREAMING_SNAKE_CASE	)
             self.assertListEqual(
                 SCREAMING_SNAKE_CASE     ,						[SPIECE_UNDERLINE, """I""", SPIECE_UNDERLINE, """w""", """a""", """s""", SPIECE_UNDERLINE, """b""", """o""", """r""", """n""", SPIECE_UNDERLINE, """i""", """n""", SPIECE_UNDERLINE, """<unk>""", """,""", SPIECE_UNDERLINE, """a""", """n""", """d""", SPIECE_UNDERLINE, """t""", """h""", """i""", """s""", SPIECE_UNDERLINE, """i""", """s""", SPIECE_UNDERLINE, """f""", """a""", """l""", """s""", """é""", """."""]	)






      @slow
      def __UpperCAmelCase       ( self			:List[str]	)       ->					Any:




             '''simple docstring'''
             # Use custom sequence because this tokenizer does not handle numbers.
             _a    : Dict     					=[
                 """Transformers (formerly known as pytorch-transformers and pytorch-pretrained-bert) provides """
                 """general-purpose architectures (BERT, GPT, RoBERTa, XLM, DistilBert, XLNet...) for Natural """
                 """Language Understanding (NLU) and Natural Language Generation (NLG) with over thirty-two pretrained """
                 """models in one hundred plus languages and deep interoperability between Jax, PyTorch and TensorFlow.""",
                 """BERT is designed to pre-train deep bidirectional representations from unlabeled text by jointly """
                 """conditioning on both left and right context in all layers.""",
                 """The quick brown fox jumps over the lazy dog.""",
             ]

             # fmt: off
             _a    : List[Any]     					={
                 """input_ids""": [
                     [4, 3_2, 1_3, 7, 9, 1_2, 1_9, 8, 1_3, 1_8, 5, 1_3, 1_2, 4, 6_4, 1_9, 8, 1_3, 1_8, 5, 1_3, 1_5, 2_2, 4, 2_8, 9, 8, 2_0, 9, 4, 7, 1_2, 4, 2_4, 2_2, 6, 8, 1_3, 1_7, 1_1, 3_9, 6, 1_3, 7, 9, 1_2, 1_9, 8, 1_3, 1_8, 5, 1_3, 1_2, 4, 7, 9, 1_4, 4, 2_4, 2_2, 6, 8, 1_3, 1_7, 1_1, 3_9, 2_4, 1_3, 5, 6, 1_3, 7, 1_0, 9, 5, 1_4, 3_9, 2_5, 5, 1_3, 6, 6_3, 4, 2_4, 1_3, 8, 2_7, 1_0, 1_4, 5, 1_2, 4, 2_1, 5, 9, 5, 1_3, 7, 1_5, 3_9, 2_4, 1_6, 1_3, 2_4, 8, 1_2, 5, 4, 7, 1_3, 1_7, 1_1, 1_0, 6, 5, 1_7, 6, 1_6, 1_3, 5, 1_2, 4, 6_4, 4_0, 4_7, 5_4, 3_2, 2_3, 4, 5_3, 4_9, 3_2, 2_3, 4, 5_4, 8, 4_0, 4_7, 5_4, 3_2, 7, 2_3, 4, 6_9, 5_2, 4_3, 2_3, 4, 5_1, 1_0, 1_2, 6, 1_0, 1_5, 4_0, 5, 1_3, 6, 2_3, 4, 6_9, 5_2, 4_8, 5, 6, 2_6, 2_6, 2_6, 6_3, 4, 1_9, 8, 1_3, 4, 4_8, 7, 6, 1_6, 1_3, 7, 1_5, 4, 5_2, 7, 9, 2_1, 1_6, 7, 2_1, 5, 4, 6_1, 9, 1_4, 5, 1_3, 1_2, 6, 7, 9, 1_4, 1_0, 9, 2_1, 4, 6_4, 4_8, 5_2, 6_1, 6_3, 4, 7, 9, 1_4, 4, 4_8, 7, 6, 1_6, 1_3, 7, 1_5, 4, 5_2, 7, 9, 2_1, 1_6, 7, 2_1, 5, 4, 5_3, 5, 9, 5, 1_3, 7, 6, 1_0, 8, 9, 4, 6_4, 4_8, 5_2, 5_3, 6_3, 4, 2_0, 1_0, 6, 1_1, 4, 8, 2_7, 5, 1_3, 4, 6, 1_1, 1_0, 1_3, 6, 2_2, 3_9, 6, 2_0, 8, 4, 2_4, 1_3, 5, 6, 1_3, 7, 1_0, 9, 5, 1_4, 4, 1_8, 8, 1_4, 5, 1_5, 1_2, 4, 1_0, 9, 4, 8, 9, 5, 4, 1_1, 1_6, 9, 1_4, 1_3, 5, 1_4, 4, 2_4, 1_5, 1_6, 1_2, 4, 1_5, 7, 9, 2_1, 1_6, 7, 2_1, 5, 1_2, 4, 7, 9, 1_4, 4, 1_4, 5, 5, 2_4, 4, 1_0, 9, 6, 5, 1_3, 8, 2_4, 5, 1_3, 7, 2_5, 1_0, 1_5, 1_0, 6, 2_2, 4, 2_5, 5, 6, 2_0, 5, 5, 9, 4, 5_8, 7, 3_7, 2_3, 4, 4_9, 2_2, 3_2, 8, 1_3, 1_7, 1_1, 4, 7, 9, 1_4, 4, 3_2, 5, 9, 1_2, 8, 1_3, 5_5, 1_5, 8, 2_0, 2_6, 2],
                     [4, 4_0, 4_7, 5_4, 3_2, 4, 1_0, 1_2, 4, 1_4, 5, 1_2, 1_0, 2_1, 9, 5, 1_4, 4, 6, 8, 4, 2_4, 1_3, 5, 3_9, 6, 1_3, 7, 1_0, 9, 4, 1_4, 5, 5, 2_4, 4, 2_5, 1_0, 1_4, 1_0, 1_3, 5, 1_7, 6, 1_0, 8, 9, 7, 1_5, 4, 1_3, 5, 2_4, 1_3, 5, 1_2, 5, 9, 6, 7, 6, 1_0, 8, 9, 1_2, 4, 1_9, 1_3, 8, 1_8, 4, 1_6, 9, 1_5, 7, 2_5, 5, 1_5, 5, 1_4, 4, 6, 5, 3_7, 6, 4, 2_5, 2_2, 4, 4_6, 8, 1_0, 9, 6, 1_5, 2_2, 4, 1_7, 8, 9, 1_4, 1_0, 6, 1_0, 8, 9, 1_0, 9, 2_1, 4, 8, 9, 4, 2_5, 8, 6, 1_1, 4, 1_5, 5, 1_9, 6, 4, 7, 9, 1_4, 4, 1_3, 1_0, 2_1, 1_1, 6, 4, 1_7, 8, 9, 6, 5, 3_7, 6, 4, 1_0, 9, 4, 7, 1_5, 1_5, 4, 1_5, 7, 2_2, 5, 1_3, 1_2, 2_6, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1],
                     [4, 3_2, 1_1, 5, 4, 4_5, 1_6, 1_0, 1_7, 2_8, 4, 2_5, 1_3, 8, 2_0, 9, 4, 1_9, 8, 3_7, 4, 4_6, 1_6, 1_8, 2_4, 1_2, 4, 8, 2_7, 5, 1_3, 4, 6, 1_1, 5, 4, 1_5, 7, 5_7, 2_2, 4, 1_4, 8, 2_1, 2_6, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1],
                 ],
                 """attention_mask""": [
                     [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1],
                     [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
                     [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
                 ]
             }
             # fmt: on

             self.tokenizer_integration_test_util(
                 expected_encoding=SCREAMING_SNAKE_CASE     ,						model_name="""microsoft/speecht5_asr"""     ,						revision="""c5ef64c71905caeccde0e4462ef3f9077224c524"""     ,						sequences=SCREAMING_SNAKE_CASE     ,						)






'''simple docstring'''







import unittest

from transformers import EsmConfig, is_torch_available
from transformers.testing_utils import TestCasePlus, require_torch, slow, torch_device

from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
   import torch

   from transformers import EsmForMaskedLM, EsmForSequenceClassification, EsmForTokenClassification, EsmModel
   from transformers.models.esm.modeling_esm import (
       ESM_PRETRAINED_MODEL_ARCHIVE_LIST,
       EsmEmbeddings,
       create_position_ids_from_input_ids,
   )







class A__       :






      def __init__( self			:Tuple     ,						SCREAMING_SNAKE_CASE			:int     ,						SCREAMING_SNAKE_CASE			:Optional[int]=1_3     ,						SCREAMING_SNAKE_CASE			:Optional[int]=7     ,						SCREAMING_SNAKE_CASE			:Tuple=False     ,						SCREAMING_SNAKE_CASE			:Dict=True     ,						SCREAMING_SNAKE_CASE			:Optional[int]=False     ,						SCREAMING_SNAKE_CASE			:Optional[Any]=True     ,						SCREAMING_SNAKE_CASE			:List[str]=3_3     ,						SCREAMING_SNAKE_CASE			:Tuple=3_2     ,						SCREAMING_SNAKE_CASE			:Tuple=5     ,						SCREAMING_SNAKE_CASE			:int=4     ,						SCREAMING_SNAKE_CASE			:Union[str, Any]=3_7     ,						SCREAMING_SNAKE_CASE			:List[str]="gelu"     ,						SCREAMING_SNAKE_CASE			:Optional[Any]=0.1     ,						SCREAMING_SNAKE_CASE			:Tuple=0.1     ,						SCREAMING_SNAKE_CASE			:str=5_1_2     ,						SCREAMING_SNAKE_CASE			:Dict=1_6     ,						SCREAMING_SNAKE_CASE			:Dict=2     ,						SCREAMING_SNAKE_CASE			:Union[str, Any]=0.02     ,						SCREAMING_SNAKE_CASE			:str=3     ,						SCREAMING_SNAKE_CASE			:List[str]=4     ,						SCREAMING_SNAKE_CASE			:List[str]=None     ,						)       ->					Union[str, Any]:




             '''simple docstring'''
             _a    : Union[str, Any]     					=parent
             _a    : List[Any]     					=batch_size
             _a    : Optional[int]     					=seq_length
             _a    : Union[str, Any]     					=is_training
             _a    : List[Any]     					=use_input_mask
             _a    : Optional[int]     					=use_token_type_ids
             _a    : int     					=use_labels
             _a    : List[str]     					=vocab_size
             _a    : List[Any]     					=hidden_size
             _a    : int     					=num_hidden_layers
             _a    : Tuple     					=num_attention_heads
             _a    : Any     					=intermediate_size
             _a    : str     					=hidden_act
             _a    : Union[str, Any]     					=hidden_dropout_prob
             _a    : Union[str, Any]     					=attention_probs_dropout_prob
             _a    : str     					=max_position_embeddings
             _a    : Dict     					=type_vocab_size
             _a    : Tuple     					=type_sequence_label_size
             _a    : Dict     					=initializer_range
             _a    : List[str]     					=num_labels
             _a    : Tuple     					=num_choices
             _a    : int     					=scope






      def __UpperCAmelCase       ( self			:Union[str, Any]	)       ->					str:




             '''simple docstring'''
             _a    : Optional[int]     					=ids_tensor([self.batch_size, self.seq_length]     ,						self.vocab_size	)

             _a    : List[Any]     					=None
             if self.use_input_mask:
                    _a    : Any     					=random_attention_mask([self.batch_size, self.seq_length]	)

             _a    : Optional[int]     					=None
             _a    : str     					=None
             _a    : Dict     					=None
             if self.use_labels:
                    _a    : Dict     					=ids_tensor([self.batch_size]     ,						self.type_sequence_label_size	)
                    _a    : str     					=ids_tensor([self.batch_size, self.seq_length]     ,						self.num_labels	)
                    _a    : List[str]     					=ids_tensor([self.batch_size]     ,						self.num_choices	)

             _a    : List[Any]     					=self.get_config()

             return config, input_ids, input_mask, sequence_labels, token_labels, choice_labels






      def __UpperCAmelCase       ( self			:str	)       ->					Optional[int]:




             '''simple docstring'''
             return EsmConfig(
                 vocab_size=self.vocab_size     ,						hidden_size=self.hidden_size     ,						pad_token_id=1     ,						num_hidden_layers=self.num_hidden_layers     ,						num_attention_heads=self.num_attention_heads     ,						intermediate_size=self.intermediate_size     ,						hidden_act=self.hidden_act     ,						hidden_dropout_prob=self.hidden_dropout_prob     ,						attention_probs_dropout_prob=self.attention_probs_dropout_prob     ,						max_position_embeddings=self.max_position_embeddings     ,						type_vocab_size=self.type_vocab_size     ,						initializer_range=self.initializer_range     ,						)






      def __UpperCAmelCase       ( self			:List[str]     ,						SCREAMING_SNAKE_CASE			:Tuple     ,						SCREAMING_SNAKE_CASE			:Optional[Any]     ,						SCREAMING_SNAKE_CASE			:Any     ,						SCREAMING_SNAKE_CASE			:str     ,						SCREAMING_SNAKE_CASE			:List[str]     ,						SCREAMING_SNAKE_CASE			:int	)       ->					Tuple:




             '''simple docstring'''
             _a    : Any     					=EsmModel(config=SCREAMING_SNAKE_CASE	)
             model.to(SCREAMING_SNAKE_CASE	)
             model.eval()
             _a    : Optional[Any]     					=model(SCREAMING_SNAKE_CASE     ,						attention_mask=SCREAMING_SNAKE_CASE	)
             _a    : Union[str, Any]     					=model(SCREAMING_SNAKE_CASE	)
             _a    : str     					=model(SCREAMING_SNAKE_CASE	)

             self.parent.assertEqual(result.last_hidden_state.shape     ,						(self.batch_size, self.seq_length, self.hidden_size)	)
             self.parent.assertEqual(result.pooler_output.shape     ,						(self.batch_size, self.hidden_size)	)






      def __UpperCAmelCase       ( self			:str     ,						SCREAMING_SNAKE_CASE			:Any     ,						SCREAMING_SNAKE_CASE			:List[str]     ,						SCREAMING_SNAKE_CASE			:str     ,						SCREAMING_SNAKE_CASE			:Dict     ,						SCREAMING_SNAKE_CASE			:Dict     ,						SCREAMING_SNAKE_CASE			:Optional[Any]	)       ->					Dict:




             '''simple docstring'''
             _a    : str     					=EsmForMaskedLM(config=SCREAMING_SNAKE_CASE	)
             model.to(SCREAMING_SNAKE_CASE	)
             model.eval()
             _a    : Union[str, Any]     					=model(SCREAMING_SNAKE_CASE     ,						attention_mask=SCREAMING_SNAKE_CASE     ,						labels=SCREAMING_SNAKE_CASE	)
             self.parent.assertEqual(result.logits.shape     ,						(self.batch_size, self.seq_length, self.vocab_size)	)






      def __UpperCAmelCase       ( self			:Optional[Any]     ,						SCREAMING_SNAKE_CASE			:Tuple     ,						SCREAMING_SNAKE_CASE			:Optional[Any]     ,						SCREAMING_SNAKE_CASE			:Union[str, Any]     ,						SCREAMING_SNAKE_CASE			:Dict     ,						SCREAMING_SNAKE_CASE			:List[Any]     ,						SCREAMING_SNAKE_CASE			:Union[str, Any]	)       ->					Optional[Any]:




             '''simple docstring'''
             _a    : int     					=self.num_labels
             _a    : Tuple     					=EsmForTokenClassification(config=SCREAMING_SNAKE_CASE	)
             model.to(SCREAMING_SNAKE_CASE	)
             model.eval()
             _a    : Tuple     					=model(SCREAMING_SNAKE_CASE     ,						attention_mask=SCREAMING_SNAKE_CASE     ,						labels=SCREAMING_SNAKE_CASE	)
             self.parent.assertEqual(result.logits.shape     ,						(self.batch_size, self.seq_length, self.num_labels)	)






      def __UpperCAmelCase       ( self			:Dict	)       ->					List[str]:




             '''simple docstring'''
             _a    : Optional[Any]     					=self.prepare_config_and_inputs()
             (
                 (
                 _a
             )					,    (
                 _a
             )					,    (
                 _a
             )					,    (
                 _a
             )					,    (
                 _a
             )					,    (
                 _a
             )					,    
             )    : Any     					=config_and_inputs
             _a    : List[Any]     					={"""input_ids""": input_ids, """attention_mask""": input_mask}
             return config, inputs_dict







@require_torch
class A__       (						UpperCAmelCase__ ,     UpperCAmelCase__ ,     unittest.TestCase      ):
      __UpperCamelCase       :  Any	       =							False

      __UpperCamelCase       :  Any	       =							(
          (
              EsmForMaskedLM,
              EsmModel,
              EsmForSequenceClassification,
              EsmForTokenClassification,
          )
          if is_torch_available()
          else ()
      )
      __UpperCamelCase       :  str	       =							()
      __UpperCamelCase       :  List[str]	       =							(
          {
              "feature-extraction": EsmModel,
              "fill-mask": EsmForMaskedLM,
              "text-classification": EsmForSequenceClassification,
              "token-classification": EsmForTokenClassification,
              "zero-shot": EsmForSequenceClassification,
          }
          if is_torch_available()
          else {}
      )
      __UpperCamelCase       :  Union[str, Any]	       =							True






      def __UpperCAmelCase       ( self			:Optional[int]	)       ->					int:




             '''simple docstring'''
             _a    : Dict     					=EsmModelTester(self	)
             _a    : Optional[Any]     					=ConfigTester(self     ,						config_class=SCREAMING_SNAKE_CASE     ,						hidden_size=3_7	)






      def __UpperCAmelCase       ( self			:Tuple	)       ->					List[Any]:




             '''simple docstring'''
             self.config_tester.run_common_tests()






      def __UpperCAmelCase       ( self			:Optional[int]	)       ->					str:




             '''simple docstring'''
             _a    : List[Any]     					=self.model_tester.prepare_config_and_inputs()
             self.model_tester.create_and_check_model(*SCREAMING_SNAKE_CASE	)






      def __UpperCAmelCase       ( self			:List[Any]	)       ->					Dict:




             '''simple docstring'''
             _a    : List[str]     					=self.model_tester.prepare_config_and_inputs()
             for type in ["absolute", "relative_key", "relative_key_query"]:
                    _a    : Dict     					=type
                    self.model_tester.create_and_check_model(*SCREAMING_SNAKE_CASE	)






      def __UpperCAmelCase       ( self			:Dict	)       ->					List[str]:




             '''simple docstring'''
             _a    : Tuple     					=self.model_tester.prepare_config_and_inputs()
             self.model_tester.create_and_check_for_masked_lm(*SCREAMING_SNAKE_CASE	)






      def __UpperCAmelCase       ( self			:List[Any]	)       ->					List[str]:




             '''simple docstring'''
             _a    : str     					=self.model_tester.prepare_config_and_inputs()
             self.model_tester.create_and_check_for_token_classification(*SCREAMING_SNAKE_CASE	)






      @slow
      def __UpperCAmelCase       ( self			:str	)       ->					Dict:




             '''simple docstring'''
             for model_name in ESM_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
                    _a    : Union[str, Any]     					=EsmModel.from_pretrained(SCREAMING_SNAKE_CASE	)
                    self.assertIsNotNone(SCREAMING_SNAKE_CASE	)






      def __UpperCAmelCase       ( self			:Tuple	)       ->					int:




             '''simple docstring'''
             _a    : Optional[Any]     					=self.model_tester.prepare_config_and_inputs()[0]
             _a    : Dict     					=EsmEmbeddings(config=SCREAMING_SNAKE_CASE	)

             _a    : Tuple     					=torch.as_tensor([[1_2, 3_1, 1_3, model.padding_idx]]	)
             _a    : Optional[Any]     					=torch.as_tensor(
                 [
                     [
                         0 + model.padding_idx + 1,
                         1 + model.padding_idx + 1,
                         2 + model.padding_idx + 1,
                         model.padding_idx,
                     ]
                 ]	)
             _a    : Any     					=create_position_ids_from_input_ids(SCREAMING_SNAKE_CASE     ,						model.padding_idx	)
             self.assertEqual(position_ids.shape     ,						expected_positions.shape	)
             self.assertTrue(torch.all(torch.eq(SCREAMING_SNAKE_CASE     ,						SCREAMING_SNAKE_CASE	)	)	)






      def __UpperCAmelCase       ( self			:Optional[Any]	)       ->					Tuple:




             '''simple docstring'''
             _a    : List[Any]     					=self.model_tester.prepare_config_and_inputs()[0]
             _a    : Optional[int]     					=EsmEmbeddings(config=SCREAMING_SNAKE_CASE	)

             _a    : Tuple     					=torch.empty(2     ,						4     ,						3_0	)
             _a    : str     					=[
                 0 + embeddings.padding_idx + 1,
                 1 + embeddings.padding_idx + 1,
                 2 + embeddings.padding_idx + 1,
                 3 + embeddings.padding_idx + 1,
             ]
             _a    : int     					=torch.as_tensor([expected_single_positions, expected_single_positions]	)
             _a    : Any     					=embeddings.create_position_ids_from_inputs_embeds(SCREAMING_SNAKE_CASE	)
             self.assertEqual(position_ids.shape     ,						expected_positions.shape	)
             self.assertTrue(torch.all(torch.eq(SCREAMING_SNAKE_CASE     ,						SCREAMING_SNAKE_CASE	)	)	)






      @unittest.skip("""Esm does not support embedding resizing"""	)
      def __UpperCAmelCase       ( self			:Tuple	)       ->					List[str]:




             '''simple docstring'''
             pass






      @unittest.skip("""Esm does not support embedding resizing"""	)
      def __UpperCAmelCase       ( self			:str	)       ->					Any:




             '''simple docstring'''
             pass






      @unittest.skip("""Will be fixed soon by reducing the size of the model used for common tests."""	)
      def __UpperCAmelCase       ( self			:Dict	)       ->					Any:




             '''simple docstring'''
             pass







@require_torch
class A__       (						UpperCAmelCase__      ):






      @slow
      def __UpperCAmelCase       ( self			:List[Any]	)       ->					str:




             '''simple docstring'''
             with torch.no_grad():
                    _a    : Optional[int]     					=EsmForMaskedLM.from_pretrained("""facebook/esm2_t6_8M_UR50D"""	)
                    model.eval()
                    _a    : Any     					=torch.tensor([[0, 1, 2, 3, 4, 5]]	)
                    _a    : Tuple     					=model(SCREAMING_SNAKE_CASE	)[0]

                    _a    : int     					=3_3

                    _a    : Tuple     					=torch.Size((1, 6, vocab_size)	)
                    self.assertEqual(output.shape     ,						SCREAMING_SNAKE_CASE	)

                    _a    : Union[str, Any]     					=torch.tensor(
                        [[[8.9_215, -10.5_898, -6.4_671], [-6.3_967, -13.9_114, -1.1_212], [-7.7_812, -13.9_516, -3.7_406]]]	)
                    self.assertTrue(torch.allclose(output[:, :3, :3]     ,						SCREAMING_SNAKE_CASE     ,						atol=1e-4	)	)






      @slow
      def __UpperCAmelCase       ( self			:Union[str, Any]	)       ->					Tuple:




             '''simple docstring'''
             with torch.no_grad():
                    _a    : Any     					=EsmModel.from_pretrained("""facebook/esm2_t6_8M_UR50D"""	)
                    model.eval()

                    _a    : Any     					=torch.tensor([[0, 6, 4, 1_3, 5, 4, 1_6, 1_2, 1_1, 7, 2]]	)
                    _a    : int     					=model(SCREAMING_SNAKE_CASE	)[0]
                    # compare the actual values for a slice.
                    _a    : str     					=torch.tensor(
                        [[[0.1_444, 0.5_413, 0.3_248], [0.3_034, 0.0_053, 0.3_108], [0.3_228, -0.2_499, 0.3_415]]]	)
                    self.assertTrue(torch.allclose(output[:, :3, :3]     ,						SCREAMING_SNAKE_CASE     ,						atol=1e-4	)	)


