




import unittest

from transformers import SPIECE_UNDERLINE
from transformers.models.speechta import SpeechTaTokenizer
from transformers.testing_utils import get_tests_dir, require_sentencepiece, require_tokenizers, slow
from transformers.tokenization_utils import AddedToken

from ...test_tokenization_common import TokenizerTesterMixin


_lowerCamelCase	:		Dict							       =   get_tests_dir("""fixtures/test_sentencepiece_bpe_char.model""")


@require_sentencepiece
@require_tokenizers
class 				UpperCamelCase_    ( UpperCAmelCase__		,					unittest.TestCase     ):

    '''simple docstring'''






    UpperCAmelCase__							  =      SpeechTaTokenizer
    UpperCAmelCase__							  =      False
    UpperCAmelCase__							  =      True


    def        SCREAMING_SNAKE_CASE							(			self      :	Any)						->Tuple:





           '''simple docstring'''

           super().setUp()

           # We have a SentencePiece fixture for testing
           A__       =    SpeechTaTokenizer(UpperCAmelCase__)

           A__       =    AddedToken('''<mask>'''					, lstrip=UpperCAmelCase__					, rstrip=UpperCAmelCase__)
           A__       =    mask_token
           tokenizer.add_special_tokens({'''mask_token''': mask_token})
           tokenizer.add_tokens(['''<ctc_blank>'''])

           tokenizer.save_pretrained(self.tmpdirname)


    def        SCREAMING_SNAKE_CASE							(			self      :	Union[str, Any]					, UpperCAmelCase__      :	List[Any])						->List[Any]:





           '''simple docstring'''

           A__       =    '''this is a test'''
           A__       =    '''this is a test'''
           return input_text, output_text


    def        SCREAMING_SNAKE_CASE							(			self      :	Optional[int]					, UpperCAmelCase__      :	int					, UpperCAmelCase__      :	Dict=False					, UpperCAmelCase__      :	Any=20					, UpperCAmelCase__      :	int=5)						->Any:





           '''simple docstring'''

           A__ ,							A__       =    self.get_input_output_texts(UpperCAmelCase__)
           A__       =    tokenizer.encode(UpperCAmelCase__					, add_special_tokens=UpperCAmelCase__)
           A__       =    tokenizer.decode(UpperCAmelCase__					, clean_up_tokenization_spaces=UpperCAmelCase__)
           return text, ids


    def        SCREAMING_SNAKE_CASE							(			self      :	Optional[int])						->int:





           '''simple docstring'''

           A__       =    '''<pad>'''
           A__       =    1

           self.assertEqual(self.get_tokenizer()._convert_token_to_id(UpperCAmelCase__)					, UpperCAmelCase__)
           self.assertEqual(self.get_tokenizer()._convert_id_to_token(UpperCAmelCase__)					, UpperCAmelCase__)


    def        SCREAMING_SNAKE_CASE							(			self      :	List[str])						->List[str]:





           '''simple docstring'''

           A__       =    list(self.get_tokenizer().get_vocab().keys())

           self.assertEqual(vocab_keys[0]					, '''<s>''')
           self.assertEqual(vocab_keys[1]					, '''<pad>''')
           self.assertEqual(vocab_keys[-4]					, '''œ''')
           self.assertEqual(vocab_keys[-2]					, '''<mask>''')
           self.assertEqual(vocab_keys[-1]					, '''<ctc_blank>''')
           self.assertEqual(len(UpperCAmelCase__)					, 81)


    def        SCREAMING_SNAKE_CASE							(			self      :	List[Any])						->List[str]:





           '''simple docstring'''

           self.assertEqual(self.get_tokenizer().vocab_size					, 79)


    def        SCREAMING_SNAKE_CASE							(			self      :	str)						->Union[str, Any]:





           '''simple docstring'''

           A__       =    self.get_tokenizers(do_lower_case=UpperCAmelCase__)
           for tokenizer in tokenizers:
                  with self.subTest(f"""{tokenizer.__class__.__name__}"""):
                         A__       =    tokenizer.vocab_size
                         A__       =    len(UpperCAmelCase__)

                         self.assertNotEqual(UpperCAmelCase__					, 0)

                         # We usually have added tokens from the start in tests because our vocab fixtures are
                         # smaller than the original vocabs - let's not assert this
                         # self.assertEqual(vocab_size, all_size)

                         A__       =    ['''aaaaa bbbbbb''', '''cccccccccdddddddd''']
                         A__       =    tokenizer.add_tokens(UpperCAmelCase__)
                         A__       =    tokenizer.vocab_size
                         A__       =    len(UpperCAmelCase__)

                         self.assertNotEqual(UpperCAmelCase__					, 0)
                         self.assertEqual(UpperCAmelCase__					, UpperCAmelCase__)
                         self.assertEqual(UpperCAmelCase__					, len(UpperCAmelCase__))
                         self.assertEqual(UpperCAmelCase__					, all_size + len(UpperCAmelCase__))

                         A__       =    tokenizer.encode('''aaaaa bbbbbb low cccccccccdddddddd l'''					, add_special_tokens=UpperCAmelCase__)

                         self.assertGreaterEqual(len(UpperCAmelCase__)					, 4)
                         self.assertGreater(tokens[0]					, tokenizer.vocab_size - 1)
                         self.assertGreater(tokens[-3]					, tokenizer.vocab_size - 1)

                         A__       =    {'''eos_token''': '''>>>>|||<||<<|<<''', '''pad_token''': '''<<<<<|||>|>>>>|>'''}
                         A__       =    tokenizer.add_special_tokens(UpperCAmelCase__)
                         A__       =    tokenizer.vocab_size
                         A__       =    len(UpperCAmelCase__)

                         self.assertNotEqual(UpperCAmelCase__					, 0)
                         self.assertEqual(UpperCAmelCase__					, UpperCAmelCase__)
                         self.assertEqual(UpperCAmelCase__					, len(UpperCAmelCase__))
                         self.assertEqual(UpperCAmelCase__					, all_size_a + len(UpperCAmelCase__))

                         A__       =    tokenizer.encode(
                             '''>>>>|||<||<<|<< aaaaabbbbbb low cccccccccdddddddd <<<<<|||>|>>>>|> l'''					, add_special_tokens=UpperCAmelCase__)

                         self.assertGreaterEqual(len(UpperCAmelCase__)					, 6)
                         self.assertGreater(tokens[0]					, tokenizer.vocab_size - 1)
                         self.assertGreater(tokens[0]					, tokens[1])
                         self.assertGreater(tokens[-3]					, tokenizer.vocab_size - 1)
                         self.assertGreater(tokens[-3]					, tokens[-4])
                         self.assertEqual(tokens[0]					, tokenizer.eos_token_id)
                         self.assertEqual(tokens[-3]					, tokenizer.pad_token_id)


    def        SCREAMING_SNAKE_CASE							(			self      :	Optional[int])						->List[Any]:





           '''simple docstring'''

           pass


    def        SCREAMING_SNAKE_CASE							(			self      :	Optional[Any])						->Tuple:





           '''simple docstring'''

           pass


    def        SCREAMING_SNAKE_CASE							(			self      :	List[str])						->List[Any]:





           '''simple docstring'''

           A__       =    self.get_tokenizer()

           A__       =    tokenizer.tokenize('''This is a test''')
           # fmt: off
           self.assertListEqual(UpperCAmelCase__					, [SPIECE_UNDERLINE, '''T''', '''h''', '''i''', '''s''', SPIECE_UNDERLINE, '''i''', '''s''', SPIECE_UNDERLINE, '''a''', SPIECE_UNDERLINE, '''t''', '''e''', '''s''', '''t'''])
           # fmt: on

           self.assertListEqual(
               tokenizer.convert_tokens_to_ids(UpperCAmelCase__)					, [4, 32, 11, 10, 12, 4, 10, 12, 4, 7, 4, 6, 5, 12, 6]					, )

           A__       =    tokenizer.tokenize('''I was born in 92000, and this is falsé.''')
           self.assertListEqual(
               UpperCAmelCase__					, [SPIECE_UNDERLINE, '''I''', SPIECE_UNDERLINE, '''w''', '''a''', '''s''', SPIECE_UNDERLINE, '''b''', '''o''', '''r''', '''n''', SPIECE_UNDERLINE, '''i''', '''n''', SPIECE_UNDERLINE, '''92000''', ''',''', SPIECE_UNDERLINE, '''a''', '''n''', '''d''', SPIECE_UNDERLINE, '''t''', '''h''', '''i''', '''s''', SPIECE_UNDERLINE, '''i''', '''s''', SPIECE_UNDERLINE, '''f''', '''a''', '''l''', '''s''', '''é''', '''.'''])

           A__       =    tokenizer.convert_tokens_to_ids(UpperCAmelCase__)
           # fmt: off
           self.assertListEqual(UpperCAmelCase__					, [4, 30, 4, 20, 7, 12, 4, 25, 8, 13, 9, 4, 10, 9, 4, 3, 23, 4, 7, 9, 14, 4, 6, 11, 10, 12, 4, 10, 12, 4, 19, 7, 15, 12, 73, 26])
           # fmt: on

           A__       =    tokenizer.convert_ids_to_tokens(UpperCAmelCase__)
           self.assertListEqual(
               UpperCAmelCase__					, [SPIECE_UNDERLINE, '''I''', SPIECE_UNDERLINE, '''w''', '''a''', '''s''', SPIECE_UNDERLINE, '''b''', '''o''', '''r''', '''n''', SPIECE_UNDERLINE, '''i''', '''n''', SPIECE_UNDERLINE, '''<unk>''', ''',''', SPIECE_UNDERLINE, '''a''', '''n''', '''d''', SPIECE_UNDERLINE, '''t''', '''h''', '''i''', '''s''', SPIECE_UNDERLINE, '''i''', '''s''', SPIECE_UNDERLINE, '''f''', '''a''', '''l''', '''s''', '''é''', '''.'''])





    @slow
    def        SCREAMING_SNAKE_CASE							(			self      :	Union[str, Any])						->List[Any]:





           '''simple docstring'''

           A__       =    [
               '''Transformers (formerly known as pytorch-transformers and pytorch-pretrained-bert) provides '''
               '''general-purpose architectures (BERT, GPT, RoBERTa, XLM, DistilBert, XLNet...) for Natural '''
               '''Language Understanding (NLU) and Natural Language Generation (NLG) with over thirty-two pretrained '''
               '''models in one hundred plus languages and deep interoperability between Jax, PyTorch and TensorFlow.''',
               '''BERT is designed to pre-train deep bidirectional representations from unlabeled text by jointly '''
               '''conditioning on both left and right context in all layers.''',
               '''The quick brown fox jumps over the lazy dog.''',
           ]

           # fmt: off
           A__       =    {
               '''input_ids''': [
                   [4, 32, 13, 7, 9, 12, 19, 8, 13, 18, 5, 13, 12, 4, 64, 19, 8, 13, 18, 5, 13, 15, 22, 4, 28, 9, 8, 20, 9, 4, 7, 12, 4, 24, 22, 6, 8, 13, 17, 11, 39, 6, 13, 7, 9, 12, 19, 8, 13, 18, 5, 13, 12, 4, 7, 9, 14, 4, 24, 22, 6, 8, 13, 17, 11, 39, 24, 13, 5, 6, 13, 7, 10, 9, 5, 14, 39, 25, 5, 13, 6, 63, 4, 24, 13, 8, 27, 10, 14, 5, 12, 4, 21, 5, 9, 5, 13, 7, 15, 39, 24, 16, 13, 24, 8, 12, 5, 4, 7, 13, 17, 11, 10, 6, 5, 17, 6, 16, 13, 5, 12, 4, 64, 40, 47, 54, 32, 23, 4, 53, 49, 32, 23, 4, 54, 8, 40, 47, 54, 32, 7, 23, 4, 69, 52, 43, 23, 4, 51, 10, 12, 6, 10, 15, 40, 5, 13, 6, 23, 4, 69, 52, 48, 5, 6, 26, 26, 26, 63, 4, 19, 8, 13, 4, 48, 7, 6, 16, 13, 7, 15, 4, 52, 7, 9, 21, 16, 7, 21, 5, 4, 61, 9, 14, 5, 13, 12, 6, 7, 9, 14, 10, 9, 21, 4, 64, 48, 52, 61, 63, 4, 7, 9, 14, 4, 48, 7, 6, 16, 13, 7, 15, 4, 52, 7, 9, 21, 16, 7, 21, 5, 4, 53, 5, 9, 5, 13, 7, 6, 10, 8, 9, 4, 64, 48, 52, 53, 63, 4, 20, 10, 6, 11, 4, 8, 27, 5, 13, 4, 6, 11, 10, 13, 6, 22, 39, 6, 20, 8, 4, 24, 13, 5, 6, 13, 7, 10, 9, 5, 14, 4, 18, 8, 14, 5, 15, 12, 4, 10, 9, 4, 8, 9, 5, 4, 11, 16, 9, 14, 13, 5, 14, 4, 24, 15, 16, 12, 4, 15, 7, 9, 21, 16, 7, 21, 5, 12, 4, 7, 9, 14, 4, 14, 5, 5, 24, 4, 10, 9, 6, 5, 13, 8, 24, 5, 13, 7, 25, 10, 15, 10, 6, 22, 4, 25, 5, 6, 20, 5, 5, 9, 4, 58, 7, 37, 23, 4, 49, 22, 32, 8, 13, 17, 11, 4, 7, 9, 14, 4, 32, 5, 9, 12, 8, 13, 55, 15, 8, 20, 26, 2],
                   [4, 40, 47, 54, 32, 4, 10, 12, 4, 14, 5, 12, 10, 21, 9, 5, 14, 4, 6, 8, 4, 24, 13, 5, 39, 6, 13, 7, 10, 9, 4, 14, 5, 5, 24, 4, 25, 10, 14, 10, 13, 5, 17, 6, 10, 8, 9, 7, 15, 4, 13, 5, 24, 13, 5, 12, 5, 9, 6, 7, 6, 10, 8, 9, 12, 4, 19, 13, 8, 18, 4, 16, 9, 15, 7, 25, 5, 15, 5, 14, 4, 6, 5, 37, 6, 4, 25, 22, 4, 46, 8, 10, 9, 6, 15, 22, 4, 17, 8, 9, 14, 10, 6, 10, 8, 9, 10, 9, 21, 4, 8, 9, 4, 25, 8, 6, 11, 4, 15, 5, 19, 6, 4, 7, 9, 14, 4, 13, 10, 21, 11, 6, 4, 17, 8, 9, 6, 5, 37, 6, 4, 10, 9, 4, 7, 15, 15, 4, 15, 7, 22, 5, 13, 12, 26, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1],
                   [4, 32, 11, 5, 4, 45, 16, 10, 17, 28, 4, 25, 13, 8, 20, 9, 4, 19, 8, 37, 4, 46, 16, 18, 24, 12, 4, 8, 27, 5, 13, 4, 6, 11, 5, 4, 15, 7, 57, 22, 4, 14, 8, 21, 26, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1],
               ],
               '''attention_mask''': [
                   [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1],
                   [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
                   [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
               ]
           }
           # fmt: on

           self.tokenizer_integration_test_util(
               expected_encoding=UpperCAmelCase__					, model_name='''microsoft/speecht5_asr'''					, revision='''c5ef64c71905caeccde0e4462ef3f9077224c524'''					, sequences=UpperCAmelCase__					, )



import argparse
import json
from pathlib import Path

import requests
import timm
import torch
from huggingface_hub import hf_hub_download
from PIL import Image

from transformers import DeiTConfig, DeiTForImageClassificationWithTeacher, DeiTImageProcessor
from transformers.utils import logging


logging.set_verbosity_info()
A   :     str       =				logging.get_logger(__name__)


def    a__       (						__UpperCamelCase   ,					__UpperCamelCase=False ):
      SCREAMING_SNAKE_CASE_      =     []
      for i in range(config.num_hidden_layers ):
            # encoder layers: output projection, 2 feedforward neural networks and 2 layernorms
            rename_keys.append((F'''blocks.{i}.norm1.weight''', F'''deit.encoder.layer.{i}.layernorm_before.weight''') )
            rename_keys.append((F'''blocks.{i}.norm1.bias''', F'''deit.encoder.layer.{i}.layernorm_before.bias''') )
            rename_keys.append((F'''blocks.{i}.attn.proj.weight''', F'''deit.encoder.layer.{i}.attention.output.dense.weight''') )
            rename_keys.append((F'''blocks.{i}.attn.proj.bias''', F'''deit.encoder.layer.{i}.attention.output.dense.bias''') )
            rename_keys.append((F'''blocks.{i}.norm2.weight''', F'''deit.encoder.layer.{i}.layernorm_after.weight''') )
            rename_keys.append((F'''blocks.{i}.norm2.bias''', F'''deit.encoder.layer.{i}.layernorm_after.bias''') )
            rename_keys.append((F'''blocks.{i}.mlp.fc1.weight''', F'''deit.encoder.layer.{i}.intermediate.dense.weight''') )
            rename_keys.append((F'''blocks.{i}.mlp.fc1.bias''', F'''deit.encoder.layer.{i}.intermediate.dense.bias''') )
            rename_keys.append((F'''blocks.{i}.mlp.fc2.weight''', F'''deit.encoder.layer.{i}.output.dense.weight''') )
            rename_keys.append((F'''blocks.{i}.mlp.fc2.bias''', F'''deit.encoder.layer.{i}.output.dense.bias''') )

      # projection layer + position embeddings
      rename_keys.extend(
          [
              ("cls_token", "deit.embeddings.cls_token"),
              ("dist_token", "deit.embeddings.distillation_token"),
              ("patch_embed.proj.weight", "deit.embeddings.patch_embeddings.projection.weight"),
              ("patch_embed.proj.bias", "deit.embeddings.patch_embeddings.projection.bias"),
              ("pos_embed", "deit.embeddings.position_embeddings"),
          ] )

      if base_model:
            # layernorm + pooler
            rename_keys.extend(
                [
                    ("norm.weight", "layernorm.weight"),
                    ("norm.bias", "layernorm.bias"),
                    ("pre_logits.fc.weight", "pooler.dense.weight"),
                    ("pre_logits.fc.bias", "pooler.dense.bias"),
                ] )

            # if just the base model, we should remove "deit" from all keys that start with "deit"
            SCREAMING_SNAKE_CASE_      =     [(pair[0], pair[1][4:]) if pair[1].startswith("deit" ) else pair for pair in rename_keys]
      else:
            # layernorm + classification heads
            rename_keys.extend(
                [
                    ("norm.weight", "deit.layernorm.weight"),
                    ("norm.bias", "deit.layernorm.bias"),
                    ("head.weight", "cls_classifier.weight"),
                    ("head.bias", "cls_classifier.bias"),
                    ("head_dist.weight", "distillation_classifier.weight"),
                    ("head_dist.bias", "distillation_classifier.bias"),
                ] )

      return rename_keys


def    a__       (						__UpperCamelCase   ,					__UpperCamelCase   ,					__UpperCamelCase=False ):
      for i in range(config.num_hidden_layers ):
            if base_model:
                  SCREAMING_SNAKE_CASE_      =     ""
            else:
                  SCREAMING_SNAKE_CASE_      =     "deit."
            # read in weights + bias of input projection layer (in timm, this is a single matrix + bias)
            SCREAMING_SNAKE_CASE_      =     state_dict.pop(F'''blocks.{i}.attn.qkv.weight''' )
            SCREAMING_SNAKE_CASE_      =     state_dict.pop(F'''blocks.{i}.attn.qkv.bias''' )
            # next, add query, keys and values (in that order) to the state dict
            SCREAMING_SNAKE_CASE_      =     in_proj_weight[
                : config.hidden_size, :
            ]
            SCREAMING_SNAKE_CASE_      =     in_proj_bias[: config.hidden_size]
            SCREAMING_SNAKE_CASE_      =     in_proj_weight[
                config.hidden_size : config.hidden_size * 2, :
            ]
            SCREAMING_SNAKE_CASE_      =     in_proj_bias[
                config.hidden_size : config.hidden_size * 2
            ]
            SCREAMING_SNAKE_CASE_      =     in_proj_weight[
                -config.hidden_size :, :
            ]
            SCREAMING_SNAKE_CASE_      =     in_proj_bias[-config.hidden_size :]


def    a__       (						__UpperCamelCase   ,					__UpperCamelCase   ,					__UpperCamelCase ):
      SCREAMING_SNAKE_CASE_      =     dct.pop(__UpperCamelCase )
      SCREAMING_SNAKE_CASE_      =     val


def    a__       (						):
      SCREAMING_SNAKE_CASE_      =     "http://images.cocodataset.org/val2017/000000039769.jpg"
      SCREAMING_SNAKE_CASE_      =     Image.open(requests.get(__UpperCamelCase   ,					stream=__UpperCamelCase ).raw )
      return im


@torch.no_grad()
def    a__       (						__UpperCamelCase   ,					__UpperCamelCase ):
      SCREAMING_SNAKE_CASE_      =     DeiTConfig()
      # all deit models have fine-tuned heads
      SCREAMING_SNAKE_CASE_      =     False
      # dataset (fine-tuned on ImageNet 2012), patch_size and image_size
      SCREAMING_SNAKE_CASE_      =     1_0_0_0
      SCREAMING_SNAKE_CASE_      =     "huggingface/label-files"
      SCREAMING_SNAKE_CASE_      =     "imagenet-1k-id2label.json"
      SCREAMING_SNAKE_CASE_      =     json.load(open(hf_hub_download(__UpperCamelCase   ,					__UpperCamelCase   ,					repo_type="dataset" )   ,					"r" ) )
      SCREAMING_SNAKE_CASE_      =     {int(__UpperCamelCase ): v for k, v in idalabel.items()}
      SCREAMING_SNAKE_CASE_      =     idalabel
      SCREAMING_SNAKE_CASE_      =     {v: k for k, v in idalabel.items()}
      SCREAMING_SNAKE_CASE_      =     int(deit_name[-6:-4] )
      SCREAMING_SNAKE_CASE_      =     int(deit_name[-3:] )
      # size of the architecture
      if deit_name[9:].startswith("tiny" ):
            SCREAMING_SNAKE_CASE_      =     1_9_2
            SCREAMING_SNAKE_CASE_      =     7_6_8
            SCREAMING_SNAKE_CASE_      =     1_2
            SCREAMING_SNAKE_CASE_      =     3
      elif deit_name[9:].startswith("small" ):
            SCREAMING_SNAKE_CASE_      =     3_8_4
            SCREAMING_SNAKE_CASE_      =     1_5_3_6
            SCREAMING_SNAKE_CASE_      =     1_2
            SCREAMING_SNAKE_CASE_      =     6
      if deit_name[9:].startswith("base" ):
            pass
      elif deit_name[4:].startswith("large" ):
            SCREAMING_SNAKE_CASE_      =     1_0_2_4
            SCREAMING_SNAKE_CASE_      =     4_0_9_6
            SCREAMING_SNAKE_CASE_      =     2_4
            SCREAMING_SNAKE_CASE_      =     1_6

      # load original model from timm
      SCREAMING_SNAKE_CASE_      =     timm.create_model(__UpperCamelCase   ,					pretrained=__UpperCamelCase )
      timm_model.eval()

      # load state_dict of original model, remove and rename some keys
      SCREAMING_SNAKE_CASE_      =     timm_model.state_dict()
      SCREAMING_SNAKE_CASE_      =     create_rename_keys(__UpperCamelCase   ,					__UpperCamelCase )
      for src, dest in rename_keys:
            rename_key(__UpperCamelCase   ,					__UpperCamelCase   ,					__UpperCamelCase )
      read_in_q_k_v(__UpperCamelCase   ,					__UpperCamelCase   ,					__UpperCamelCase )

      # load HuggingFace model
      SCREAMING_SNAKE_CASE_      =     DeiTForImageClassificationWithTeacher(__UpperCamelCase ).eval()
      model.load_state_dict(__UpperCamelCase )

      # Check outputs on an image, prepared by DeiTImageProcessor
      SCREAMING_SNAKE_CASE_      =     int(
          (2_5_6 / 2_2_4) * config.image_size )  # to maintain same ratio w.r.t. 224 images, see https://github.com/facebookresearch/deit/blob/ab5715372db8c6cad5740714b2216d55aeae052e/datasets.py#L103
      SCREAMING_SNAKE_CASE_      =     DeiTImageProcessor(size=__UpperCamelCase   ,					crop_size=config.image_size )
      SCREAMING_SNAKE_CASE_      =     image_processor(images=prepare_img()   ,					return_tensors="pt" )
      SCREAMING_SNAKE_CASE_      =     encoding["pixel_values"]
      SCREAMING_SNAKE_CASE_      =     model(__UpperCamelCase )

      SCREAMING_SNAKE_CASE_      =     timm_model(__UpperCamelCase )
      assert timm_logits.shape == outputs.logits.shape
      assert torch.allclose(__UpperCamelCase   ,					outputs.logits   ,					atol=1E-3 )

      Path(__UpperCamelCase ).mkdir(exist_ok=__UpperCamelCase )
      print(F'''Saving model {deit_name} to {pytorch_dump_folder_path}''' )
      model.save_pretrained(__UpperCamelCase )
      print(F'''Saving image processor to {pytorch_dump_folder_path}''' )
      image_processor.save_pretrained(__UpperCamelCase )


if __name__ == "__main__":
      A   :     Tuple       =				argparse.ArgumentParser()
      # Required parameters
      parser.add_argument(
          "--deit_name",
          default="vit_deit_base_distilled_patch16_224",
          type=str,
          help="Name of the DeiT timm model you'd like to convert.",
      )
      parser.add_argument(
          "--pytorch_dump_folder_path", default=None, type=str, help="Path to the output PyTorch model directory."
      )

      A   :     Dict       =				parser.parse_args()
      convert_deit_checkpoint(args.deit_name, args.pytorch_dump_folder_path)


