2022 the huggingface team licensed under the apache license version 2 0 the license you may not use this file except in compliance with the license you may obtain a copy of the license at http www apache orglicenseslicense2 0 unless required by applicable law or agreed to in writing software distributed under the license is distributed on an as is basis without warranties or conditions of any kind either express or implied see the license for the specific language governing permissions and limitations under the license test converttokentoid and convertidtotoken token where tokenid 14436 self assertequalself gettokenizer converttokentoidtoken tokenid self assertequalself gettokenizer convertidtotokentokenid token def testgetvocabself vocabkeys listself gettokenizer getvocab keys self assertequalvocabkeys0 self assertequalvocabkeys1 self assertequalvocabkeys1 30 00 self assertequallenvocabkeys 51865 def testvocabsizeself self assertequalself gettokenizer vocabsize 50258 def testfulltokenizerself tokenizer whispertokenizer frompretrainedself tmpdirname tokens tokenizer tokenizethis is a test self assertlistequaltokens this is a test self assertlistequal tokenizer converttokenstoidstokens 5723 307 257 220 31636 tokens tokenizer tokenizei was born in 92000 and this is fals self assertlistequal tokens i was born in 9 2000 and this is fals fmt skip fmt skip ids tokenizer converttokenstoidstokens self assertlistequalids 40 390 4232 294 1722 25743 11 293 220 11176 307 16720 526 13 backtokens tokenizer convertidstotokensids self assertlistequal backtokens i was born in 9 2000 and this is fals fmt skip fmt skip def testtokenizerslowstorefullsignatureself pass def testtokenizerfaststorefullsignatureself pass def testspecialtokensinitializationself whisper relies on specific additional special tokens so we skip this general test in particular this test loads fast tokenizer from slow tokenizer and the conversion uses prefixtokens where we reference additional special tokens by specific indices hence overriding the list with less tokens leads to out of index error pass slow def testtokenizerintegrationself expectedencoding inputids 50257 50362 41762 364 357 36234 1900 355 12972 13165 354 12 35636 364 290 12972 13165 354 12 5310 13363 12 4835 8 3769 2276 12 29983 45619 357 13246 51 11 402 11571 12 17 11 5564 13246 38586 11 16276 44 11 4307 346 33 861 11 16276 7934 23029 329 12068 15417 28491 357 32572 52 8 290 12068 15417 16588 357 32572 38 8 351 625 3933 10 2181 13363 4981 287 1802 10 8950 290 2769 48817 1799 1022 449 897 11 9485 15884 354 290 309 22854 37535 13 50256 50257 50362 13246 51 318 3562 284 662 12 27432 2769 8406 4154 282 24612 422 9642 9608 276 2420 416 26913 21143 319 1111 1364 290 826 4732 287 477 11685 13 50256 50257 50362 464 2068 7586 21831 18045 625 262 16931 3290 13 50256 attentionmask 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 fmt skip self tokenizerintegrationtestutil expectedencodingexpectedencoding modelnameopenaiwhispertiny en paddingfalse def testoutputoffsetsself tokenizer self gettokenizer previoussequence 51492 406 3163 1953 466 13 51612 51612 self assertequal tokenizer decodeprevioussequence outputoffsetstrue text not worth thinking about offsets text not worth thinking about timestamp 22 56 24 96 merge when the previous sequence is a suffix of the next sequence nextsequences1 50364 295 6177 3391 11 19817 3337 507 307 406 3163 1953 466 13 50614 50614 2812 9836 14783 390 6263 538 257 1359 11 8199 6327 1090 322 702 7443 13 50834 50257 fmt skip self assertequal tokenizer decodenextsequences1 outputoffsetstrue text of spectators retrievality is not worth thinking about his instant panic was followed by a small sharp blow high on his chest endoftext offsets text of spectators retrievality is not worth thinking about timestamp 0 0 5 0 text his instant panic was followed by a small sharp blow high on his chest timestamp 5 0 9 4 def testfindlongestcommonsubsequenceself previoussequence 1 2 3 nextsequence 2 3 4 5 merge findlongestcommonsequenceprevioussequence nextsequence self assertequalmerge 1 2 3 4 5 now previous is larger than next we merge what we can and remove the extra right side of the left sequence previoussequence 1 2 3 4 5 6 7 nextsequence 2 3 4 5 merge findlongestcommonsequenceprevioussequence nextsequence self assertequalmerge 1 2 3 4 5 nothing in common previoussequence 1 2 3 nextsequence 4 5 6 merge findlongestcommonsequenceprevioussequence nextsequence self assertequalmerge 1 2 3 4 5 6 some errors in the overlap we take from previous on the left from the next on the right of the overlap previoussequence 1 2 3 4 99 nextsequence 2 98 4 5 6 merge findlongestcommonsequenceprevioussequence nextsequence self assertequalmerge 1 2 3 4 5 6 we take from previous on the left from the next on the right of the overlap previoussequence 1 2 99 4 5 nextsequence 2 3 4 98 6 merge findlongestcommonsequenceprevioussequence nextsequence self assertequalmerge 1 2 99 4 98 6 this works on 3 sequences seq1 1 2 3 seq2 2 3 4 seq3 3 4 5 merge findlongestcommonsequenceseq1 seq2 seq3 self assertequalmerge 1 2 3 4 5 this works on 3 sequences with errors seq1 1 2 3 98 5 seq2 2 99 4 5 6 7 seq3 4 97 6 7 8 merge findlongestcommonsequenceseq1 seq2 seq3 self assertequalmerge 1 2 3 4 5 6 7 8 def testskipspecialtokensskipspromptidsself tokenizer self gettokenizer rusttokenizer self getrusttokenizer fmt off encodedinput 50361 2221 13 2326 388 391 50258 50259 50359 50363 1282 264 2674 9156 295 1523 11 2221 13 2326 388 391 13657 365 2681 21296 17711 13 50257 fmt on expectedwithspecialtokens startofprev mr quilterstartoftranscriptentranscribenotimestamps on the general principles of art mr quilter writes with equal lucidity endoftext expectedwithoutspecialtokens on the general principles of art mr quilter writes with equal lucidity self assertequaltokenizer decodeencodedinput skipspecialtokensfalse expectedwithspecialtokens self assertequaltokenizer decodeencodedinput skipspecialtokenstrue expectedwithoutspecialtokens self assertequalrusttokenizer decodeencodedinput skipspecialtokensfalse expectedwithspecialtokens self assertequal rusttokenizer decodeencodedinput skipspecialtokenstrue expectedwithoutspecialtokens def testskipspecialtokenswithtimestampsself tokenizer self gettokenizer rusttokenizer self getrusttokenizer fmt off encodedinput 50258 50363 50364 634 575 12525 22618 1968 6144 35617 20084 1756 311 589 307 534 10281 934 439 293 50676 50676 393 4411 294 309 457 707 295 33301 286 392 6628 13 50836 50257 fmt on expectedwithspecialtokens startoftranscriptnotimestamps0 00 he has grave doubts whether sir frederick layton s work is really greek after all and6 246 24 can discover in it but little of rocky ithaca 9 44endoftext expectedwithoutspecialtokens 0 00 he has grave doubts whether sir frederick layton s work is really greek after all and6 246 24 can discover in it but little of rocky ithaca 9 44 self assertequal tokenizer decodeencodedinput decodewithtimestampstrue skipspecialtokensfalse expectedwithspecialtokens self assertequal tokenizer decodeencodedinput decodewithtimestampstrue skipspecialtokenstrue expectedwithoutspecialtokens self assertequal rusttokenizer decodeencodedinput decodewithtimestampstrue skipspecialtokensfalse expectedwithspecialtokens self assertequal rusttokenizer decodeencodedinput decodewithtimestampstrue skipspecialtokenstrue expectedwithoutspecialtokens def testfasttokenizergetpromptidsself tokenizer self gettokenizer rusttokenizer self getrusttokenizer prompt this is test prompt text tokenizerpromptids tokenizer getpromptidsprompt fasttokenizerpromptids rusttokenizer getpromptidsprompt self assertlistequaltokenizerpromptids tolist fasttokenizerpromptids tolist def testcombinetokensintowordsself tokenizer self gettokenizer rusttokenizer self getrusttokenizer whatever whatever said someone clever encodedinput 1363 7969 503 1363 7969 1 848 1580 11 13494 7323 expectedwords whatever whatever said someone clever expectedtokens 1363 7969 503 1363 7969 1 848 1580 11 13494 7323 expectedindices 0 1 2 3 4 5 6 7 8 9 10 output combinetokensintowordstokenizer encodedinput self assertequalexpectedwords output0 self assertequalexpectedtokens output1 self assertequalexpectedindices output2 outputrust combinetokensintowordsrusttokenizer encodedinput self assertequalexpectedwords outputrust0 self assertequalexpectedtokens outputrust1 self assertequalexpectedindices outputrust2 def testbasicnormalizerself tokenizer self gettokenizer rusttokenizer self getrusttokenizer inputstr hola gey expectedoutputnormalize hola gey expectedoutputdiacritics hola guey tokenizer tests encodedinput tokenizerinputstr inputids decodedoutput tokenizer decodeencodedinput skipspecialtokenstrue basicnormalizefalse self assertequaldecodedoutput inputstr decodedoutputnormalize tokenizer decodeencodedinput skipspecialtokenstrue basicnormalizetrue self assertequaldecodedoutputnormalize expectedoutputnormalize decodedoutputdiacritics tokenizer decode encodedinput skipspecialtokenstrue basicnormalizetrue removediacriticstrue self assertequaldecodedoutputdiacritics expectedoutputdiacritics fast tokenizer tests encodedinput rusttokenizerinputstr inputids decodedoutput rusttokenizer decodeencodedinput skipspecialtokenstrue basicnormalizefalse self assertequaldecodedoutput inputstr decodedoutputnormalize rusttokenizer decodeencodedinput skipspecialtokenstrue basicnormalizetrue self assertequaldecodedoutputnormalize expectedoutputnormalize decodedoutputdiacritics rusttokenizer decode encodedinput skipspecialtokenstrue basicnormalizetrue removediacriticstrue self assertequaldecodedoutputdiacritics expectedoutputdiacritics class speechtotexttokenizermultilinguialtestunittest testcase checkpointname openaiwhispersmall en classmethod def setupclasscls cls tokenizer whispertokenizer whispertokenizer frompretrainedcls checkpointname return cls def testtokenizerequivalenceself text multilingualtokenizer whispertokenizer frompretrainedopenaiwhispertiny languagekorean monolingualtokenizer whispertokenizer frompretrainedopenaiwhispertiny en monolingualtokens monolingualtokenizer encodetext addspecialtokensfalse multilingualtokens multilingualtokenizer encodetext addspecialtokensfalse assert monolingualtokenizer decodemonolingualtokens text assert multilingualtokenizer decodemultilingualtokens text assert lenmonolingualtokens lenmultilingualtokens fmt off expectedeng 46695 97 167 252 234 168 98 238 220 169 245 234 23821 111 229 167 108 242 169 222 112 168 245 238 220 169 225 222 166 111 254 169 234 234 expectedmulti 9835 22855 168 98 238 13431 234 43517 229 47053 169 222 19086 19840 1313 17974 fmt on self assertlistequalmonolingualtokens expectedeng self assertlistequalmultilingualtokens expectedmulti def testtokenizerspecialself multilingualtokenizer whispertokenizer frompretrained openaiwhispertiny languageenglish tasktranscribe text hey how are you feeling j ai l impression que est prt multilingualtokens multilingualtokenizer encodetext fmt off format startoftranscript langid task notimestamps transcription ids endoftext expectedmulti startoftranscript encode transcribe notimestamps 7057 0 1012 366 291 2633 30 508 6 1301 287 6 36107 631 220 11178 115 15567 871 44393 endoftranscript expectedspecialtext startoftranscriptentranscribenotimestampshey how are you feeling j ai l impression que est prtendoftext fmt on self assertlistequalmultilingualtokens expectedmulti specialtranscript multilingualtokenizer decodemultilingualtokens skipspecialtokensfalse self assertequalspecialtranscript expectedspecialtext transcript multilingualtokenizer decodemultilingualtokens skipspecialtokenstrue self assertequaltranscript text def testvocabsizeself self assertequalself tokenizer vocabsize 50257 copied from tests models speechtotext testtokenizationspeechtotext speechtotexttokenizermultilinguialtest testtokenizerdecodeignoreslanguagecodes def testtokenizerdecodeignoreslanguagecodesself self assertinescode self tokenizer allspecialids generatedids escode 4 1601 47 7647 2 result self tokenizer decodegeneratedids skipspecialtokenstrue expectedspanish self tokenizer decodegeneratedids1 skipspecialtokenstrue self assertequalresult expectedspanish self assertnotinself tokenizer eostoken result def testbatchencodingself multilingualtokenizer whispertokenizer frompretrained openaiwhispertiny languagespanish tasktranslate batch el gato el gato se sent batchoutput multilingualtokenizer batchencodeplusbatch paddingtrue inputids fmt off expectedmulti startoftranscript escode translate notimestamps 17356 290 2513 220 endoftranscript endoftranscript endoftranscript startoftranscript escode translate notimestamps 17356 290 2513 369 2279 812 endoftranscript fmt on self assertlistequalbatchoutput expectedmulti def testsetprefixtokensself multilingualtokenizer whispertokenizer frompretrained openaiwhispertiny languagespanish tasktranslate change the language prefix token from spanish to english multilingualtokenizer setprefixtokenslanguageenglish batch the cat the cat sat batchoutput multilingualtokenizer batchencodeplusbatch paddingtrue inputids fmt off expectedmulti startoftranscript encode translate notimestamps 3322 3857 endoftranscript endoftranscript startoftranscript encode translate notimestamps 3322 3857 3227 endoftranscript fmt on self assertlistequalbatchoutput expectedmulti def testbatchencodingdecodingself multilingualtokenizer whispertokenizer frompretrainedopenaiwhispertiny languagespanish batch hola gey que onda batchencoding multilingualtokenizer batchencodeplusbatch paddingtrue inputids transcription multilingualtokenizer batchdecodebatchencoding skipspecialtokenstrue self assertlistequalbatch transcription def testoffsetdecodingself multilingualtokenizer whispertokenizer frompretrainedopenaiwhispertiny fmt off inputtokens 50258 50259 50359 50364 441 1857 4174 11 5242 366 257 1333 295 493 2794 2287 293 12018 14880 11 293 25730 311 454 34152 4496 904 50724 50724 366 382 4048 382 257 361 18459 13065 13 2221 13 7145 74 325 38756 311 29822 7563 412 472 709 294 264 51122 51122 912 636 300 2221 13 2741 5767 1143 281 7319 702 7798 13 400 2221 13 2619 4004 811 2709 702 51449 51449 50257 fmt on output multilingualtokenizer decodeinputtokens outputoffsetstrueoffsets self assertequal output text lennils pictures are a sort of upguards and atom paintings and mason s exquisite idles timestamp 0 0 7 2 text are as national as a jingo poem mr birkut foster s landscapes smile at one much in the timestamp 7 2 15 16 text same way that mr carker used to flash his teeth and mr john colier gives his timestamp 15 16 21 7 test decodewithoffsets output multilingualtokenizer decodeinputtokens decodewithtimestampstrue self assertequal output startoftranscriptentranscribe0 00 lennils pictures are a sort of upguards and atom paintings and mason s exquisite idles7 207 20 are as national as a jingo poem mr birkut foster s landscapes smile at one much in the15 1615 16 same way that mr carker used to flash his teeth and mr john colier gives his21 7021 70endoftext test a single sequence with timestamps fmt off inputtokens 50364 441 1857 4174 11 5242 366 257 1333 295 493 2794 2287 293 12018 14880 11 293 25730 311 454 34152 4496 904 50724 fmt on output multilingualtokenizer decodeinputtokens outputoffsetstrueoffsets self assertequal output0 text lennils pictures are a sort of upguards and atom paintings and mason s exquisite idles timestamp 0 0 7 2 test a sequence without a single timestamps fmt off inputtokens 441 1857 4174 11 5242 366 257 1333 295 493 2794 2287 293 12018 14880 11 293 25730 311 454 34152 4496 904 50724 fmt on output multilingualtokenizer decodeinputtokens outputoffsetstrueoffsets self assertequaloutput requirejinja def testtokenizationforchatself multilingualtokenizer whispertokenizer frompretrainedopenaiwhispertiny this is in english but it s just here to make sure the chat control tokens are being added properly testchats role system content you are a helpful chatbot role user content hello role system content you are a helpful chatbot role user content hello role assistant content nice to meet you role assistant content nice to meet you role user content hello tokenizedchats multilingualtokenizer applychattemplatetestchat for testchat in testchats expectedtokens 3223 366 257 4961 5081 18870 13 50257 15947 0 50257 3223 366 257 4961 5081 18870 13 50257 15947 0 50257 37717 220 1353 1677 291 13 50257 37717 220 1353 1677 291 13 50257 15947 0 50257 for tokenizedchat expectedtokens in ziptokenizedchats expectedtokens self assertlistequaltokenizedchat expectedtokens 2022 the huggingface team licensed under the apache license version 2 0 the license you may not use this file except in compliance with the license you may obtain a copy of the license at http www apache org licenses license 2 0 unless required by applicable law or agreed to in writing software distributed under the license is distributed on an as is basis without warranties or conditions of any kind either express or implied see the license for the specific language governing permissions and limitations under the license test _convert_token_to_id and _convert_id_to_token fmt skip fmt skip fmt skip fmt skip whisper relies on specific additional special tokens so we skip this general test in particular this test loads fast tokenizer from slow tokenizer and the conversion uses prefix_tokens where we reference additional special tokens by specific indices hence overriding the list with less tokens leads to out of index error fmt skip merge when the previous sequence is a suffix of the next sequence fmt skip now previous is larger than next we merge what we can and remove the extra right side of the left sequence nothing in common some errors in the overlap we take from previous on the left from the next on the right of the overlap we take from previous on the left from the next on the right of the overlap this works on 3 sequences this works on 3 sequences with errors fmt off fmt on fmt off fmt on whatever whatever said someone clever tokenizer tests fast tokenizer tests fmt off fmt on fmt off format startoftranscript lang id task notimestamps transcription ids endoftext fmt on copied from tests models speech_to_text test_tokenization_speech_to_text speechtotexttokenizermultilinguialtest test_tokenizer_decode_ignores_language_codes fmt off fmt on change the language prefix token from spanish to english fmt off fmt on fmt off fmt on test decode_with_offsets test a single sequence with timestamps fmt off fmt on test a sequence without a single timestamps fmt off fmt on this is in english but it s just here to make sure the chat control tokens are being added properly
import unittest

from transformers.models.whisper import WhisperTokenizer, WhisperTokenizerFast
from transformers.models.whisper.tokenization_whisper import _combine_tokens_into_words, _find_longest_common_sequence
from transformers.testing_utils import require_jinja, slow

from ...test_tokenization_common import TokenizerTesterMixin


ES_CODE = 50262
EN_CODE = 50259
END_OF_TRANSCRIPT = 50257
START_OF_TRANSCRIPT = 50258
TRANSLATE = 50358
TRANSCRIBE = 50359
NOTIMESTAMPS = 50363


class WhisperTokenizerTest(TokenizerTesterMixin, unittest.TestCase):
    tokenizer_class = WhisperTokenizer
    rust_tokenizer_class = WhisperTokenizerFast
    test_rust_tokenizer = True
    test_sentencepiece = False
    test_seq2seq = False

    def setUp(self):
        super().setUp()
        tokenizer = WhisperTokenizer.from_pretrained("openai/whisper-tiny")
        tokenizer.pad_token_id = 50256
        tokenizer.pad_token = "<|endoftext|>"
        tokenizer.save_pretrained(self.tmpdirname)

    def test_convert_token_and_id(self):
        
        token = "Where"
        token_id = 14436

        self.assertEqual(self.get_tokenizer()._convert_token_to_id(token), token_id)
        self.assertEqual(self.get_tokenizer()._convert_id_to_token(token_id), token)

    def test_get_vocab(self):
        vocab_keys = list(self.get_tokenizer().get_vocab().keys())

        self.assertEqual(vocab_keys[0], "!")
        self.assertEqual(vocab_keys[1], '"')
        self.assertEqual(vocab_keys[-1], "<|30.00|>")
        self.assertEqual(len(vocab_keys), 51865)

    def test_vocab_size(self):
        self.assertEqual(self.get_tokenizer().vocab_size, 50258)

    def test_full_tokenizer(self):
        tokenizer = WhisperTokenizer.from_pretrained(self.tmpdirname)

        tokens = tokenizer.tokenize("This is a test")
        self.assertListEqual(tokens, ["This", "Ġis", "Ġa", "Ġ", "test"])

        self.assertListEqual(
            tokenizer.convert_tokens_to_ids(tokens),
            [5723, 307, 257, 220, 31636],
        )

        tokens = tokenizer.tokenize("I was born in 92000, and this is falsé.")
        self.assertListEqual(
            tokens,
            ["I", "Ġwas", "Ġborn", "Ġin", "Ġ9", "2000", ",", "Ġand", "Ġ", "this", "Ġis", "Ġfals", "Ã©", "."],  
        )  
        ids = tokenizer.convert_tokens_to_ids(tokens)
        self.assertListEqual(ids, [40, 390, 4232, 294, 1722, 25743, 11, 293, 220, 11176, 307, 16720, 526, 13])

        back_tokens = tokenizer.convert_ids_to_tokens(ids)
        self.assertListEqual(
            back_tokens,
            ["I", "Ġwas", "Ġborn", "Ġin", "Ġ9", "2000", ",", "Ġand", "Ġ", "this", "Ġis", "Ġfals", "Ã©", "."],  
        )  

    def test_tokenizer_slow_store_full_signature(self):
        pass

    def test_tokenizer_fast_store_full_signature(self):
        pass

    def test_special_tokens_initialization(self):
        
        
        
        
        
        pass

    @slow
    def test_tokenizer_integration(self):
        expected_encoding = {'input_ids': [[50257, 50362, 41762, 364, 357, 36234, 1900, 355, 12972, 13165, 354, 12, 35636, 364, 290, 12972, 13165, 354, 12, 5310, 13363, 12, 4835, 8, 3769, 2276, 12, 29983, 45619, 357, 13246, 51, 11, 402, 11571, 12, 17, 11, 5564, 13246, 38586, 11, 16276, 44, 11, 4307, 346, 33, 861, 11, 16276, 7934, 23029, 329, 12068, 15417, 28491, 357, 32572, 52, 8, 290, 12068, 15417, 16588, 357, 32572, 38, 8, 351, 625, 3933, 10, 2181, 13363, 4981, 287, 1802, 10, 8950, 290, 2769, 48817, 1799, 1022, 449, 897, 11, 9485, 15884, 354, 290, 309, 22854, 37535, 13, 50256], [50257, 50362, 13246, 51, 318, 3562, 284, 662, 12, 27432, 2769, 8406, 4154, 282, 24612, 422, 9642, 9608, 276, 2420, 416, 26913, 21143, 319, 1111, 1364, 290, 826, 4732, 287, 477, 11685, 13, 50256], [50257, 50362, 464, 2068, 7586, 21831, 18045, 625, 262, 16931, 3290, 13, 50256]], 'attention_mask': [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]]}  

        self.tokenizer_integration_test_util(
            expected_encoding=expected_encoding, model_name="openai/whisper-tiny.en", padding=False
        )

    def test_output_offsets(self):
        tokenizer = self.get_tokenizer()
        previous_sequence = [51492, 406, 3163, 1953, 466, 13, 51612, 51612]
        self.assertEqual(
            tokenizer.decode(previous_sequence, output_offsets=True),
            {
                "text": " not worth thinking about.",
                "offsets": [{"text": " not worth thinking about.", "timestamp": (22.56, 24.96)}],
            },
        )

        
        next_sequences_1 = [50364, 295, 6177, 3391, 11, 19817, 3337, 507, 307, 406, 3163, 1953, 466, 13, 50614, 50614, 2812, 9836, 14783, 390, 6263, 538, 257, 1359, 11, 8199, 6327, 1090, 322, 702, 7443, 13, 50834, 50257]  
        self.assertEqual(
            tokenizer.decode(next_sequences_1, output_offsets=True),
            {
                "text": (
                    " of spectators, retrievality is not worth thinking about. His instant panic was followed by a"
                    " small, sharp blow high on his chest.<|endoftext|>"
                ),
                "offsets": [
                    {"text": " of spectators, retrievality is not worth thinking about.", "timestamp": (0.0, 5.0)},
                    {
                        "text": " His instant panic was followed by a small, sharp blow high on his chest.",
                        "timestamp": (5.0, 9.4),
                    },
                ],
            },
        )

    def test_find_longest_common_subsequence(self):
        previous_sequence = [1, 2, 3]
        next_sequence = [2, 3, 4, 5]
        merge = _find_longest_common_sequence([previous_sequence, next_sequence])
        self.assertEqual(merge, [1, 2, 3, 4, 5])

        
        
        previous_sequence = [1, 2, 3, 4, 5, 6, 7]
        next_sequence = [2, 3, 4, 5]
        merge = _find_longest_common_sequence([previous_sequence, next_sequence])
        self.assertEqual(merge, [1, 2, 3, 4, 5])

        
        previous_sequence = [1, 2, 3]
        next_sequence = [4, 5, 6]
        merge = _find_longest_common_sequence([previous_sequence, next_sequence])
        self.assertEqual(merge, [1, 2, 3, 4, 5, 6])

        
        
        previous_sequence = [1, 2, 3, 4, 99]
        next_sequence = [2, 98, 4, 5, 6]
        merge = _find_longest_common_sequence([previous_sequence, next_sequence])
        self.assertEqual(merge, [1, 2, 3, 4, 5, 6])

        
        previous_sequence = [1, 2, 99, 4, 5]
        next_sequence = [2, 3, 4, 98, 6]
        merge = _find_longest_common_sequence([previous_sequence, next_sequence])
        self.assertEqual(merge, [1, 2, 99, 4, 98, 6])

        
        seq1 = [1, 2, 3]
        seq2 = [2, 3, 4]
        seq3 = [3, 4, 5]
        merge = _find_longest_common_sequence([seq1, seq2, seq3])
        self.assertEqual(merge, [1, 2, 3, 4, 5])

        
        seq1 = [1, 2, 3, 98, 5]
        seq2 = [2, 99, 4, 5, 6, 7]
        seq3 = [4, 97, 6, 7, 8]
        merge = _find_longest_common_sequence([seq1, seq2, seq3])
        self.assertEqual(merge, [1, 2, 3, 4, 5, 6, 7, 8])

    def test_skip_special_tokens_skips_prompt_ids(self):
        tokenizer = self.get_tokenizer()
        rust_tokenizer = self.get_rust_tokenizer()
        
        encoded_input = [
            50361, 2221, 13, 2326, 388, 391, 50258, 50259, 50359,
            50363, 1282, 264, 2674, 9156, 295, 1523, 11, 2221, 13,
            2326, 388, 391, 13657, 365, 2681, 21296, 17711, 13, 50257,
        ]
        
        expected_with_special_tokens = "<|startofprev|> Mr. Quilter<|startoftranscript|><|en|><|transcribe|><|notimestamps|> On the general principles of art, Mr. Quilter writes with equal lucidity.<|endoftext|>"
        expected_without_special_tokens = " On the general principles of art, Mr. Quilter writes with equal lucidity."
        self.assertEqual(tokenizer.decode(encoded_input, skip_special_tokens=False), expected_with_special_tokens)
        self.assertEqual(tokenizer.decode(encoded_input, skip_special_tokens=True), expected_without_special_tokens)
        self.assertEqual(rust_tokenizer.decode(encoded_input, skip_special_tokens=False), expected_with_special_tokens)
        self.assertEqual(
            rust_tokenizer.decode(encoded_input, skip_special_tokens=True), expected_without_special_tokens
        )

    def test_skip_special_tokens_with_timestamps(self):
        tokenizer = self.get_tokenizer()
        rust_tokenizer = self.get_rust_tokenizer()

        
        encoded_input = [
            50258, 50363, 50364, 634, 575, 12525, 22618, 1968, 6144,
            35617, 20084, 1756, 311, 589, 307, 534, 10281, 934,
            439, 293, 50676, 50676, 393, 4411, 294, 309, 457,
            707, 295, 33301, 286, 392, 6628, 13, 50836, 50257,
        ]
        

        expected_with_special_tokens = "<|startoftranscript|><|notimestamps|><|0.00|> He has grave doubts whether Sir Frederick Layton's work is really Greek after all and<|6.24|><|6.24|> can discover in it but little of rocky Ithaca.<|9.44|><|endoftext|>"
        expected_without_special_tokens = "<|0.00|> He has grave doubts whether Sir Frederick Layton's work is really Greek after all and<|6.24|><|6.24|> can discover in it but little of rocky Ithaca.<|9.44|>"
        self.assertEqual(
            tokenizer.decode(encoded_input, decode_with_timestamps=True, skip_special_tokens=False),
            expected_with_special_tokens,
        )
        self.assertEqual(
            tokenizer.decode(encoded_input, decode_with_timestamps=True, skip_special_tokens=True),
            expected_without_special_tokens,
        )
        self.assertEqual(
            rust_tokenizer.decode(encoded_input, decode_with_timestamps=True, skip_special_tokens=False),
            expected_with_special_tokens,
        )
        self.assertEqual(
            rust_tokenizer.decode(encoded_input, decode_with_timestamps=True, skip_special_tokens=True),
            expected_without_special_tokens,
        )

    def test_fast_tokenizer_get_prompt_ids(self):
        tokenizer = self.get_tokenizer()
        rust_tokenizer = self.get_rust_tokenizer()

        prompt = "This is test prompt text."
        tokenizer_prompt_ids = tokenizer.get_prompt_ids(prompt)
        fast_tokenizer_prompt_ids = rust_tokenizer.get_prompt_ids(prompt)

        self.assertListEqual(tokenizer_prompt_ids.tolist(), fast_tokenizer_prompt_ids.tolist())

    def test_combine_tokens_into_words(self):
        tokenizer = self.get_tokenizer()
        rust_tokenizer = self.get_rust_tokenizer()

        
        encoded_input = [1363, 7969, 503, 1363, 7969, 1, 848, 1580, 11, 13494, 7323]
        expected_words = ["whatever", ' "whatever"', " said", " someone,", " clever!?"]
        expected_tokens = [[1363, 7969], [503, 1363, 7969, 1], [848], [1580, 11], [13494, 7323]]
        expected_indices = [[0, 1], [2, 3, 4, 5], [6], [7, 8], [9, 10]]
        output = _combine_tokens_into_words(tokenizer, encoded_input)
        self.assertEqual(expected_words, output[0])
        self.assertEqual(expected_tokens, output[1])
        self.assertEqual(expected_indices, output[2])
        output_rust = _combine_tokens_into_words(rust_tokenizer, encoded_input)
        self.assertEqual(expected_words, output_rust[0])
        self.assertEqual(expected_tokens, output_rust[1])
        self.assertEqual(expected_indices, output_rust[2])

    def test_basic_normalizer(self):
        tokenizer = self.get_tokenizer()
        rust_tokenizer = self.get_rust_tokenizer()

        input_str = "Hola güey!"
        expected_output_normalize = "hola güey "
        expected_output_diacritics = "hola guey "

        
        encoded_input = tokenizer(input_str).input_ids
        decoded_output = tokenizer.decode(encoded_input, skip_special_tokens=True, basic_normalize=False)
        self.assertEqual(decoded_output, input_str)

        decoded_output_normalize = tokenizer.decode(encoded_input, skip_special_tokens=True, basic_normalize=True)
        self.assertEqual(decoded_output_normalize, expected_output_normalize)

        decoded_output_diacritics = tokenizer.decode(
            encoded_input, skip_special_tokens=True, basic_normalize=True, remove_diacritics=True
        )
        self.assertEqual(decoded_output_diacritics, expected_output_diacritics)

        
        encoded_input = rust_tokenizer(input_str).input_ids
        decoded_output = rust_tokenizer.decode(encoded_input, skip_special_tokens=True, basic_normalize=False)
        self.assertEqual(decoded_output, input_str)

        decoded_output_normalize = rust_tokenizer.decode(encoded_input, skip_special_tokens=True, basic_normalize=True)
        self.assertEqual(decoded_output_normalize, expected_output_normalize)

        decoded_output_diacritics = rust_tokenizer.decode(
            encoded_input, skip_special_tokens=True, basic_normalize=True, remove_diacritics=True
        )
        self.assertEqual(decoded_output_diacritics, expected_output_diacritics)


class SpeechToTextTokenizerMultilinguialTest(unittest.TestCase):
    checkpoint_name = "openai/whisper-small.en"

    @classmethod
    def setUpClass(cls):
        cls.tokenizer: WhisperTokenizer = WhisperTokenizer.from_pretrained(cls.checkpoint_name)
        return cls

    def test_tokenizer_equivalence(self):
        text = "다람쥐 헌 쳇바퀴에 타고파"
        multilingual_tokenizer = WhisperTokenizer.from_pretrained("openai/whisper-tiny", language="korean")
        monolingual_tokenizer = WhisperTokenizer.from_pretrained("openai/whisper-tiny.en")

        monolingual_tokens = monolingual_tokenizer.encode(text, add_special_tokens=False)
        multilingual_tokens = multilingual_tokenizer.encode(text, add_special_tokens=False)

        assert monolingual_tokenizer.decode(monolingual_tokens) == text
        assert multilingual_tokenizer.decode(multilingual_tokens) == text
        assert len(monolingual_tokens) > len(multilingual_tokens)

        
        EXPECTED_ENG = [
            46695, 97, 167, 252, 234, 168, 98, 238, 220, 169,
            245, 234, 23821, 111, 229, 167, 108, 242, 169, 222,
            112, 168, 245, 238, 220, 169, 225, 222, 166, 111,
            254, 169, 234, 234
        ]
        EXPECTED_MULTI = [
            9835, 22855, 168, 98, 238, 13431, 234, 43517, 229, 47053,
            169, 222, 19086, 19840, 1313, 17974
        ]
        

        self.assertListEqual(monolingual_tokens, EXPECTED_ENG)
        self.assertListEqual(multilingual_tokens, EXPECTED_MULTI)

    def test_tokenizer_special(self):
        multilingual_tokenizer = WhisperTokenizer.from_pretrained(
            "openai/whisper-tiny", language="english", task="transcribe"
        )
        text = "Hey! How are you feeling? J'ai l'impression que 郷さん est prêt"

        multilingual_tokens = multilingual_tokenizer.encode(text)

        
        
        EXPECTED_MULTI = [
            START_OF_TRANSCRIPT, EN_CODE, TRANSCRIBE, NOTIMESTAMPS, 7057, 0, 1012, 366, 291,
            2633, 30, 508, 6, 1301, 287, 6, 36107, 631, 220, 11178,
            115, 15567, 871, 44393, END_OF_TRANSCRIPT
        ]
        EXPECTED_SPECIAL_TEXT = (
            "<|startoftranscript|><|en|><|transcribe|><|notimestamps|>Hey! How are you feeling? "
            "J'ai l'impression que 郷さん est prêt<|endoftext|>"
        )
        

        self.assertListEqual(multilingual_tokens, EXPECTED_MULTI)

        special_transcript = multilingual_tokenizer.decode(multilingual_tokens, skip_special_tokens=False)
        self.assertEqual(special_transcript, EXPECTED_SPECIAL_TEXT)

        transcript = multilingual_tokenizer.decode(multilingual_tokens, skip_special_tokens=True)
        self.assertEqual(transcript, text)

    def test_vocab_size(self):
        self.assertEqual(self.tokenizer.vocab_size, 50257)

    
    def test_tokenizer_decode_ignores_language_codes(self):
        self.assertIn(ES_CODE, self.tokenizer.all_special_ids)
        generated_ids = [ES_CODE, 4, 1601, 47, 7647, 2]
        result = self.tokenizer.decode(generated_ids, skip_special_tokens=True)
        expected_spanish = self.tokenizer.decode(generated_ids[1:], skip_special_tokens=True)
        self.assertEqual(result, expected_spanish)
        self.assertNotIn(self.tokenizer.eos_token, result)

    def test_batch_encoding(self):
        multilingual_tokenizer = WhisperTokenizer.from_pretrained(
            "openai/whisper-tiny", language="spanish", task="translate"
        )
        batch = ["El gato ", "El gato se sentó"]
        batch_output = multilingual_tokenizer.batch_encode_plus(batch, padding=True).input_ids

        
        EXPECTED_MULTI = [
            [START_OF_TRANSCRIPT, ES_CODE, TRANSLATE, NOTIMESTAMPS, 17356, 290, 2513, 220,
             END_OF_TRANSCRIPT, END_OF_TRANSCRIPT, END_OF_TRANSCRIPT],
            [START_OF_TRANSCRIPT, ES_CODE, TRANSLATE, NOTIMESTAMPS, 17356, 290, 2513, 369,
             2279, 812, END_OF_TRANSCRIPT]
        ]
        

        self.assertListEqual(batch_output, EXPECTED_MULTI)

    def test_set_prefix_tokens(self):
        multilingual_tokenizer = WhisperTokenizer.from_pretrained(
            "openai/whisper-tiny", language="spanish", task="translate"
        )

        
        multilingual_tokenizer.set_prefix_tokens(language="english")

        batch = ["the cat", "the cat sat"]
        batch_output = multilingual_tokenizer.batch_encode_plus(batch, padding=True).input_ids

        
        EXPECTED_MULTI = [
            [START_OF_TRANSCRIPT, EN_CODE, TRANSLATE, NOTIMESTAMPS, 3322, 3857,
             END_OF_TRANSCRIPT, END_OF_TRANSCRIPT],
            [START_OF_TRANSCRIPT, EN_CODE, TRANSLATE, NOTIMESTAMPS, 3322, 3857,
             3227, END_OF_TRANSCRIPT]
        ]
        

        self.assertListEqual(batch_output, EXPECTED_MULTI)

    def test_batch_encoding_decoding(self):
        multilingual_tokenizer = WhisperTokenizer.from_pretrained("openai/whisper-tiny", language="spanish")
        batch = ["hola güey", "que onda"]
        batch_encoding = multilingual_tokenizer.batch_encode_plus(batch, padding=True).input_ids
        transcription = multilingual_tokenizer.batch_decode(batch_encoding, skip_special_tokens=True)
        self.assertListEqual(batch, transcription)

    def test_offset_decoding(self):
        multilingual_tokenizer = WhisperTokenizer.from_pretrained("openai/whisper-tiny")
        
        INPUT_TOKENS = [
            50258, 50259, 50359, 50364, 441, 1857, 4174, 11, 5242, 366,
            257, 1333, 295, 493, 2794, 2287, 293, 12018, 14880, 11,
            293, 25730, 311, 454, 34152, 4496, 904, 50724, 50724, 366,
            382, 4048, 382, 257, 361, 18459, 13065, 13, 2221, 13,
            7145, 74, 325, 38756, 311, 29822, 7563, 412, 472, 709,
            294, 264, 51122, 51122, 912, 636, 300, 2221, 13, 2741,
            5767, 1143, 281, 7319, 702, 7798, 13, 400, 2221, 13,
            2619, 4004, 811, 2709, 702, 51449, 51449, 50257
        ]
        
        output = multilingual_tokenizer.decode(INPUT_TOKENS, output_offsets=True)["offsets"]

        self.assertEqual(
            output,
            [
                {
                    "text": (
                        " Lennils, pictures are a sort of upguards and atom paintings, and Mason's exquisite idles"
                    ),
                    "timestamp": (0.0, 7.2),
                },
                {
                    "text": (
                        " are as national as a jingo poem. Mr. Birkut Foster's landscapes smile at one much in the"
                    ),
                    "timestamp": (7.2, 15.16),
                },
                {
                    "text": " same way that Mr. Carker used to flash his teeth. And Mr. John Colier gives his",
                    "timestamp": (15.16, 21.7),
                },
            ],
        )
        
        output = multilingual_tokenizer.decode(INPUT_TOKENS, decode_with_timestamps=True)
        self.assertEqual(
            output,
            "<|startoftranscript|><|en|><|transcribe|><|0.00|> Lennils, pictures are a sort of upguards and atom"
            " paintings, and Mason's exquisite idles<|7.20|><|7.20|> are as national as a jingo poem. Mr. Birkut"
            " Foster's landscapes smile at one much in the<|15.16|><|15.16|> same way that Mr. Carker used to flash"
            " his teeth. And Mr. John Colier gives his<|21.70|><|21.70|><|endoftext|>",
        )
        
        
        INPUT_TOKENS = [
            50364, 441, 1857, 4174, 11, 5242, 366,
            257, 1333, 295, 493, 2794, 2287, 293, 12018, 14880, 11,
            293, 25730, 311, 454, 34152, 4496, 904, 50724
        ]
        

        output = multilingual_tokenizer.decode(INPUT_TOKENS, output_offsets=True)["offsets"]
        self.assertEqual(
            output[0],
            {
                "text": " Lennils, pictures are a sort of upguards and atom paintings, and Mason's exquisite idles",
                "timestamp": (0.0, 7.2),
            },
        )

        
        
        INPUT_TOKENS = [
            441, 1857, 4174, 11, 5242, 366,
            257, 1333, 295, 493, 2794, 2287, 293, 12018, 14880, 11,
            293, 25730, 311, 454, 34152, 4496, 904, 50724
        ]
        

        output = multilingual_tokenizer.decode(INPUT_TOKENS, output_offsets=True)["offsets"]
        self.assertEqual(output, [])

    @require_jinja
    def test_tokenization_for_chat(self):
        multilingual_tokenizer = WhisperTokenizer.from_pretrained("openai/whisper-tiny")
        
        test_chats = [
            [{"role": "system", "content": "You are a helpful chatbot."}, {"role": "user", "content": "Hello!"}],
            [
                {"role": "system", "content": "You are a helpful chatbot."},
                {"role": "user", "content": "Hello!"},
                {"role": "assistant", "content": "Nice to meet you."},
            ],
            [{"role": "assistant", "content": "Nice to meet you."}, {"role": "user", "content": "Hello!"}],
        ]
        tokenized_chats = [multilingual_tokenizer.apply_chat_template(test_chat) for test_chat in test_chats]
        expected_tokens = [
            [3223, 366, 257, 4961, 5081, 18870, 13, 50257, 15947, 0, 50257],
            [3223, 366, 257, 4961, 5081, 18870, 13, 50257, 15947, 0, 50257, 37717, 220, 1353, 1677, 291, 13, 50257],
            [37717, 220, 1353, 1677, 291, 13, 50257, 15947, 0, 50257],
        ]
        for tokenized_chat, expected_tokens in zip(tokenized_chats, expected_tokens):
            self.assertListEqual(tokenized_chat, expected_tokens)