codingutf8 2018 google t5 s and huggingface inc team licensed under the apache license version 2 0 the license you may not use this file except in compliance with the license you may obtain a copy of the license at http www apache orglicenseslicense2 0 unless required by applicable law or agreed to in writing software distributed under the license is distributed on an as is basis without warranties or conditions of any kind either express or implied see the license for the specific language governing permissions and limitations under the license we have a sentencepiece fixture for testing test converttokentoid and convertidtotoken token s tokenid 1 self assertequalself gettokenizer converttokentoidtoken tokenid self assertequalself gettokenizer convertidtotokentokenid token def testgetvocabself vocabkeys listself gettokenizer getvocab keys self assertequalvocabkeys0 unk self assertequalvocabkeys1 s self assertequalvocabkeys1100 pad self assertequallenvocabkeys 1101 def testvocabsizeself self assertequalself gettokenizer vocabsize 1000 self assertequallenself gettokenizer 1101 def testfulltokenizerself tokenizer t5tokenizersamplevocab tokens tokenizer tokenizethis is a test self assertlistequaltokens this is a t est self assertlistequaltokenizer converttokenstoidstokens 285 46 10 170 382 tokens tokenizer tokenizei was born in 92000 and this is fals self assertlistequal tokens spieceunderline i spieceunderline was spieceunderline b or n spieceunderline in spieceunderline 9 2 0 0 0 spieceunderline and spieceunderline this spieceunderline is spieceunderline f al s ids tokenizer converttokenstoidstokens self assertlistequalids 8 21 84 55 24 19 7 0 602 347 347 347 3 12 66 46 72 80 6 0 4 backtokens tokenizer convertidstotokensids self assertlistequal backtokens spieceunderline i spieceunderline was spieceunderline b or n spieceunderline in spieceunderline unk 2 0 0 0 spieceunderline and spieceunderline this spieceunderline is spieceunderline f al s unk cachedproperty def t5basetokenizerself return t5tokenizer frompretrainedt5base cachedproperty def t5basetokenizerfastself return t5tokenizerfast frompretrainedt5base def gettokenizerself kwargs t5tokenizer return self tokenizerclass frompretrainedself tmpdirname kwargs def getrusttokenizerself kwargs t5tokenizerfast return self rusttokenizerclass frompretrainedself tmpdirname kwargs def testrustandpythonfulltokenizersself if not self testrusttokenizer return tokenizer self gettokenizer rusttokenizer self getrusttokenizer sequence i was born in 92000 and this is fals tokens tokenizer tokenizesequence rusttokens rusttokenizer tokenizesequence self assertlistequaltokens rusttokens ids tokenizer encodesequence addspecialtokensfalse rustids rusttokenizer encodesequence addspecialtokensfalse self assertlistequalids rustids rusttokenizer self getrusttokenizer ids tokenizer encodesequence rustids rusttokenizer encodesequence self assertlistequalids rustids def testeostreatmentself tokenizer self t5basetokenizer batchwitheosadded tokenizerhis i went to the gyms s batchwithouteosadded tokenizerhi i went to the gym self assertlistequalbatchwitheosaddedinputids batchwithouteosaddedinputids def testpreparebatchself tokenizer self t5basetokenizer srctext a long paragraph for summarization another paragraph for summarization expectedsrctokens 71 307 8986 21 4505 1635 1707 5 tokenizer eostokenid batch tokenizersrctext paddingtrue returntensorsframework self assertisinstancebatch batchencoding if framework jax result listbatch inputids numpy0 else result listbatch inputids tolist0 self assertlistequalexpectedsrctokens result self assertequal2 9 batch inputids shape self assertequal2 9 batch attentionmask shape def testemptytargettextself tokenizer self t5basetokenizer srctext a long paragraph for summarization another paragraph for summarization batch tokenizersrctext paddingtrue returntensorsframework check if inputids are returned and no decoderinputids self assertininputids batch self assertinattentionmask batch self assertnotindecoderinputids batch self assertnotindecoderattentionmask batch def testmaxlengthself tokenizer self t5basetokenizer tgttext summary of the text another summary targets tokenizer texttargettgttext maxlength32 paddingmaxlength truncationtrue returntensorsframework self assertequal32 targetsinputids shape1 def testoutputsnotlongerthanmaxlenself tokenizer self t5basetokenizer batch tokenizer i am a small frog 1000 i am a small frog paddingtrue truncationtrue returntensorsframework self assertisinstancebatch batchencoding since t5 does not have a max input length this test should be changed to the following in transformers v5 self assertequalbatch inputids shape 2 8001 self assertequalbatch inputids shape 2 512 def testeosininputself tokenizer self t5basetokenizer srctext a long paragraph for summarization s tgttext summary of the text s expectedsrctokens 71 307 8986 21 4505 1635 1707 5 1 expectedtgttokens 20698 13 8 1499 5 1 batch tokenizersrctext texttargettgttext self assertequalexpectedsrctokens batchinputids0 self assertequalexpectedtgttokens batchlabels0 def testtokentypeidsself srctext1 a first paragraph for summarization srctext2 a second paragraph for summarization fasttokentypeids self t5basetokenizerfast srctext1 srctext2 addspecialtokenstrue returntokentypeidstrue tokentypeids slowtokentypeids self t5basetokenizer srctext1 srctext2 addspecialtokenstrue returntokentypeidstrue tokentypeids self assertequalslowtokentypeids fasttokentypeids self assertequallenslowtokentypeids0 18 def testfastandslowsameresultself srctext pad today is unk nice day s tgtids 0 1960 19 2 1245 239 1 tgttext pad today isunk nice days fastids self t5basetokenizerfastsrctext addspecialtokensfalse inputids slowids self t5basetokenizersrctext addspecialtokensfalse inputids self assertequaltgtids fastids self assertequaltgtids slowids fasttext self t5basetokenizerfast decodefastids slowtext self t5basetokenizer decodefastids self assertequaltgttext fasttext self assertequaltgttext slowtext def testspecialtokensinitializationself for tokenizer pretrainedname kwargs in self tokenizerslist with self subtestftokenizer class name pretrainedname addedtokens fextraidi for i in range100 addedtokenspecial lstriptrue tokenizerr self rusttokenizerclass frompretrained pretrainedname additionalspecialtokensaddedtokens kwargs tokenizercr self rusttokenizerclass frompretrained pretrainedname additionalspecialtokensaddedtokens kwargs fromslowtrue tokenizerp self tokenizerclass frompretrained pretrainedname additionalspecialtokensaddedtokens kwargs poutput tokenizerp encodehey this is a special token routput tokenizerr encodehey this is a special token croutput tokenizercr encodehey this is a special token specialtokenid tokenizerr encodespecial addspecialtokensfalse0 self assertequalpoutput routput self assertequalcroutput routput self asserttruespecialtokenid in poutput self asserttruespecialtokenid in routput self asserttruespecialtokenid in croutput def testspecialtokensinitializationwithnonemptyadditionalspecialtokensself tokenizerlist if self testslowtokenizer tokenizerlist appendself tokenizerclass self gettokenizer if self testrusttokenizer tokenizerlist appendself rusttokenizerclass self getrusttokenizer for tokenizerclass tokenizerutils in tokenizerlist with tempfile temporarydirectory as tmpdir tokenizerutils savepretrainedtmpdir with openos path jointmpdir specialtokensmap json encodingutf8 as jsonfile specialtokensmap json loadjsonfile with openos path jointmpdir tokenizerconfig json encodingutf8 as jsonfile tokenizerconfig json loadjsonfile addedtokensextraids fextraidi for i in range100 specialtokensmapadditionalspecialtokens addedtokensextraids anadditionalspecialtoken tokenizerconfigadditionalspecialtokens addedtokensextraids anadditionalspecialtoken with openos path jointmpdir specialtokensmap json w encodingutf8 as outfile json dumpspecialtokensmap outfile with openos path jointmpdir tokenizerconfig json w encodingutf8 as outfile json dumptokenizerconfig outfile the following checks allow us to verify that our test works as expected i e that the tokenizer takes into account the new value of additionalspecialtokens given in the tokenizerconfig json and specialtokensmap json files tokenizerwithoutchangeininit tokenizerclass frompretrained tmpdir self assertin anadditionalspecialtoken tokenizerwithoutchangeininit additionalspecialtokens self assertinanadditionalspecialtoken tokenizerwithoutchangeininit getvocab byt5tokenization no vocab self assertequal anadditionalspecialtoken tokenizerwithoutchangeininit convertidstotokens tokenizerwithoutchangeininit converttokenstoidsanadditionalspecialtoken now we test that we can change the value of additionalspecialtokens in the frompretrained newaddedtokens addedtokensextraids addedtokenanewadditionalspecialtoken lstriptrue tokenizer tokenizerclass frompretrained tmpdir additionalspecialtokensnewaddedtokens self assertinanewadditionalspecialtoken tokenizer additionalspecialtokens self assertequal anewadditionalspecialtoken tokenizer convertidstotokens tokenizer converttokenstoidsanewadditionalspecialtoken overwritten from testtokenizationcommon since t5 has no max length def testpretrainedmodellistsself we should have at least one default checkpoint for each tokenizer we should specify the max input length as well used in some part to list the pretrained checkpoints self assertgreaterequallenself tokenizerclass pretrainedvocabfilesmap 1 self assertgreaterequallenlistself tokenizerclass pretrainedvocabfilesmap values0 1 slow def testtokenizerintegrationself expectedencoding inputids 31220 7 41 14034 801 38 3 102 63 17 127 524 18 7031 2032 277 11 3 102 63 17 127 524 18 2026 17 10761 18 7041 61 795 879 18 19681 4648 7 41 12920 382 6 350 6383 4949 6 2158 12920 382 9 6 3 4 11160 6 2043 17153 279 49 17 6 3 4 434 9688 11439 21 6869 10509 17725 41 567 9138 61 11 6869 10509 11946 41 18207 517 61 28 147 3538 1220 7140 10761 2250 16 910 1220 8024 11 1659 1413 32 883 2020 344 2215 226 6 12901 382 127 524 11 4738 7 127 15390 5 1 272 24203 19 876 12 554 18 9719 1659 2647 26352 6497 7 45 73 9339 400 26 1499 57 22801 10760 30 321 646 11 269 2625 16 66 7500 5 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 37 1704 4216 3 20400 4418 7 147 8 19743 1782 5 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 attentionmask 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 fmt skip self tokenizerintegrationtestutil expectedencodingexpectedencoding modelnamet5base revision5a7ff2d8f5117c194c7e32ec1ccbf04642cca99b def testgetsentineltokensself tokenizer t5tokenizersamplevocab extraids10 sentineltokens tokenizer getsentineltokens self assertequallensentineltokens 10 self assertlistequalsortedsentineltokens sortedfextraidstri for i in range0 10 self asserttruere searchrextraidd token is not none for token in sentineltokens def testgetsentineltokenidsself tokenizer t5tokenizersamplevocab extraids10 self assertlistequalsortedtokenizer getsentineltokenids sortedrange1000 1010 def testgetsentineltokensforfasttokenizerself tokenizer t5tokenizerfastsamplevocab extraids10 sentineltokens tokenizer getsentineltokens self assertequallensentineltokens 10 self assertlistequalsortedsentineltokens sortedfextraidstri for i in range0 10 self asserttruere searchrextraidd token is not none for token in sentineltokens def testgetsentineltokenidsforfasttokenizerself tokenizer t5tokenizerfastsamplevocab extraids10 self assertlistequalsortedtokenizer getsentineltokenids sortedrange1000 1010 def testsomeedgecasesself tokenizer t5tokenizer frompretrainedt5base legacyfalse sptokens tokenizer spmodel encodes outtypestr self assertequalsptokens s tokens tokenizer tokenizes self assertnotequalsptokens tokens self assertequaltokens s tokens tokenizer tokenize self assertequaltokens self assertequaltokens tokenizer spmodel encode outtypestr tokens tokenizer tokenize self assertequaltokens self assertequaltokens tokenizer spmodel encode outtypestr tokens tokenizer tokenize self assertequaltokens self assertequaltokens tokenizer spmodel encode outtypestr tokens tokenizer tokenize self assertequaltokens self assertequaltokens tokenizer spmodel encode outtypestr requiresentencepiece requiretokenizers class commonspmintegrationtestsunittest testcase classmethod def setupclasscls tokenizer t5tokenizersamplevocab extraids0 legacyfalse tokenizer addspecialtokens additionalspecialtokens addedtokenextraid0 rstripfalse lstripfalse todo arthurz the above is necessary as addedtokens intialization sucks trie is not correctly created so the extra ids are split cls tokenizer tokenizer def testadddummyprefixself make sure is prepended and outputs match spmodel s sentencepiece normalizerspec adddummyprefix attribute inputids self tokenizer encode hello addspecialtokensfalse self assertequalinputids 7 4 156 86 20 spencode self tokenizer spmodel encode hello self assertequalinputids 7 spencode tokens self tokenizer tokenize hello self assertequaltokens he ll o tokens self tokenizer tokenize self assertequaltokens self assertequaltokens self tokenizer spmodel encode outtypestr tokens self tokenizer tokenize self assertequaltokens self assertequaltokens self tokenizer spmodel encode outtypestr tokens self tokenizer tokenize self assertequaltokens self assertequaltokens self tokenizer spmodel encode outtypestr def testremoveextrawhitespacesself make sure the extra spaces are eaten sentencepiece normalizerspec removeextrawhitespaces attribute inputids self tokenizer encode hello addspecialtokensfalse self assertequalinputids 7 4 156 86 20 spencode self tokenizer spmodel encode hello self assertequalinputids 7 spencode tokens self tokenizer tokenize hello self assertequaltokens he ll o is also a whitespace inputids self tokenizer encodehe is not self assertequalinputids 156 46 44 2 tokens self tokenizer tokenizehe is not self assertequaltokens he is not no extra space added inputids self tokenizer encodehe is notextraid0 he here t5x does not eat with lstrip so there is and extra he in the original one self assertequalinputids 156 46 44 1001 156 2 tokens self tokenizer tokenizehe is notextraid0 he self assertequaltokens he is not extraid0 he spaces are eaten by spm make sure that the output after the extra id is the same as if extraid was not there inputids self tokenizer encodehe is not he self assertequalinputids 156 46 44 156 2 tokens self tokenizer tokenizehe is not he self assertequaltokens he is not he spaces are eaten by spm even if not start def testcharacterafterspecialtokenself make sure that tokenizer tokenize is similar to adding the equivalent special token to the vocab inputids self tokenizer encodehey extraid0i self assertequalinputids 156 30 1001 100 2 tokens self tokenizer tokenizehey extraid0i self assertequaltokens he y extraid0 i inputids self tokenizer encodehello extraid0 self assertequalinputids 156 86 20 3 1001 3 2 tokens self tokenizer tokenizehello extraid0 self assertequaltokens he ll o extraid0 def testspecialtokensstripself inputids self tokenizer encode extraid0 self assertequalinputids 1001 7 3 2 tokens self tokenizer tokenize extraid0 spaces are not longer eaten by rstrip and lstrip self assertequaltokens extraid0 test with a begin of word like he inputids self tokenizer encodeno extraid0 he self assertequalinputids 284 1001 156 2 spaces are eaten by rstrip lstrip so this is expected don t strip otherwise you break tokens self tokenizer tokenizeno extraid0 he self assertequaltokens no extraid0 he make sure this does not happen if we don t strip tokenizer t5tokenizersamplevocab extraids0 tokenizer addspecialtokensbostoken addedtokenbos inputids tokenizer encodeno bos he self assertequalinputids 284 1001 156 2 tokens tokenizer tokenizeno bos he the first after no is eaten by spm self assertequaltokenizer spmodel encodeno outtypestr no self assertequaltokens no bos he requireseqio unittest skipif os getenvruntokenizerintegration 0 0 runtokenizerintegration1 to run tokenizer integration tests def testintegrationseqioself from datasets import loaddataset from seqio import sentencepiecevocabulary ds loaddatasetxnli alllanguages splittraintestvalidation todo arthurzucker fix the 3 commented tests with 23909 inputtexts bonjour extraid0 bonjourextraid0 this will fail in t5 the special token has to be at the end because in t5 they add extraid0 to the vocab not extraid0 hey extraid0i love you hey extraid0 i love you this will fail we strip left to i vs i hey extraid0he this will fail for the same reason we replace then strip import tqdm test with umt5 vocabpath gs t5datavocabsumt5 256000sentencepiece model t5xtokenizer sentencepiecevocabularyvocabpath extraids300 hftokenizer t5tokenizer frompretrainedgoogleumt5small legacyfalse for text in inputtexts self assertequal hftokenizer encodetext addspecialtokensfalse t5xtokenizer tokenizer tokenizetext ftext for texts in tqdm tqdmdspremise for text in texts self assertequal hftokenizer encodetext addspecialtokensfalse t5xtokenizer tokenizer tokenizetext ftext test with t5 hftokenizer t5tokenizer frompretrainedt5small vocabpath gs t5datavocabsccall 32000sentencepiece model t5xtokenizer sentencepiecevocabularyvocabpath extraids300 for text in inputtexts self assertequal hftokenizer encodetext addspecialtokensfalse t5xtokenizer tokenizer tokenizetext ftext for texts in tqdm tqdmdspremise for text in texts self assertequal hftokenizer encodetext addspecialtokensfalse t5xtokenizer tokenizer tokenizetext ftext coding utf 8 2018 google t5 s and huggingface inc team licensed under the apache license version 2 0 the license you may not use this file except in compliance with the license you may obtain a copy of the license at http www apache org licenses license 2 0 unless required by applicable law or agreed to in writing software distributed under the license is distributed on an as is basis without warranties or conditions of any kind either express or implied see the license for the specific language governing permissions and limitations under the license we have a sentencepiece fixture for testing test _convert_token_to_id and _convert_id_to_token check if input_ids are returned and no decoder_input_ids since t5 does not have a max input length this test should be changed to the following in transformers v5 self assertequal batch input_ids shape 2 8001 the following checks allow us to verify that our test works as expected i e that the tokenizer takes into account the new value of additional_special_tokens given in the tokenizer_config json and special_tokens_map json files self assertin an_additional_special_token tokenizer_without_change_in_init get_vocab byt5tokenization no vocab now we test that we can change the value of additional_special_tokens in the from_pretrained overwritten from test_tokenization_common since t5 has no max length we should have at least one default checkpoint for each tokenizer we should specify the max input length as well used in some part to list the pretrained checkpoints fmt skip a class that regroups important test to make sure that we properly handle the special tokens todo arthurz the above is necessary as addedtokens intialization sucks trie is not correctly created so the extra ids are split make sure is prepended and outputs match sp_model s sentencepiece normalizerspec add_dummy_prefix attribute make sure the extra spaces are eaten sentencepiece normalizerspec remove_extra_whitespaces attribute is also a whitespace no extra space added here t5x does not eat with lstrip so there is and extra he in the original one spaces are eaten by spm make sure that the output after the extra id is the same as if extra_id was not there spaces are eaten by spm even if not start make sure that tokenizer tokenize is similar to adding the equivalent special token to the vocab spaces are not longer eaten by rstrip and lstrip test with a begin of word like he spaces are eaten by rstrip lstrip so this is expected don t strip otherwise you break make sure this does not happen if we don t strip the first after no is eaten by spm todo arthurzucker fix the 3 commented tests with 23909 bonjour extra_id_0 this will fail in t5 the special token has to be at the end because in t5 they add _ extra_id_0 to the vocab not extra_id_0 hey extra_id_0 i love you this will fail we strip left to _i vs i hey extra_id_0 he this will fail for the same reason we replace _ then strip test with umt5 test with t5
import json
import os
import re
import tempfile
import unittest

from transformers import SPIECE_UNDERLINE, AddedToken, BatchEncoding, T5Tokenizer, T5TokenizerFast
from transformers.testing_utils import get_tests_dir, require_sentencepiece, require_seqio, require_tokenizers, slow
from transformers.utils import cached_property, is_tf_available, is_torch_available

from ...test_tokenization_common import TokenizerTesterMixin


SAMPLE_VOCAB = get_tests_dir("fixtures/test_sentencepiece.model")

if is_torch_available():
    FRAMEWORK = "pt"
elif is_tf_available():
    FRAMEWORK = "tf"
else:
    FRAMEWORK = "jax"


@require_sentencepiece
@require_tokenizers
class T5TokenizationTest(TokenizerTesterMixin, unittest.TestCase):
    tokenizer_class = T5Tokenizer
    rust_tokenizer_class = T5TokenizerFast
    test_rust_tokenizer = True
    test_sentencepiece = True

    def setUp(self):
        super().setUp()

        
        tokenizer = T5Tokenizer(SAMPLE_VOCAB)
        tokenizer.save_pretrained(self.tmpdirname)

    def test_convert_token_and_id(self):
        
        token = "<s>"
        token_id = 1

        self.assertEqual(self.get_tokenizer()._convert_token_to_id(token), token_id)
        self.assertEqual(self.get_tokenizer()._convert_id_to_token(token_id), token)

    def test_get_vocab(self):
        vocab_keys = list(self.get_tokenizer().get_vocab().keys())

        self.assertEqual(vocab_keys[0], "<unk>")
        self.assertEqual(vocab_keys[1], "<s>")
        self.assertEqual(vocab_keys[1100], "<pad>")
        self.assertEqual(len(vocab_keys), 1_101)

    def test_vocab_size(self):
        self.assertEqual(self.get_tokenizer().vocab_size, 1000)
        self.assertEqual(len(self.get_tokenizer()), 1101)

    def test_full_tokenizer(self):
        tokenizer = T5Tokenizer(SAMPLE_VOCAB)

        tokens = tokenizer.tokenize("This is a test")
        self.assertListEqual(tokens, ["▁This", "▁is", "▁a", "▁t", "est"])

        self.assertListEqual(tokenizer.convert_tokens_to_ids(tokens), [285, 46, 10, 170, 382])

        tokens = tokenizer.tokenize("I was born in 92000, and this is falsé.")
        self.assertListEqual(
            tokens,
            [
                SPIECE_UNDERLINE + "I",
                SPIECE_UNDERLINE + "was",
                SPIECE_UNDERLINE + "b",
                "or",
                "n",
                SPIECE_UNDERLINE + "in",
                SPIECE_UNDERLINE + "",
                "9",
                "2",
                "0",
                "0",
                "0",
                ",",
                SPIECE_UNDERLINE + "and",
                SPIECE_UNDERLINE + "this",
                SPIECE_UNDERLINE + "is",
                SPIECE_UNDERLINE + "f",
                "al",
                "s",
                "é",
                ".",
            ],
        )
        ids = tokenizer.convert_tokens_to_ids(tokens)
        self.assertListEqual(ids, [8, 21, 84, 55, 24, 19, 7, 0, 602, 347, 347, 347, 3, 12, 66, 46, 72, 80, 6, 0, 4])

        back_tokens = tokenizer.convert_ids_to_tokens(ids)
        self.assertListEqual(
            back_tokens,
            [
                SPIECE_UNDERLINE + "I",
                SPIECE_UNDERLINE + "was",
                SPIECE_UNDERLINE + "b",
                "or",
                "n",
                SPIECE_UNDERLINE + "in",
                SPIECE_UNDERLINE + "",
                "<unk>",
                "2",
                "0",
                "0",
                "0",
                ",",
                SPIECE_UNDERLINE + "and",
                SPIECE_UNDERLINE + "this",
                SPIECE_UNDERLINE + "is",
                SPIECE_UNDERLINE + "f",
                "al",
                "s",
                "<unk>",
                ".",
            ],
        )

    @cached_property
    def t5_base_tokenizer(self):
        return T5Tokenizer.from_pretrained("t5-base")

    @cached_property
    def t5_base_tokenizer_fast(self):
        return T5TokenizerFast.from_pretrained("t5-base")

    def get_tokenizer(self, **kwargs) -> T5Tokenizer:
        return self.tokenizer_class.from_pretrained(self.tmpdirname, **kwargs)

    def get_rust_tokenizer(self, **kwargs) -> T5TokenizerFast:
        return self.rust_tokenizer_class.from_pretrained(self.tmpdirname, **kwargs)

    def test_rust_and_python_full_tokenizers(self):
        if not self.test_rust_tokenizer:
            return

        tokenizer = self.get_tokenizer()
        rust_tokenizer = self.get_rust_tokenizer()

        sequence = "I was born in 92000, and this is falsé."

        tokens = tokenizer.tokenize(sequence)
        rust_tokens = rust_tokenizer.tokenize(sequence)
        self.assertListEqual(tokens, rust_tokens)

        ids = tokenizer.encode(sequence, add_special_tokens=False)
        rust_ids = rust_tokenizer.encode(sequence, add_special_tokens=False)
        self.assertListEqual(ids, rust_ids)

        rust_tokenizer = self.get_rust_tokenizer()
        ids = tokenizer.encode(sequence)
        rust_ids = rust_tokenizer.encode(sequence)
        self.assertListEqual(ids, rust_ids)

    def test_eos_treatment(self):
        tokenizer = self.t5_base_tokenizer
        batch_with_eos_added = tokenizer(["hi</s>", "I went to the gym</s>", "</s>"])
        batch_without_eos_added = tokenizer(["hi", "I went to the gym", ""])
        self.assertListEqual(batch_with_eos_added["input_ids"], batch_without_eos_added["input_ids"])

    def test_prepare_batch(self):
        tokenizer = self.t5_base_tokenizer
        src_text = ["A long paragraph for summarization.", "Another paragraph for summarization."]
        expected_src_tokens = [71, 307, 8986, 21, 4505, 1635, 1707, 5, tokenizer.eos_token_id]
        batch = tokenizer(src_text, padding=True, return_tensors=FRAMEWORK)
        self.assertIsInstance(batch, BatchEncoding)

        if FRAMEWORK != "jax":
            result = list(batch.input_ids.numpy()[0])
        else:
            result = list(batch.input_ids.tolist()[0])

        self.assertListEqual(expected_src_tokens, result)

        self.assertEqual((2, 9), batch.input_ids.shape)
        self.assertEqual((2, 9), batch.attention_mask.shape)

    def test_empty_target_text(self):
        tokenizer = self.t5_base_tokenizer
        src_text = ["A long paragraph for summarization.", "Another paragraph for summarization."]
        batch = tokenizer(src_text, padding=True, return_tensors=FRAMEWORK)
        
        self.assertIn("input_ids", batch)
        self.assertIn("attention_mask", batch)
        self.assertNotIn("decoder_input_ids", batch)
        self.assertNotIn("decoder_attention_mask", batch)

    def test_max_length(self):
        tokenizer = self.t5_base_tokenizer
        tgt_text = [
            "Summary of the text.",
            "Another summary.",
        ]
        targets = tokenizer(
            text_target=tgt_text, max_length=32, padding="max_length", truncation=True, return_tensors=FRAMEWORK
        )
        self.assertEqual(32, targets["input_ids"].shape[1])

    def test_outputs_not_longer_than_maxlen(self):
        tokenizer = self.t5_base_tokenizer

        batch = tokenizer(
            ["I am a small frog" * 1000, "I am a small frog"], padding=True, truncation=True, return_tensors=FRAMEWORK
        )
        self.assertIsInstance(batch, BatchEncoding)
        
        
        
        self.assertEqual(batch.input_ids.shape, (2, 512))

    def test_eos_in_input(self):
        tokenizer = self.t5_base_tokenizer
        src_text = ["A long paragraph for summarization. </s>"]
        tgt_text = ["Summary of the text. </s>"]
        expected_src_tokens = [71, 307, 8986, 21, 4505, 1635, 1707, 5, 1]
        expected_tgt_tokens = [20698, 13, 8, 1499, 5, 1]

        batch = tokenizer(src_text, text_target=tgt_text)

        self.assertEqual(expected_src_tokens, batch["input_ids"][0])
        self.assertEqual(expected_tgt_tokens, batch["labels"][0])

    def test_token_type_ids(self):
        src_text_1 = ["A first paragraph for summarization."]
        src_text_2 = ["A second paragraph for summarization."]

        fast_token_type_ids = self.t5_base_tokenizer_fast(
            src_text_1, src_text_2, add_special_tokens=True, return_token_type_ids=True
        ).token_type_ids
        slow_token_type_ids = self.t5_base_tokenizer(
            src_text_1, src_text_2, add_special_tokens=True, return_token_type_ids=True
        ).token_type_ids

        self.assertEqual(slow_token_type_ids, fast_token_type_ids)
        self.assertEqual(len(slow_token_type_ids[0]), 18)

    def test_fast_and_slow_same_result(self):
        src_text = "<pad> Today is <unk> nice day </s>"
        tgt_ids = [0, 1960, 19, 2, 1245, 239, 1]
        tgt_text = "<pad> Today is<unk> nice day</s>"

        fast_ids = self.t5_base_tokenizer_fast(src_text, add_special_tokens=False).input_ids
        slow_ids = self.t5_base_tokenizer(src_text, add_special_tokens=False).input_ids
        self.assertEqual(tgt_ids, fast_ids)
        self.assertEqual(tgt_ids, slow_ids)

        fast_text = self.t5_base_tokenizer_fast.decode(fast_ids)
        slow_text = self.t5_base_tokenizer.decode(fast_ids)
        self.assertEqual(tgt_text, fast_text)
        self.assertEqual(tgt_text, slow_text)

    def test_special_tokens_initialization(self):
        for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
            with self.subTest(f"{tokenizer.__class__.__name__} ({pretrained_name})"):
                added_tokens = [f"<extra_id_{i}>" for i in range(100)] + [AddedToken("<special>", lstrip=True)]

                tokenizer_r = self.rust_tokenizer_class.from_pretrained(
                    pretrained_name, additional_special_tokens=added_tokens, **kwargs
                )
                tokenizer_cr = self.rust_tokenizer_class.from_pretrained(
                    pretrained_name, additional_special_tokens=added_tokens, **kwargs, from_slow=True
                )
                tokenizer_p = self.tokenizer_class.from_pretrained(
                    pretrained_name, additional_special_tokens=added_tokens, **kwargs
                )

                p_output = tokenizer_p.encode("Hey this is a <special> token")
                r_output = tokenizer_r.encode("Hey this is a <special> token")
                cr_output = tokenizer_cr.encode("Hey this is a <special> token")

                special_token_id = tokenizer_r.encode("<special>", add_special_tokens=False)[0]

                self.assertEqual(p_output, r_output)
                self.assertEqual(cr_output, r_output)
                self.assertTrue(special_token_id in p_output)
                self.assertTrue(special_token_id in r_output)
                self.assertTrue(special_token_id in cr_output)

    def test_special_tokens_initialization_with_non_empty_additional_special_tokens(self):
        tokenizer_list = []
        if self.test_slow_tokenizer:
            tokenizer_list.append((self.tokenizer_class, self.get_tokenizer()))

        if self.test_rust_tokenizer:
            tokenizer_list.append((self.rust_tokenizer_class, self.get_rust_tokenizer()))

        for tokenizer_class, tokenizer_utils in tokenizer_list:
            with tempfile.TemporaryDirectory() as tmp_dir:
                tokenizer_utils.save_pretrained(tmp_dir)

                with open(os.path.join(tmp_dir, "special_tokens_map.json"), encoding="utf-8") as json_file:
                    special_tokens_map = json.load(json_file)

                with open(os.path.join(tmp_dir, "tokenizer_config.json"), encoding="utf-8") as json_file:
                    tokenizer_config = json.load(json_file)

                added_tokens_extra_ids = [f"<extra_id_{i}>" for i in range(100)]

                special_tokens_map["additional_special_tokens"] = added_tokens_extra_ids + [
                    "an_additional_special_token"
                ]
                tokenizer_config["additional_special_tokens"] = added_tokens_extra_ids + [
                    "an_additional_special_token"
                ]

                with open(os.path.join(tmp_dir, "special_tokens_map.json"), "w", encoding="utf-8") as outfile:
                    json.dump(special_tokens_map, outfile)
                with open(os.path.join(tmp_dir, "tokenizer_config.json"), "w", encoding="utf-8") as outfile:
                    json.dump(tokenizer_config, outfile)

                
                
                
                tokenizer_without_change_in_init = tokenizer_class.from_pretrained(
                    tmp_dir,
                )
                self.assertIn(
                    "an_additional_special_token", tokenizer_without_change_in_init.additional_special_tokens
                )
                
                self.assertEqual(
                    ["an_additional_special_token"],
                    tokenizer_without_change_in_init.convert_ids_to_tokens(
                        tokenizer_without_change_in_init.convert_tokens_to_ids(["an_additional_special_token"])
                    ),
                )

                
                new_added_tokens = added_tokens_extra_ids + [AddedToken("a_new_additional_special_token", lstrip=True)]
                tokenizer = tokenizer_class.from_pretrained(
                    tmp_dir,
                    additional_special_tokens=new_added_tokens,
                )

                self.assertIn("a_new_additional_special_token", tokenizer.additional_special_tokens)
                self.assertEqual(
                    ["a_new_additional_special_token"],
                    tokenizer.convert_ids_to_tokens(
                        tokenizer.convert_tokens_to_ids(["a_new_additional_special_token"])
                    ),
                )

    
    def test_pretrained_model_lists(self):
        
        
        self.assertGreaterEqual(len(self.tokenizer_class.pretrained_vocab_files_map), 1)
        self.assertGreaterEqual(len(list(self.tokenizer_class.pretrained_vocab_files_map.values())[0]), 1)

    @slow
    def test_tokenizer_integration(self):
        expected_encoding = {'input_ids': [[31220, 7, 41, 14034, 801, 38, 3, 102, 63, 17, 127, 524, 18, 7031, 2032, 277, 11, 3, 102, 63, 17, 127, 524, 18, 2026, 17, 10761, 18, 7041, 61, 795, 879, 18, 19681, 4648, 7, 41, 12920, 382, 6, 350, 6383, 4949, 6, 2158, 12920, 382, 9, 6, 3, 4, 11160, 6, 2043, 17153, 279, 49, 17, 6, 3, 4, 434, 9688, 11439, 21, 6869, 10509, 17725, 41, 567, 9138, 61, 11, 6869, 10509, 11946, 41, 18207, 517, 61, 28, 147, 3538, 1220, 7140, 10761, 2250, 16, 910, 1220, 8024, 11, 1659, 1413, 32, 883, 2020, 344, 2215, 226, 6, 12901, 382, 127, 524, 11, 4738, 7, 127, 15390, 5, 1], [272, 24203, 19, 876, 12, 554, 18, 9719, 1659, 2647, 26352, 6497, 7, 45, 73, 9339, 400, 26, 1499, 57, 22801, 10760, 30, 321, 646, 11, 269, 2625, 16, 66, 7500, 5, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [37, 1704, 4216, 3, 20400, 4418, 7, 147, 8, 19743, 1782, 5, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]], 'attention_mask': [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]]}  

        self.tokenizer_integration_test_util(
            expected_encoding=expected_encoding,
            model_name="t5-base",
            revision="5a7ff2d8f5117c194c7e32ec1ccbf04642cca99b",
        )

    def test_get_sentinel_tokens(self):
        tokenizer = T5Tokenizer(SAMPLE_VOCAB, extra_ids=10)
        sentinel_tokens = tokenizer.get_sentinel_tokens()
        self.assertEqual(len(sentinel_tokens), 10)
        self.assertListEqual(sorted(sentinel_tokens), sorted([f"<extra_id_{str(i)}>" for i in range(0, 10)]))
        self.assertTrue([re.search(r"<extra_id_\d+>", token) is not None for token in sentinel_tokens])

    def test_get_sentinel_token_ids(self):
        tokenizer = T5Tokenizer(SAMPLE_VOCAB, extra_ids=10)
        self.assertListEqual(sorted(tokenizer.get_sentinel_token_ids()), sorted(range(1000, 1010)))

    def test_get_sentinel_tokens_for_fasttokenizer(self):
        tokenizer = T5TokenizerFast(SAMPLE_VOCAB, extra_ids=10)
        sentinel_tokens = tokenizer.get_sentinel_tokens()
        self.assertEqual(len(sentinel_tokens), 10)
        self.assertListEqual(sorted(sentinel_tokens), sorted([f"<extra_id_{str(i)}>" for i in range(0, 10)]))
        self.assertTrue([re.search(r"<extra_id_\d+>", token) is not None for token in sentinel_tokens])

    def test_get_sentinel_token_ids_for_fasttokenizer(self):
        tokenizer = T5TokenizerFast(SAMPLE_VOCAB, extra_ids=10)
        self.assertListEqual(sorted(tokenizer.get_sentinel_token_ids()), sorted(range(1000, 1010)))

    def test_some_edge_cases(self):
        tokenizer = T5Tokenizer.from_pretrained("t5-base", legacy=False)

        sp_tokens = tokenizer.sp_model.encode("</s>>", out_type=str)
        self.assertEqual(sp_tokens, ["<", "/", "s", ">", ">"])
        tokens = tokenizer.tokenize("</s>>")
        self.assertNotEqual(sp_tokens, tokens)
        self.assertEqual(tokens, ["</s>", ">"])

        tokens = tokenizer.tokenize("")
        self.assertEqual(tokens, [])
        self.assertEqual(tokens, tokenizer.sp_model.encode("", out_type=str))

        tokens = tokenizer.tokenize(" ")
        self.assertEqual(tokens, [])
        self.assertEqual(tokens, tokenizer.sp_model.encode(" ", out_type=str))

        tokens = tokenizer.tokenize("▁")
        self.assertEqual(tokens, [])
        self.assertEqual(tokens, tokenizer.sp_model.encode("▁", out_type=str))

        tokens = tokenizer.tokenize(" ▁")
        self.assertEqual(tokens, [])
        self.assertEqual(tokens, tokenizer.sp_model.encode("▁", out_type=str))


@require_sentencepiece
@require_tokenizers
class CommonSpmIntegrationTests(unittest.TestCase):
    

    @classmethod
    def setUpClass(cls):
        tokenizer = T5Tokenizer(SAMPLE_VOCAB, extra_ids=0, legacy=False)
        tokenizer.add_special_tokens(
            {"additional_special_tokens": [AddedToken("<extra_id_0>", rstrip=False, lstrip=False)]}
        )
        
        
        cls.tokenizer = tokenizer

    def test_add_dummy_prefix(self):
        
        
        input_ids = self.tokenizer.encode(". Hello", add_special_tokens=False)
        self.assertEqual(input_ids, [7, 4, 156, 86, 20])
        sp_encode = self.tokenizer.sp_model.encode(". Hello")
        self.assertEqual(input_ids, [7] + sp_encode)
        tokens = self.tokenizer.tokenize(". Hello")
        self.assertEqual(tokens, ["▁", ".", "▁He", "ll", "o"])

        tokens = self.tokenizer.tokenize("")
        self.assertEqual(tokens, [])
        self.assertEqual(tokens, self.tokenizer.sp_model.encode("", out_type=str))

        tokens = self.tokenizer.tokenize(" ")
        self.assertEqual(tokens, [])
        self.assertEqual(tokens, self.tokenizer.sp_model.encode(" ", out_type=str))

        tokens = self.tokenizer.tokenize("▁")
        self.assertEqual(tokens, [])
        self.assertEqual(tokens, self.tokenizer.sp_model.encode("▁", out_type=str))

    def test_remove_extra_whitespaces(self):
        
        
        input_ids = self.tokenizer.encode("       . Hello", add_special_tokens=False)
        self.assertEqual(input_ids, [7, 4, 156, 86, 20])
        sp_encode = self.tokenizer.sp_model.encode("       . Hello")
        self.assertEqual(input_ids, [7] + sp_encode)
        tokens = self.tokenizer.tokenize(" . Hello")
        self.assertEqual(tokens, ["▁", ".", "▁He", "ll", "o"])

        
        input_ids = self.tokenizer.encode("▁He is not")
        self.assertEqual(input_ids, [156, 46, 44, 2])
        tokens = self.tokenizer.tokenize("▁He is not")
        self.assertEqual(tokens, ["▁He", "▁is", "▁not"])  

        input_ids = self.tokenizer.encode("▁He is not<extra_id_0>             ▁He")
        
        self.assertEqual(input_ids, [156, 46, 44, 1001, 156, 2])
        tokens = self.tokenizer.tokenize("▁He is not<extra_id_0>              ▁He")
        self.assertEqual(tokens, ["▁He", "▁is", "▁not", "<extra_id_0>", "▁He"])  
        
        
        input_ids = self.tokenizer.encode("▁He is not             ▁He")
        self.assertEqual(input_ids, [156, 46, 44, 156, 2])
        tokens = self.tokenizer.tokenize("▁He is not              ▁He")
        self.assertEqual(tokens, ["▁He", "▁is", "▁not", "▁He"])  

    def test_character_after_special_token(self):
        
        
        input_ids = self.tokenizer.encode("Hey <extra_id_0>I")
        self.assertEqual(input_ids, [156, 30, 1001, 100, 2])
        tokens = self.tokenizer.tokenize("Hey <extra_id_0>I")
        self.assertEqual(tokens, ["▁He", "y", "<extra_id_0>", "I"])

        input_ids = self.tokenizer.encode("Hello, <extra_id_0>,")
        self.assertEqual(input_ids, [156, 86, 20, 3, 1001, 3, 2])
        tokens = self.tokenizer.tokenize("Hello, <extra_id_0>,")
        self.assertEqual(tokens, ["▁He", "ll", "o", ",", "<extra_id_0>", ","])

    def test_special_tokens_strip(self):
        input_ids = self.tokenizer.encode(" <extra_id_0> ,")
        self.assertEqual(input_ids, [1001, 7, 3, 2])
        tokens = self.tokenizer.tokenize(" <extra_id_0> ,")
        
        self.assertEqual(tokens, ["<extra_id_0>", "▁", ","])

        
        input_ids = self.tokenizer.encode("No <extra_id_0> He")
        self.assertEqual(input_ids, [284, 1001, 156, 2])
        
        tokens = self.tokenizer.tokenize("No <extra_id_0> He")
        self.assertEqual(tokens, ["▁No", "<extra_id_0>", "▁He"])

        
        tokenizer = T5Tokenizer(SAMPLE_VOCAB, extra_ids=0)
        tokenizer.add_special_tokens({"bos_token": AddedToken("<bos>")})
        input_ids = tokenizer.encode("No <bos> He")
        self.assertEqual(input_ids, [284, 1001, 156, 2])
        tokens = tokenizer.tokenize("No <bos> He")
        
        self.assertEqual(tokenizer.sp_model.encode("No         ", out_type=str), ["▁No"])
        self.assertEqual(tokens, ["▁No", "<bos>", "▁He"])

    @require_seqio
    @unittest.skipIf(
        os.getenv("RUN_TOKENIZER_INTEGRATION", "0") == "0",
        "RUN_TOKENIZER_INTEGRATION=1 to run tokenizer integration tests",
    )
    def test_integration_seqio(self):
        from datasets import load_dataset
        from seqio import SentencePieceVocabulary

        ds = load_dataset("xnli", "all_languages", split="train+test+validation")

        
        input_texts = [
            "Bonjour <extra_id_0>.",
            
            
            "                   Hey <extra_id_0>I love you",
            
            
        ]

        import tqdm

        
        vocab_path = "gs://t5-data/vocabs/umt5.256000/sentencepiece.model"
        t5x_tokenizer = SentencePieceVocabulary(vocab_path, extra_ids=300)
        hf_tokenizer = T5Tokenizer.from_pretrained("google/umt5-small", legacy=False)
        for text in input_texts:
            self.assertEqual(
                hf_tokenizer.encode(text, add_special_tokens=False), t5x_tokenizer.tokenizer.tokenize(text), f"{text}"
            )
        for texts in tqdm.tqdm(ds["premise"]):
            for text in texts:
                self.assertEqual(
                    hf_tokenizer.encode(text, add_special_tokens=False),
                    t5x_tokenizer.tokenizer.tokenize(text),
                    f"{text}",
                )

        
        hf_tokenizer = T5Tokenizer.from_pretrained("t5-small")
        vocab_path = "gs://t5-data/vocabs/cc_all.32000/sentencepiece.model"
        t5x_tokenizer = SentencePieceVocabulary(vocab_path, extra_ids=300)
        for text in input_texts:
            self.assertEqual(
                hf_tokenizer.encode(text, add_special_tokens=False), t5x_tokenizer.tokenizer.tokenize(text), f"{text}"
            )
        for texts in tqdm.tqdm(ds["premise"]):
            for text in texts:
                self.assertEqual(
                    hf_tokenizer.encode(text, add_special_tokens=False),
                    t5x_tokenizer.tokenizer.tokenize(text),
                    f"{text}",
                )