codingutf8 2019 hugging face inc licensed under the apache license version 2 0 the license you may not use this file except in compliance with the license you may obtain a copy of the license at http www apache orglicenseslicense2 0 unless required by applicable law or agreed to in writing software distributed under the license is distributed on an as is basis without warranties or conditions of any kind either express or implied see the license for the specific language governing permissions and limitations under the license we have a sentencepiece fixture for testing test converttokentoid and convertidtotoken token pad tokenid 0 self assertequalself gettokenizer converttokentoidtoken tokenid self assertequalself gettokenizer convertidtotokentokenid token def testgetvocabself vocabkeys listself gettokenizer getvocab keys self assertequalvocabkeys0 pad self assertequalvocabkeys1 unk self assertequalvocabkeys1 eloquent self assertequallenvocabkeys 30000 def testvocabsizeself self assertequalself gettokenizer vocabsize 30000 def testrustandpythonfulltokenizersself if not self testrusttokenizer return tokenizer self gettokenizer rusttokenizer self getrusttokenizer sequence i was born in 92000 and this is fals tokens tokenizer tokenizesequence rusttokens rusttokenizer tokenizesequence self assertlistequaltokens rusttokens ids tokenizer encodesequence addspecialtokensfalse rustids rusttokenizer encodesequence addspecialtokensfalse self assertlistequalids rustids rusttokenizer self getrusttokenizer ids tokenizer encodesequence rustids rusttokenizer encodesequence self assertlistequalids rustids def testfulltokenizerself tokenizer fnettokenizersamplevocab keepaccentstrue tokens tokenizer tokenizethis is a test self assertlistequaltokens t his is a test self assertlistequaltokenizer converttokenstoidstokens 13 1 4398 25 21 1289 tokens tokenizer tokenizei was born in 92000 and this is fals self assertlistequal tokens i was born in 9 2000 and this is fal s ids tokenizer converttokenstoidstokens self assertlistequalids 13 1 23 386 19 561 3050 15 17 48 25 8256 18 1 9 backtokens tokenizer convertidstotokensids self assertlistequal backtokens unk was born in 9 2000 and this is fal s unk def testsequencebuildersself tokenizer fnettokenizersamplevocab text tokenizer encodesequence builders text2 tokenizer encodemultisequence build encodedsentence tokenizer buildinputswithspecialtokenstext encodedpair tokenizer buildinputswithspecialtokenstext text2 assert encodedsentence tokenizer clstokenid text tokenizer septokenid assert encodedpair tokenizer clstokenid text tokenizer septokenid text2 tokenizer septokenid overriden tests loading the fast tokenizer from slow just takes too long def testspecialtokensinitializationself for tokenizer pretrainedname kwargs in self tokenizerslist with self subtestftokenizer class name pretrainedname addedtokens addedtokenspecial lstriptrue tokenizerr self rusttokenizerclass frompretrained pretrainedname additionalspecialtokensaddedtokens kwargs routput tokenizerr encodehey this is a special token specialtokenid tokenizerr encodespecial addspecialtokensfalse0 self asserttruespecialtokenid in routput if self testslowtokenizer tokenizerp self tokenizerclass frompretrained pretrainedname additionalspecialtokensaddedtokens kwargs poutput tokenizerp encodehey this is a special token croutput tokenizerr encodehey this is a special token self assertequalpoutput routput self assertequalcroutput routput self asserttruespecialtokenid in poutput self asserttruespecialtokenid in croutput tooslow def testspecialtokensinitializationfromslowself for tokenizer pretrainedname kwargs in self tokenizerslist with self subtestftokenizer class name pretrainedname addedtokens addedtokenspecial lstriptrue tokenizerr self rusttokenizerclass frompretrained pretrainedname additionalspecialtokensaddedtokens kwargs fromslowtrue specialtokenid tokenizerr encodespecial addspecialtokensfalse0 tokenizerp self tokenizerclass frompretrained pretrainedname additionalspecialtokensaddedtokens kwargs poutput tokenizerp encodehey this is a special token croutput tokenizerr encodehey this is a special token self assertequalpoutput croutput self asserttruespecialtokenid in poutput self asserttruespecialtokenid in croutput overriden tests def testpaddingself maxlength50 if not self testslowtokenizer as we don t have a slow version we can t compare the outputs between slow and fast versions return for tokenizer pretrainedname kwargs in self tokenizerslist with self subtestftokenizer class name pretrainedname tokenizerr self rusttokenizerclass frompretrainedpretrainedname kwargs tokenizerp self tokenizerclass frompretrainedpretrainedname kwargs self assertequaltokenizerp padtokenid tokenizerr padtokenid padtokenid tokenizerp padtokenid encode simple input inputr tokenizerr encodethis is a simple input maxlengthmaxlength padtomaxlengthtrue inputp tokenizerp encodethis is a simple input maxlengthmaxlength padtomaxlengthtrue self assertpaddedinputmatchinputr inputp maxlength padtokenid inputr tokenizerr encodethis is a simple input maxlengthmaxlength paddingmaxlength inputp tokenizerp encodethis is a simple input maxlengthmaxlength paddingmaxlength self assertpaddedinputmatchinputr inputp maxlength padtokenid inputr tokenizerr encodethis is a simple input paddinglongest inputp tokenizerp encodethis is a simple input paddingtrue self assertpaddedinputmatchinputr inputp leninputr padtokenid encode pair input inputr tokenizerr encode this is a simple input this is a pair maxlengthmaxlength padtomaxlengthtrue inputp tokenizerp encode this is a simple input this is a pair maxlengthmaxlength padtomaxlengthtrue self assertpaddedinputmatchinputr inputp maxlength padtokenid inputr tokenizerr encode this is a simple input this is a pair maxlengthmaxlength paddingmaxlength inputp tokenizerp encode this is a simple input this is a pair maxlengthmaxlength paddingmaxlength self assertpaddedinputmatchinputr inputp maxlength padtokenid inputr tokenizerr encodethis is a simple input this is a pair paddingtrue inputp tokenizerp encodethis is a simple input this is a pair paddinglongest self assertpaddedinputmatchinputr inputp leninputr padtokenid encodeplus simple input inputr tokenizerr encodeplus this is a simple input maxlengthmaxlength padtomaxlengthtrue inputp tokenizerp encodeplus this is a simple input maxlengthmaxlength padtomaxlengthtrue self assertpaddedinputmatchinputrinputids inputpinputids maxlength padtokenid inputr tokenizerr encodeplus this is a simple input maxlengthmaxlength paddingmaxlength inputp tokenizerp encodeplus this is a simple input maxlengthmaxlength paddingmaxlength self assertpaddedinputmatchinputrinputids inputpinputids maxlength padtokenid inputr tokenizerr encodeplusthis is a simple input paddinglongest inputp tokenizerp encodeplusthis is a simple input paddingtrue self assertpaddedinputmatch inputrinputids inputpinputids leninputrinputids padtokenid encodeplus pair input inputr tokenizerr encodeplus this is a simple input this is a pair maxlengthmaxlength padtomaxlengthtrue inputp tokenizerp encodeplus this is a simple input this is a pair maxlengthmaxlength padtomaxlengthtrue self assertpaddedinputmatchinputrinputids inputpinputids maxlength padtokenid inputr tokenizerr encodeplus this is a simple input this is a pair maxlengthmaxlength paddingmaxlength inputp tokenizerp encodeplus this is a simple input this is a pair maxlengthmaxlength paddingmaxlength self assertpaddedinputmatchinputrinputids inputpinputids maxlength padtokenid inputr tokenizerr encodeplusthis is a simple input this is a pair paddinglongest inputp tokenizerp encodeplusthis is a simple input this is a pair paddingtrue self assertpaddedinputmatch inputrinputids inputpinputids leninputrinputids padtokenid batchencodeplus simple input inputr tokenizerr batchencodeplus this is a simple input 1 this is a simple input 2 maxlengthmaxlength padtomaxlengthtrue inputp tokenizerp batchencodeplus this is a simple input 1 this is a simple input 2 maxlengthmaxlength padtomaxlengthtrue self assertbatchpaddedinputmatchinputr inputp maxlength padtokenid inputr tokenizerr batchencodeplus this is a simple input 1 this is a simple input 2 maxlengthmaxlength paddingmaxlength inputp tokenizerp batchencodeplus this is a simple input 1 this is a simple input 2 maxlengthmaxlength paddingmaxlength self assertbatchpaddedinputmatchinputr inputp maxlength padtokenid inputr tokenizerr batchencodeplus this is a simple input 1 this is a simple input 2 maxlengthmaxlength paddinglongest inputp tokenizerp batchencodeplus this is a simple input 1 this is a simple input 2 maxlengthmaxlength paddingtrue self assertbatchpaddedinputmatchinputr inputp leninputrinputids0 padtokenid inputr tokenizerr batchencodeplus this is a simple input 1 this is a simple input 2 paddinglongest inputp tokenizerp batchencodeplus this is a simple input 1 this is a simple input 2 paddingtrue self assertbatchpaddedinputmatchinputr inputp leninputrinputids0 padtokenid batchencodeplus pair input inputr tokenizerr batchencodeplus this is a simple input 1 this is a simple input 2 this is a simple pair 1 this is a simple pair 2 maxlengthmaxlength truncationtrue paddingmaxlength inputp tokenizerp batchencodeplus this is a simple input 1 this is a simple input 2 this is a simple pair 1 this is a simple pair 2 maxlengthmaxlength truncationtrue paddingmaxlength self assertbatchpaddedinputmatchinputr inputp maxlength padtokenid inputr tokenizerr batchencodeplus this is a simple input 1 this is a simple input 2 this is a simple pair 1 this is a simple pair 2 paddingtrue inputp tokenizerp batchencodeplus this is a simple input 1 this is a simple input 2 this is a simple pair 1 this is a simple pair 2 paddinglongest self assertbatchpaddedinputmatchinputr inputp leninputrinputids0 padtokenid using pad on single examples after tokenization inputr tokenizerr encodeplusthis is a input 1 inputr tokenizerr padinputr inputp tokenizerr encodeplusthis is a input 1 inputp tokenizerr padinputp self assertpaddedinputmatch inputrinputids inputpinputids leninputrinputids padtokenid using pad on single examples after tokenization inputr tokenizerr encodeplusthis is a input 1 inputr tokenizerr padinputr maxlengthmaxlength paddingmaxlength inputp tokenizerr encodeplusthis is a input 1 inputp tokenizerr padinputp maxlengthmaxlength paddingmaxlength self assertpaddedinputmatchinputrinputids inputpinputids maxlength padtokenid using pad after tokenization inputr tokenizerr batchencodeplus this is a input 1 this is a much longer input whilch should be padded inputr tokenizerr padinputr inputp tokenizerr batchencodeplus this is a input 1 this is a much longer input whilch should be padded inputp tokenizerr padinputp self assertbatchpaddedinputmatchinputr inputp leninputrinputids0 padtokenid using pad after tokenization inputr tokenizerr batchencodeplus this is a input 1 this is a much longer input whilch should be padded inputr tokenizerr padinputr maxlengthmaxlength paddingmaxlength inputp tokenizerr batchencodeplus this is a input 1 this is a much longer input whilch should be padded inputp tokenizerr padinputp maxlengthmaxlength paddingmaxlength self assertbatchpaddedinputmatchinputr inputp maxlength padtokenid slow def testsavepretrainedself super testsavepretrained slow def testsaveslowfromfastandreloadfastself super testsaveslowfromfastandreloadfast def assertbatchpaddedinputmatch self inputr dict inputp dict maxlength int padtokenid int modelmaininputname str inputids for ir in inputr values self assertequallenir 2 self assertequallenir0 maxlength self assertequallenir1 maxlength self assertequallenir 2 self assertequallenir0 maxlength self assertequallenir1 maxlength for ir ip in zipinputrmodelmaininputname inputpmodelmaininputname self assertpaddedinputmatchir ip maxlength padtokenid slow def testtokenizerintegrationself expectedencoding inputids 4 4616 107 163 328 14 63 1726 106 11954 16659 23 83 16688 11427 328 107 36 11954 16659 23 83 16688 6153 82 961 16688 3474 16710 1696 2306 16688 10854 2524 3827 561 163 3474 16680 62 226 2092 16680 379 3474 16660 16680 2436 16667 16671 16680 999 87 3474 16680 2436 16667 5208 800 16710 68 2018 2959 3037 163 16663 11617 16710 36 2018 2959 4737 163 16663 16667 16674 16710 91 372 5087 16745 2205 82 961 3608 38 1770 16745 7984 36 2565 751 9017 1204 864 218 1244 16680 11954 16659 23 83 36 14686 23 7619 16678 5 4 28 532 65 1929 33 391 16688 3979 9 2565 7849 299 225 34 2040 305 167 289 16667 16078 32 1966 181 4626 63 10575 71 851 1491 36 624 4757 38 208 8038 16678 5 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 4 13 1467 5187 26 2521 4567 16664 372 13 16209 3314 16678 5 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 tokentypeids 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 fmt skip self tokenizerintegrationtestutil expectedencodingexpectedencoding modelnamegooglefnetbase revision34219a71ca20e280cc6000b89673a169c65d605c coding utf 8 2019 hugging face inc licensed under the apache license version 2 0 the license you may not use this file except in compliance with the license you may obtain a copy of the license at http www apache org licenses license 2 0 unless required by applicable law or agreed to in writing software distributed under the license is distributed on an as is basis without warranties or conditions of any kind either express or implied see the license for the specific language governing permissions and limitations under the license we have a sentencepiece fixture for testing test _convert_token_to_id and _convert_id_to_token overriden tests loading the fast tokenizer from slow just takes too long overriden tests as we don t have a slow version we can t compare the outputs between slow and fast versions encode simple input encode pair input encode_plus simple input encode_plus pair input batch_encode_plus simple input batch_encode_plus pair input using pad on single examples after tokenization using pad on single examples after tokenization using pad after tokenization using pad after tokenization fmt skip
import unittest

from transformers import FNetTokenizer, FNetTokenizerFast
from transformers.testing_utils import get_tests_dir, require_sentencepiece, require_tokenizers, slow, tooslow
from transformers.tokenization_utils import AddedToken

from ...test_tokenization_common import TokenizerTesterMixin


SAMPLE_VOCAB = get_tests_dir("fixtures/spiece.model")


@require_sentencepiece
@require_tokenizers
class FNetTokenizationTest(TokenizerTesterMixin, unittest.TestCase):
    tokenizer_class = FNetTokenizer
    rust_tokenizer_class = FNetTokenizerFast
    test_rust_tokenizer = True
    test_sentencepiece = True
    test_sentencepiece_ignore_case = True
    test_seq2seq = False

    def setUp(self):
        super().setUp()

        
        tokenizer = FNetTokenizer(SAMPLE_VOCAB)
        tokenizer.save_pretrained(self.tmpdirname)

    def get_input_output_texts(self, tokenizer):
        input_text = "this is a test"
        output_text = "this is a test"
        return input_text, output_text

    def test_convert_token_and_id(self):
        
        token = "<pad>"
        token_id = 0

        self.assertEqual(self.get_tokenizer()._convert_token_to_id(token), token_id)
        self.assertEqual(self.get_tokenizer()._convert_id_to_token(token_id), token)

    def test_get_vocab(self):
        vocab_keys = list(self.get_tokenizer().get_vocab().keys())

        self.assertEqual(vocab_keys[0], "<pad>")
        self.assertEqual(vocab_keys[1], "<unk>")
        self.assertEqual(vocab_keys[-1], "▁eloquent")
        self.assertEqual(len(vocab_keys), 30_000)

    def test_vocab_size(self):
        self.assertEqual(self.get_tokenizer().vocab_size, 30_000)

    def test_rust_and_python_full_tokenizers(self):
        if not self.test_rust_tokenizer:
            return

        tokenizer = self.get_tokenizer()
        rust_tokenizer = self.get_rust_tokenizer()

        sequence = "I was born in 92000, and this is falsé."

        tokens = tokenizer.tokenize(sequence)
        rust_tokens = rust_tokenizer.tokenize(sequence)
        self.assertListEqual(tokens, rust_tokens)

        ids = tokenizer.encode(sequence, add_special_tokens=False)
        rust_ids = rust_tokenizer.encode(sequence, add_special_tokens=False)
        self.assertListEqual(ids, rust_ids)

        rust_tokenizer = self.get_rust_tokenizer()
        ids = tokenizer.encode(sequence)
        rust_ids = rust_tokenizer.encode(sequence)
        self.assertListEqual(ids, rust_ids)

    def test_full_tokenizer(self):
        tokenizer = FNetTokenizer(SAMPLE_VOCAB, keep_accents=True)

        tokens = tokenizer.tokenize("This is a test")
        self.assertListEqual(tokens, ["▁", "T", "his", "▁is", "▁a", "▁test"])

        self.assertListEqual(tokenizer.convert_tokens_to_ids(tokens), [13, 1, 4398, 25, 21, 1289])

        tokens = tokenizer.tokenize("I was born in 92000, and this is falsé.")
        self.assertListEqual(
            tokens,
            ["▁", "I", "▁was", "▁born", "▁in", "▁9", "2000", ",", "▁and", "▁this", "▁is", "▁fal", "s", "é", "."],
        )
        ids = tokenizer.convert_tokens_to_ids(tokens)
        self.assertListEqual(ids, [13, 1, 23, 386, 19, 561, 3050, 15, 17, 48, 25, 8256, 18, 1, 9])

        back_tokens = tokenizer.convert_ids_to_tokens(ids)
        self.assertListEqual(
            back_tokens,
            [
                "▁",
                "<unk>",
                "▁was",
                "▁born",
                "▁in",
                "▁9",
                "2000",
                ",",
                "▁and",
                "▁this",
                "▁is",
                "▁fal",
                "s",
                "<unk>",
                ".",
            ],
        )

    def test_sequence_builders(self):
        tokenizer = FNetTokenizer(SAMPLE_VOCAB)

        text = tokenizer.encode("sequence builders")
        text_2 = tokenizer.encode("multi-sequence build")

        encoded_sentence = tokenizer.build_inputs_with_special_tokens(text)
        encoded_pair = tokenizer.build_inputs_with_special_tokens(text, text_2)

        assert encoded_sentence == [tokenizer.cls_token_id] + text + [tokenizer.sep_token_id]
        assert encoded_pair == [tokenizer.cls_token_id] + text + [tokenizer.sep_token_id] + text_2 + [
            tokenizer.sep_token_id
        ]

    
    def test_special_tokens_initialization(self):
        for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
            with self.subTest(f"{tokenizer.__class__.__name__} ({pretrained_name})"):
                added_tokens = [AddedToken("<special>", lstrip=True)]

                tokenizer_r = self.rust_tokenizer_class.from_pretrained(
                    pretrained_name, additional_special_tokens=added_tokens, **kwargs
                )
                r_output = tokenizer_r.encode("Hey this is a <special> token")

                special_token_id = tokenizer_r.encode("<special>", add_special_tokens=False)[0]

                self.assertTrue(special_token_id in r_output)

                if self.test_slow_tokenizer:
                    tokenizer_p = self.tokenizer_class.from_pretrained(
                        pretrained_name, additional_special_tokens=added_tokens, **kwargs
                    )

                    p_output = tokenizer_p.encode("Hey this is a <special> token")

                    cr_output = tokenizer_r.encode("Hey this is a <special> token")

                    self.assertEqual(p_output, r_output)
                    self.assertEqual(cr_output, r_output)
                    self.assertTrue(special_token_id in p_output)
                    self.assertTrue(special_token_id in cr_output)

    @tooslow
    def test_special_tokens_initialization_from_slow(self):
        for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
            with self.subTest(f"{tokenizer.__class__.__name__} ({pretrained_name})"):
                added_tokens = [AddedToken("<special>", lstrip=True)]
                tokenizer_r = self.rust_tokenizer_class.from_pretrained(
                    pretrained_name, additional_special_tokens=added_tokens, **kwargs, from_slow=True
                )
                special_token_id = tokenizer_r.encode("<special>", add_special_tokens=False)[0]
                tokenizer_p = self.tokenizer_class.from_pretrained(
                    pretrained_name, additional_special_tokens=added_tokens, **kwargs
                )

                p_output = tokenizer_p.encode("Hey this is a <special> token")
                cr_output = tokenizer_r.encode("Hey this is a <special> token")

                self.assertEqual(p_output, cr_output)
                self.assertTrue(special_token_id in p_output)
                self.assertTrue(special_token_id in cr_output)

    
    def test_padding(self, max_length=50):
        if not self.test_slow_tokenizer:
            
            return

        for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
            with self.subTest(f"{tokenizer.__class__.__name__} ({pretrained_name})"):
                tokenizer_r = self.rust_tokenizer_class.from_pretrained(pretrained_name, **kwargs)
                tokenizer_p = self.tokenizer_class.from_pretrained(pretrained_name, **kwargs)

                self.assertEqual(tokenizer_p.pad_token_id, tokenizer_r.pad_token_id)
                pad_token_id = tokenizer_p.pad_token_id

                
                input_r = tokenizer_r.encode("This is a simple input", max_length=max_length, pad_to_max_length=True)
                input_p = tokenizer_p.encode("This is a simple input", max_length=max_length, pad_to_max_length=True)
                self.assert_padded_input_match(input_r, input_p, max_length, pad_token_id)
                input_r = tokenizer_r.encode("This is a simple input", max_length=max_length, padding="max_length")
                input_p = tokenizer_p.encode("This is a simple input", max_length=max_length, padding="max_length")
                self.assert_padded_input_match(input_r, input_p, max_length, pad_token_id)

                input_r = tokenizer_r.encode("This is a simple input", padding="longest")
                input_p = tokenizer_p.encode("This is a simple input", padding=True)
                self.assert_padded_input_match(input_r, input_p, len(input_r), pad_token_id)

                
                input_r = tokenizer_r.encode(
                    "This is a simple input", "This is a pair", max_length=max_length, pad_to_max_length=True
                )
                input_p = tokenizer_p.encode(
                    "This is a simple input", "This is a pair", max_length=max_length, pad_to_max_length=True
                )
                self.assert_padded_input_match(input_r, input_p, max_length, pad_token_id)
                input_r = tokenizer_r.encode(
                    "This is a simple input", "This is a pair", max_length=max_length, padding="max_length"
                )
                input_p = tokenizer_p.encode(
                    "This is a simple input", "This is a pair", max_length=max_length, padding="max_length"
                )
                self.assert_padded_input_match(input_r, input_p, max_length, pad_token_id)
                input_r = tokenizer_r.encode("This is a simple input", "This is a pair", padding=True)
                input_p = tokenizer_p.encode("This is a simple input", "This is a pair", padding="longest")
                self.assert_padded_input_match(input_r, input_p, len(input_r), pad_token_id)

                
                input_r = tokenizer_r.encode_plus(
                    "This is a simple input", max_length=max_length, pad_to_max_length=True
                )
                input_p = tokenizer_p.encode_plus(
                    "This is a simple input", max_length=max_length, pad_to_max_length=True
                )
                self.assert_padded_input_match(input_r["input_ids"], input_p["input_ids"], max_length, pad_token_id)

                input_r = tokenizer_r.encode_plus(
                    "This is a simple input", max_length=max_length, padding="max_length"
                )
                input_p = tokenizer_p.encode_plus(
                    "This is a simple input", max_length=max_length, padding="max_length"
                )
                self.assert_padded_input_match(input_r["input_ids"], input_p["input_ids"], max_length, pad_token_id)

                input_r = tokenizer_r.encode_plus("This is a simple input", padding="longest")
                input_p = tokenizer_p.encode_plus("This is a simple input", padding=True)
                self.assert_padded_input_match(
                    input_r["input_ids"], input_p["input_ids"], len(input_r["input_ids"]), pad_token_id
                )

                
                input_r = tokenizer_r.encode_plus(
                    "This is a simple input", "This is a pair", max_length=max_length, pad_to_max_length=True
                )
                input_p = tokenizer_p.encode_plus(
                    "This is a simple input", "This is a pair", max_length=max_length, pad_to_max_length=True
                )
                self.assert_padded_input_match(input_r["input_ids"], input_p["input_ids"], max_length, pad_token_id)

                input_r = tokenizer_r.encode_plus(
                    "This is a simple input", "This is a pair", max_length=max_length, padding="max_length"
                )
                input_p = tokenizer_p.encode_plus(
                    "This is a simple input", "This is a pair", max_length=max_length, padding="max_length"
                )
                self.assert_padded_input_match(input_r["input_ids"], input_p["input_ids"], max_length, pad_token_id)

                input_r = tokenizer_r.encode_plus("This is a simple input", "This is a pair", padding="longest")
                input_p = tokenizer_p.encode_plus("This is a simple input", "This is a pair", padding=True)
                self.assert_padded_input_match(
                    input_r["input_ids"], input_p["input_ids"], len(input_r["input_ids"]), pad_token_id
                )

                
                input_r = tokenizer_r.batch_encode_plus(
                    ["This is a simple input 1", "This is a simple input 2"],
                    max_length=max_length,
                    pad_to_max_length=True,
                )
                input_p = tokenizer_p.batch_encode_plus(
                    ["This is a simple input 1", "This is a simple input 2"],
                    max_length=max_length,
                    pad_to_max_length=True,
                )
                self.assert_batch_padded_input_match(input_r, input_p, max_length, pad_token_id)

                input_r = tokenizer_r.batch_encode_plus(
                    ["This is a simple input 1", "This is a simple input 2"],
                    max_length=max_length,
                    padding="max_length",
                )
                input_p = tokenizer_p.batch_encode_plus(
                    ["This is a simple input 1", "This is a simple input 2"],
                    max_length=max_length,
                    padding="max_length",
                )
                self.assert_batch_padded_input_match(input_r, input_p, max_length, pad_token_id)

                input_r = tokenizer_r.batch_encode_plus(
                    ["This is a simple input 1", "This is a simple input 2"],
                    max_length=max_length,
                    padding="longest",
                )
                input_p = tokenizer_p.batch_encode_plus(
                    ["This is a simple input 1", "This is a simple input 2"],
                    max_length=max_length,
                    padding=True,
                )
                self.assert_batch_padded_input_match(input_r, input_p, len(input_r["input_ids"][0]), pad_token_id)

                input_r = tokenizer_r.batch_encode_plus(
                    ["This is a simple input 1", "This is a simple input 2"], padding="longest"
                )
                input_p = tokenizer_p.batch_encode_plus(
                    ["This is a simple input 1", "This is a simple input 2"], padding=True
                )
                self.assert_batch_padded_input_match(input_r, input_p, len(input_r["input_ids"][0]), pad_token_id)

                
                input_r = tokenizer_r.batch_encode_plus(
                    [
                        ("This is a simple input 1", "This is a simple input 2"),
                        ("This is a simple pair 1", "This is a simple pair 2"),
                    ],
                    max_length=max_length,
                    truncation=True,
                    padding="max_length",
                )
                input_p = tokenizer_p.batch_encode_plus(
                    [
                        ("This is a simple input 1", "This is a simple input 2"),
                        ("This is a simple pair 1", "This is a simple pair 2"),
                    ],
                    max_length=max_length,
                    truncation=True,
                    padding="max_length",
                )
                self.assert_batch_padded_input_match(input_r, input_p, max_length, pad_token_id)

                input_r = tokenizer_r.batch_encode_plus(
                    [
                        ("This is a simple input 1", "This is a simple input 2"),
                        ("This is a simple pair 1", "This is a simple pair 2"),
                    ],
                    padding=True,
                )
                input_p = tokenizer_p.batch_encode_plus(
                    [
                        ("This is a simple input 1", "This is a simple input 2"),
                        ("This is a simple pair 1", "This is a simple pair 2"),
                    ],
                    padding="longest",
                )
                self.assert_batch_padded_input_match(input_r, input_p, len(input_r["input_ids"][0]), pad_token_id)

                
                input_r = tokenizer_r.encode_plus("This is a input 1")
                input_r = tokenizer_r.pad(input_r)

                input_p = tokenizer_r.encode_plus("This is a input 1")
                input_p = tokenizer_r.pad(input_p)

                self.assert_padded_input_match(
                    input_r["input_ids"], input_p["input_ids"], len(input_r["input_ids"]), pad_token_id
                )

                
                input_r = tokenizer_r.encode_plus("This is a input 1")
                input_r = tokenizer_r.pad(input_r, max_length=max_length, padding="max_length")

                input_p = tokenizer_r.encode_plus("This is a input 1")
                input_p = tokenizer_r.pad(input_p, max_length=max_length, padding="max_length")

                self.assert_padded_input_match(input_r["input_ids"], input_p["input_ids"], max_length, pad_token_id)

                
                input_r = tokenizer_r.batch_encode_plus(
                    ["This is a input 1", "This is a much longer input whilch should be padded"]
                )
                input_r = tokenizer_r.pad(input_r)

                input_p = tokenizer_r.batch_encode_plus(
                    ["This is a input 1", "This is a much longer input whilch should be padded"]
                )
                input_p = tokenizer_r.pad(input_p)

                self.assert_batch_padded_input_match(input_r, input_p, len(input_r["input_ids"][0]), pad_token_id)

                
                input_r = tokenizer_r.batch_encode_plus(
                    ["This is a input 1", "This is a much longer input whilch should be padded"]
                )
                input_r = tokenizer_r.pad(input_r, max_length=max_length, padding="max_length")

                input_p = tokenizer_r.batch_encode_plus(
                    ["This is a input 1", "This is a much longer input whilch should be padded"]
                )
                input_p = tokenizer_r.pad(input_p, max_length=max_length, padding="max_length")

                self.assert_batch_padded_input_match(input_r, input_p, max_length, pad_token_id)

    @slow
    def test_save_pretrained(self):
        super().test_save_pretrained()

    @slow
    def test_save_slow_from_fast_and_reload_fast(self):
        super().test_save_slow_from_fast_and_reload_fast()

    def assert_batch_padded_input_match(
        self,
        input_r: dict,
        input_p: dict,
        max_length: int,
        pad_token_id: int,
        model_main_input_name: str = "input_ids",
    ):
        for i_r in input_r.values():
            (
                self.assertEqual(len(i_r), 2),
                self.assertEqual(len(i_r[0]), max_length),
                self.assertEqual(len(i_r[1]), max_length),
            )
            (
                self.assertEqual(len(i_r), 2),
                self.assertEqual(len(i_r[0]), max_length),
                self.assertEqual(len(i_r[1]), max_length),
            )

        for i_r, i_p in zip(input_r[model_main_input_name], input_p[model_main_input_name]):
            self.assert_padded_input_match(i_r, i_p, max_length, pad_token_id)

    @slow
    def test_tokenizer_integration(self):
        expected_encoding = {'input_ids': [[4, 4616, 107, 163, 328, 14, 63, 1726, 106, 11954, 16659, 23, 83, 16688, 11427, 328, 107, 36, 11954, 16659, 23, 83, 16688, 6153, 82, 961, 16688, 3474, 16710, 1696, 2306, 16688, 10854, 2524, 3827, 561, 163, 3474, 16680, 62, 226, 2092, 16680, 379, 3474, 16660, 16680, 2436, 16667, 16671, 16680, 999, 87, 3474, 16680, 2436, 16667, 5208, 800, 16710, 68, 2018, 2959, 3037, 163, 16663, 11617, 16710, 36, 2018, 2959, 4737, 163, 16663, 16667, 16674, 16710, 91, 372, 5087, 16745, 2205, 82, 961, 3608, 38, 1770, 16745, 7984, 36, 2565, 751, 9017, 1204, 864, 218, 1244, 16680, 11954, 16659, 23, 83, 36, 14686, 23, 7619, 16678, 5], [4, 28, 532, 65, 1929, 33, 391, 16688, 3979, 9, 2565, 7849, 299, 225, 34, 2040, 305, 167, 289, 16667, 16078, 32, 1966, 181, 4626, 63, 10575, 71, 851, 1491, 36, 624, 4757, 38, 208, 8038, 16678, 5, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3], [4, 13, 1467, 5187, 26, 2521, 4567, 16664, 372, 13, 16209, 3314, 16678, 5, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3]], 'token_type_ids': [[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]]}  

        self.tokenizer_integration_test_util(
            expected_encoding=expected_encoding,
            model_name="google/fnet-base",
            revision="34219a71ca20e280cc6000b89673a169c65d605c",
        )