codingutf8 2020 the huggingface team licensed under the apache license version 2 0 the license you may not use this file except in compliance with the license you may obtain a copy of the license at http www apache orglicenseslicense2 0 unless required by applicable law or agreed to in writing software distributed under the license is distributed on an as is basis without warranties or conditions of any kind either express or implied see the license for the specific language governing permissions and limitations under the license we have a sentencepiece fixture for testing test converttokentoid and convertidtotoken token pad tokenid 1 self assertequalself gettokenizer converttokentoidtoken tokenid self assertequalself gettokenizer convertidtotokentokenid token def testgetvocabself vocabkeys listself gettokenizer getvocab keys self assertequalvocabkeys0 s self assertequalvocabkeys1 pad self assertequalvocabkeys1 mask self assertequallenvocabkeys 1002 def testvocabsizeself self assertequalself gettokenizer vocabsize 1002 def testfulltokenizerself tokenizer xlmrobertatokenizersamplevocab keepaccentstrue tokens tokenizer tokenizethis is a test self assertlistequaltokens this is a t est self assertlistequal tokenizer converttokenstoidstokens value tokenizer fairseqoffset for value in 285 46 10 170 382 tokens tokenizer tokenizei was born in 92000 and this is fals self assertlistequal tokens spieceunderline i spieceunderline was spieceunderline b or n spieceunderline in spieceunderline 9 2 0 0 0 spieceunderline and spieceunderline this spieceunderline is spieceunderline f al s ids tokenizer converttokenstoidstokens self assertlistequal ids value tokenizer fairseqoffset for value in 8 21 84 55 24 19 7 2 602 347 347 347 3 12 66 46 72 80 6 2 4 unk 2 1 3 unk 2 1 3 backtokens tokenizer convertidstotokensids self assertlistequal backtokens spieceunderline i spieceunderline was spieceunderline b or n spieceunderline in spieceunderline unk 2 0 0 0 spieceunderline and spieceunderline this spieceunderline is spieceunderline f al s unk overwrite from testtokenizationcommon to speed up test def testsavepretrainedself if not self testslowtokenizer as we don t have a slow version we can t compare the outputs between slow and fast versions return self tokenizerslist0 self rusttokenizerclass hfinternaltestingtinyxlmroberta for tokenizer pretrainedname kwargs in self tokenizerslist with self subtestftokenizer class name pretrainedname tokenizerr self rusttokenizerclass frompretrainedpretrainedname kwargs tokenizerp self tokenizerclass frompretrainedpretrainedname kwargs tmpdirname2 tempfile mkdtemp tokenizerrfiles tokenizerr savepretrainedtmpdirname2 tokenizerpfiles tokenizerp savepretrainedtmpdirname2 checks it save with the same files the tokenizer json file for the fast one self asserttrueanytokenizer json in f for f in tokenizerrfiles tokenizerrfiles tuplef for f in tokenizerrfiles if tokenizer json not in f self assertsequenceequaltokenizerrfiles tokenizerpfiles checks everything loads correctly in the same way tokenizerrp tokenizerr frompretrainedtmpdirname2 tokenizerpp tokenizerp frompretrainedtmpdirname2 check special tokens are set accordingly on rust and python for key in tokenizerpp specialtokensmap self asserttruehasattrtokenizerrp key self assertequalgetattrtokenizerrp key getattrtokenizerpp key self assertequalgetattrtokenizerrp key id getattrtokenizerpp key id shutil rmtreetmpdirname2 save tokenizer rust legacyformattrue tmpdirname2 tempfile mkdtemp tokenizerrfiles tokenizerr savepretrainedtmpdirname2 legacyformattrue tokenizerpfiles tokenizerp savepretrainedtmpdirname2 checks it save with the same files self assertsequenceequaltokenizerrfiles tokenizerpfiles checks everything loads correctly in the same way tokenizerrp tokenizerr frompretrainedtmpdirname2 tokenizerpp tokenizerp frompretrainedtmpdirname2 check special tokens are set accordingly on rust and python for key in tokenizerpp specialtokensmap self asserttruehasattrtokenizerrp key shutil rmtreetmpdirname2 save tokenizer rust legacyformatfalse tmpdirname2 tempfile mkdtemp tokenizerrfiles tokenizerr savepretrainedtmpdirname2 legacyformatfalse tokenizerpfiles tokenizerp savepretrainedtmpdirname2 checks it saved the tokenizer json file self asserttrueanytokenizer json in f for f in tokenizerrfiles checks everything loads correctly in the same way tokenizerrp tokenizerr frompretrainedtmpdirname2 tokenizerpp tokenizerp frompretrainedtmpdirname2 check special tokens are set accordingly on rust and python for key in tokenizerpp specialtokensmap self asserttruehasattrtokenizerrp key shutil rmtreetmpdirname2 cachedproperty def bigtokenizerself return xlmrobertatokenizer frompretrainedxlmrobertabase def testpicklablewithoutdiskself with tempfile namedtemporaryfile as f shutil copyfilesamplevocab f name tokenizer xlmrobertatokenizerf name keepaccentstrue pickledtokenizer pickle dumpstokenizer pickle loadspickledtokenizer def testrustandpythonfulltokenizersself if not self testrusttokenizer return tokenizer self gettokenizer rusttokenizer self getrusttokenizer sequence i was born in 92000 and this is fals tokens tokenizer tokenizesequence rusttokens rusttokenizer tokenizesequence self assertlistequaltokens rusttokens ids tokenizer encodesequence addspecialtokensfalse rustids rusttokenizer encodesequence addspecialtokensfalse self assertlistequalids rustids rusttokenizer self getrusttokenizer ids tokenizer encodesequence rustids rusttokenizer encodesequence self assertlistequalids rustids slow def testtokenizationbaseeasysymbolsself symbols hello world originaltokenizerencodings 0 35378 6661 38 2 xlmr torch hub load pytorchfairseq xlmr base xlmr large has same tokenizer xlmr eval xlmr encodesymbols self assertlistequaloriginaltokenizerencodings self bigtokenizer encodesymbols slow def testtokenizationbasehardsymbolsself symbols this is a very long text with a lot of weird characters such as also we will add words that should not exsist and be tokenized to unk such as saoneuhaoesuth originaltokenizerencodings 0 3293 83 10 4552 4989 7986 678 10 5915 111 179459 124850 4 6044 237 12 6 5 6 4 6780 705 15 1388 44 378 10114 711 152 20 6 5 22376 642 1221 15190 34153 450 5608 959 1119 57702 136 186 47 1098 29367 47 4426 what fairseq tokenizes from unk 3678 what fairseq tokenizes from unk unk 2740 what fairseq tokenizes from unk 3 what we tokenize from unk unk 6 residue from the tokenization an extra sentencepiece underline 4 6044 237 6284 50901 528 31 90 34 927 2 xlmr torch hub load pytorchfairseq xlmr base xlmr large has same tokenizer xlmr eval xlmr encodesymbols self assertlistequaloriginaltokenizerencodings self bigtokenizer encodesymbols slow def testtokenizerintegrationself expectedencoding inputids 0 11062 82772 7 15 82772 538 51529 237 17198 1290 206 9 215175 1314 136 17198 1290 206 9 56359 42 122009 9 16466 16 87344 4537 9 4717 78381 6 159958 7 15 24480 618 4 527 22693 5428 4 2777 24480 9874 4 43523 594 4 803 18392 33189 18 4 43523 24447 12399 100 24955 83658 9626 144057 15 839 22335 16 136 24955 83658 83479 15 39102 724 16 678 645 2789 1328 4589 42 122009 115774 23 805 1328 46876 7 136 53894 1940 42227 41159 17721 823 425 4 27512 98722 206 136 5531 4970 919 17336 5 2 0 20080 618 83 82775 47 479 9 1517 73 53894 333 80581 110117 18811 5256 1295 51 152526 297 7986 390 124416 538 35431 214 98 15044 25737 136 7108 43701 23 756 135355 7 5 2 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 581 63773 119455 6 147797 88203 7 645 70 21 3285 10269 5 2 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 attentionmask 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 fmt skip self tokenizerintegrationtestutil expectedencodingexpectedencoding modelnamexlmrobertabase revisiond9d8a8ea5eb94b1c6654ae9249df7793cd2933d3 coding utf 8 2020 the huggingface team licensed under the apache license version 2 0 the license you may not use this file except in compliance with the license you may obtain a copy of the license at http www apache org licenses license 2 0 unless required by applicable law or agreed to in writing software distributed under the license is distributed on an as is basis without warranties or conditions of any kind either express or implied see the license for the specific language governing permissions and limitations under the license we have a sentencepiece fixture for testing test _convert_token_to_id and _convert_id_to_token unk 2 1 3 unk 2 1 3 overwrite from test_tokenization_common to speed up test as we don t have a slow version we can t compare the outputs between slow and fast versions checks it save with the same files the tokenizer json file for the fast one checks everything loads correctly in the same way check special tokens are set accordingly on rust and python self assertequal getattr tokenizer_rp key getattr tokenizer_pp key self assertequal getattr tokenizer_rp key _id getattr tokenizer_pp key _id save tokenizer rust legacy_format true checks it save with the same files checks everything loads correctly in the same way check special tokens are set accordingly on rust and python save tokenizer rust legacy_format false checks it saved the tokenizer json file checks everything loads correctly in the same way check special tokens are set accordingly on rust and python xlmr torch hub load pytorch fairseq xlmr base xlmr large has same tokenizer xlmr eval xlmr encode symbols 4426 what fairseq tokenizes from unk _ 3678 what fairseq tokenizes from unk unk 2740 what fairseq tokenizes from unk what we tokenize from unk unk residue from the tokenization an extra sentencepiece underline xlmr torch hub load pytorch fairseq xlmr base xlmr large has same tokenizer xlmr eval xlmr encode symbols fmt skip
import pickle
import shutil
import tempfile
import unittest

from transformers import SPIECE_UNDERLINE, XLMRobertaTokenizer, XLMRobertaTokenizerFast
from transformers.testing_utils import get_tests_dir, require_sentencepiece, require_tokenizers, slow
from transformers.utils import cached_property

from ...test_tokenization_common import TokenizerTesterMixin


SAMPLE_VOCAB = get_tests_dir("fixtures/test_sentencepiece.model")


@require_sentencepiece
@require_tokenizers
class XLMRobertaTokenizationTest(TokenizerTesterMixin, unittest.TestCase):
    tokenizer_class = XLMRobertaTokenizer
    rust_tokenizer_class = XLMRobertaTokenizerFast
    test_rust_tokenizer = True
    test_sentencepiece = True

    def setUp(self):
        super().setUp()

        
        tokenizer = XLMRobertaTokenizer(SAMPLE_VOCAB, keep_accents=True)
        tokenizer.save_pretrained(self.tmpdirname)

    def test_convert_token_and_id(self):
        
        token = "<pad>"
        token_id = 1

        self.assertEqual(self.get_tokenizer()._convert_token_to_id(token), token_id)
        self.assertEqual(self.get_tokenizer()._convert_id_to_token(token_id), token)

    def test_get_vocab(self):
        vocab_keys = list(self.get_tokenizer().get_vocab().keys())

        self.assertEqual(vocab_keys[0], "<s>")
        self.assertEqual(vocab_keys[1], "<pad>")
        self.assertEqual(vocab_keys[-1], "<mask>")
        self.assertEqual(len(vocab_keys), 1_002)

    def test_vocab_size(self):
        self.assertEqual(self.get_tokenizer().vocab_size, 1_002)

    def test_full_tokenizer(self):
        tokenizer = XLMRobertaTokenizer(SAMPLE_VOCAB, keep_accents=True)

        tokens = tokenizer.tokenize("This is a test")
        self.assertListEqual(tokens, ["▁This", "▁is", "▁a", "▁t", "est"])

        self.assertListEqual(
            tokenizer.convert_tokens_to_ids(tokens),
            [value + tokenizer.fairseq_offset for value in [285, 46, 10, 170, 382]],
        )

        tokens = tokenizer.tokenize("I was born in 92000, and this is falsé.")
        self.assertListEqual(
            tokens,
            [
                SPIECE_UNDERLINE + "I",
                SPIECE_UNDERLINE + "was",
                SPIECE_UNDERLINE + "b",
                "or",
                "n",
                SPIECE_UNDERLINE + "in",
                SPIECE_UNDERLINE + "",
                "9",
                "2",
                "0",
                "0",
                "0",
                ",",
                SPIECE_UNDERLINE + "and",
                SPIECE_UNDERLINE + "this",
                SPIECE_UNDERLINE + "is",
                SPIECE_UNDERLINE + "f",
                "al",
                "s",
                "é",
                ".",
            ],
        )
        ids = tokenizer.convert_tokens_to_ids(tokens)
        self.assertListEqual(
            ids,
            [
                value + tokenizer.fairseq_offset
                for value in [8, 21, 84, 55, 24, 19, 7, 2, 602, 347, 347, 347, 3, 12, 66, 46, 72, 80, 6, 2, 4]
                
            ],
        )

        back_tokens = tokenizer.convert_ids_to_tokens(ids)
        self.assertListEqual(
            back_tokens,
            [
                SPIECE_UNDERLINE + "I",
                SPIECE_UNDERLINE + "was",
                SPIECE_UNDERLINE + "b",
                "or",
                "n",
                SPIECE_UNDERLINE + "in",
                SPIECE_UNDERLINE + "",
                "<unk>",
                "2",
                "0",
                "0",
                "0",
                ",",
                SPIECE_UNDERLINE + "and",
                SPIECE_UNDERLINE + "this",
                SPIECE_UNDERLINE + "is",
                SPIECE_UNDERLINE + "f",
                "al",
                "s",
                "<unk>",
                ".",
            ],
        )

    
    def test_save_pretrained(self):
        if not self.test_slow_tokenizer:
            
            return

        self.tokenizers_list[0] = (self.rust_tokenizer_class, "hf-internal-testing/tiny-xlm-roberta", {})
        for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
            with self.subTest(f"{tokenizer.__class__.__name__} ({pretrained_name})"):
                tokenizer_r = self.rust_tokenizer_class.from_pretrained(pretrained_name, **kwargs)
                tokenizer_p = self.tokenizer_class.from_pretrained(pretrained_name, **kwargs)

                tmpdirname2 = tempfile.mkdtemp()

                tokenizer_r_files = tokenizer_r.save_pretrained(tmpdirname2)
                tokenizer_p_files = tokenizer_p.save_pretrained(tmpdirname2)

                
                self.assertTrue(any("tokenizer.json" in f for f in tokenizer_r_files))
                tokenizer_r_files = tuple(f for f in tokenizer_r_files if "tokenizer.json" not in f)
                self.assertSequenceEqual(tokenizer_r_files, tokenizer_p_files)

                
                tokenizer_rp = tokenizer_r.from_pretrained(tmpdirname2)
                tokenizer_pp = tokenizer_p.from_pretrained(tmpdirname2)

                
                for key in tokenizer_pp.special_tokens_map:
                    self.assertTrue(hasattr(tokenizer_rp, key))
                    
                    

                shutil.rmtree(tmpdirname2)

                
                tmpdirname2 = tempfile.mkdtemp()

                tokenizer_r_files = tokenizer_r.save_pretrained(tmpdirname2, legacy_format=True)
                tokenizer_p_files = tokenizer_p.save_pretrained(tmpdirname2)

                
                self.assertSequenceEqual(tokenizer_r_files, tokenizer_p_files)

                
                tokenizer_rp = tokenizer_r.from_pretrained(tmpdirname2)
                tokenizer_pp = tokenizer_p.from_pretrained(tmpdirname2)

                
                for key in tokenizer_pp.special_tokens_map:
                    self.assertTrue(hasattr(tokenizer_rp, key))

                shutil.rmtree(tmpdirname2)

                
                tmpdirname2 = tempfile.mkdtemp()

                tokenizer_r_files = tokenizer_r.save_pretrained(tmpdirname2, legacy_format=False)
                tokenizer_p_files = tokenizer_p.save_pretrained(tmpdirname2)

                
                self.assertTrue(any("tokenizer.json" in f for f in tokenizer_r_files))

                
                tokenizer_rp = tokenizer_r.from_pretrained(tmpdirname2)
                tokenizer_pp = tokenizer_p.from_pretrained(tmpdirname2)

                
                for key in tokenizer_pp.special_tokens_map:
                    self.assertTrue(hasattr(tokenizer_rp, key))

                shutil.rmtree(tmpdirname2)

    @cached_property
    def big_tokenizer(self):
        return XLMRobertaTokenizer.from_pretrained("xlm-roberta-base")

    def test_picklable_without_disk(self):
        with tempfile.NamedTemporaryFile() as f:
            shutil.copyfile(SAMPLE_VOCAB, f.name)
            tokenizer = XLMRobertaTokenizer(f.name, keep_accents=True)
            pickled_tokenizer = pickle.dumps(tokenizer)
        pickle.loads(pickled_tokenizer)

    def test_rust_and_python_full_tokenizers(self):
        if not self.test_rust_tokenizer:
            return

        tokenizer = self.get_tokenizer()
        rust_tokenizer = self.get_rust_tokenizer()

        sequence = "I was born in 92000, and this is falsé."

        tokens = tokenizer.tokenize(sequence)
        rust_tokens = rust_tokenizer.tokenize(sequence)
        self.assertListEqual(tokens, rust_tokens)

        ids = tokenizer.encode(sequence, add_special_tokens=False)
        rust_ids = rust_tokenizer.encode(sequence, add_special_tokens=False)
        self.assertListEqual(ids, rust_ids)

        rust_tokenizer = self.get_rust_tokenizer()
        ids = tokenizer.encode(sequence)
        rust_ids = rust_tokenizer.encode(sequence)
        self.assertListEqual(ids, rust_ids)

    @slow
    def test_tokenization_base_easy_symbols(self):
        symbols = "Hello World!"
        original_tokenizer_encodings = [0, 35378, 6661, 38, 2]
        
        
        

        self.assertListEqual(original_tokenizer_encodings, self.big_tokenizer.encode(symbols))

    @slow
    def test_tokenization_base_hard_symbols(self):
        symbols = (
            'This is a very long text with a lot of weird characters, such as: . , ~ ? ( ) " [ ] ! : - . Also we will'
            " add words that should not exsist and be tokenized to <unk>, such as saoneuhaoesuth"
        )
        original_tokenizer_encodings = [
            0,
            3293,
            83,
            10,
            4552,
            4989,
            7986,
            678,
            10,
            5915,
            111,
            179459,
            124850,
            4,
            6044,
            237,
            12,
            6,
            5,
            6,
            4,
            6780,
            705,
            15,
            1388,
            44,
            378,
            10114,
            711,
            152,
            20,
            6,
            5,
            22376,
            642,
            1221,
            15190,
            34153,
            450,
            5608,
            959,
            1119,
            57702,
            136,
            186,
            47,
            1098,
            29367,
            47,
            
            
            
            3,  
            6,  
            4,
            6044,
            237,
            6284,
            50901,
            528,
            31,
            90,
            34,
            927,
            2,
        ]
        
        
        

        self.assertListEqual(original_tokenizer_encodings, self.big_tokenizer.encode(symbols))

    @slow
    def test_tokenizer_integration(self):
        expected_encoding = {'input_ids': [[0, 11062, 82772, 7, 15, 82772, 538, 51529, 237, 17198, 1290, 206, 9, 215175, 1314, 136, 17198, 1290, 206, 9, 56359, 42, 122009, 9, 16466, 16, 87344, 4537, 9, 4717, 78381, 6, 159958, 7, 15, 24480, 618, 4, 527, 22693, 5428, 4, 2777, 24480, 9874, 4, 43523, 594, 4, 803, 18392, 33189, 18, 4, 43523, 24447, 12399, 100, 24955, 83658, 9626, 144057, 15, 839, 22335, 16, 136, 24955, 83658, 83479, 15, 39102, 724, 16, 678, 645, 2789, 1328, 4589, 42, 122009, 115774, 23, 805, 1328, 46876, 7, 136, 53894, 1940, 42227, 41159, 17721, 823, 425, 4, 27512, 98722, 206, 136, 5531, 4970, 919, 17336, 5, 2], [0, 20080, 618, 83, 82775, 47, 479, 9, 1517, 73, 53894, 333, 80581, 110117, 18811, 5256, 1295, 51, 152526, 297, 7986, 390, 124416, 538, 35431, 214, 98, 15044, 25737, 136, 7108, 43701, 23, 756, 135355, 7, 5, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [0, 581, 63773, 119455, 6, 147797, 88203, 7, 645, 70, 21, 3285, 10269, 5, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]], 'attention_mask': [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]]}  

        self.tokenizer_integration_test_util(
            expected_encoding=expected_encoding,
            model_name="xlm-roberta-base",
            revision="d9d8a8ea5eb94b1c6654ae9249df7793cd2933d3",
        )