

import inspect
import unittest

import torch
import torch.nn as nn

from accelerate.hooks import (
    AlignDevicesHook,
    ModelHook,
    SequentialHook,
    add_hook_to_module,
    attach_align_device_hook,
    remove_hook_from_module,
    remove_hook_from_submodules,
)
from accelerate.test_utils import require_multi_gpu


class  A						(							nn.Module ):







    """simple docstring"""





    def __init__(      self :       List[str]		)->    Union[str, Any]:





        '''simple docstring'''







        super().__init__()
        A__          =   nn.Linear(3,4		)
        A__          =   nn.BatchNormad(4		)
        A__          =   nn.Linear(4,5		)


    def 						snake_case__     (      self :       Optional[int],lowercase_ :       List[Any]		)->    Any:





        '''simple docstring'''







        return self.lineara(self.batchnorm(self.lineara(lowercase_		)		)		)




class  A						(							_UpperCAmelCase ):







    """simple docstring"""





    def 						snake_case__     (      self :       List[Any],lowercase_ :       List[str],*lowercase_ :       Optional[int],**lowercase_ :       Tuple		)->    Any:





        '''simple docstring'''







        return (args[0] + 1,) + args[1:], kwargs




class  A						(							_UpperCAmelCase ):







    """simple docstring"""





    def 						snake_case__     (      self :       List[str],lowercase_ :       int,lowercase_ :       Optional[Any]		)->    List[str]:





        '''simple docstring'''







        return output + 1




class  A						(							unittest.TestCase ):







    """simple docstring"""





    def 						snake_case__     (      self :       Union[str, Any]		)->    Tuple:





        '''simple docstring'''







        A__          =   ModelForTest()
        A__          =   ModelHook()

        add_hook_to_module(lowercase_,lowercase_		)
        self.assertEqual(test_model._hf_hook,lowercase_		)
        self.assertTrue(hasattr(lowercase_,'_old_forward'		)		)

        # Check adding the hook did not change the name or the signature
        self.assertEqual(test_model.forward.__name__,'forward'		)
        self.assertListEqual(list(inspect.signature(test_model.forward		).parameters		),['x']		)

        remove_hook_from_module(lowercase_		)
        self.assertFalse(hasattr(lowercase_,'_hf_hook'		)		)
        self.assertFalse(hasattr(lowercase_,'_old_forward'		)		)


    def 						snake_case__     (      self :       Optional[Any]		)->    str:





        '''simple docstring'''







        A__          =   ModelForTest()
        A__          =   ModelHook()

        add_hook_to_module(lowercase_,lowercase_		)
        add_hook_to_module(lowercase_,lowercase_,append=lowercase_		)

        self.assertEqual(isinstance(test_model._hf_hook,lowercase_		),lowercase_		)
        self.assertEqual(len(test_model._hf_hook.hooks		),2		)
        self.assertTrue(hasattr(lowercase_,'_old_forward'		)		)

        # Check adding the hook did not change the name or the signature
        self.assertEqual(test_model.forward.__name__,'forward'		)
        self.assertListEqual(list(inspect.signature(test_model.forward		).parameters		),['x']		)

        remove_hook_from_module(lowercase_		)
        self.assertFalse(hasattr(lowercase_,'_hf_hook'		)		)
        self.assertFalse(hasattr(lowercase_,'_old_forward'		)		)


    def 						snake_case__     (      self :       str		)->    Any:





        '''simple docstring'''







        A__          =   ModelForTest()
        A__          =   torch.randn(2,3		)
        A__          =   test_model(x + 1		)
        A__          =   test_model(x + 2		)

        A__          =   PreForwardHook()
        add_hook_to_module(lowercase_,lowercase_		)
        A__          =   test_model(lowercase_		)
        self.assertTrue(torch.allclose(lowercase_,lowercase_,atol=1E-5		)		)

        # Attaching a hook to a model when it already has one replaces, does not chain
        A__          =   PreForwardHook()
        add_hook_to_module(lowercase_,lowercase_		)
        A__          =   test_model(lowercase_		)
        self.assertTrue(torch.allclose(lowercase_,lowercase_,atol=1E-5		)		)

        # You need to use the sequential hook to chain two or more hooks
        A__          =   SequentialHook(PreForwardHook(),PreForwardHook()		)
        add_hook_to_module(lowercase_,lowercase_		)

        A__          =   test_model(lowercase_		)
        assert torch.allclose(lowercase_,lowercase_,atol=1E-5		)


    def 						snake_case__     (      self :       Optional[Any]		)->    Optional[Any]:





        '''simple docstring'''







        A__          =   ModelForTest()
        A__          =   torch.randn(2,3		)
        A__          =   test_model(lowercase_		)

        A__          =   PostForwardHook()
        add_hook_to_module(lowercase_,lowercase_		)
        A__          =   test_model(lowercase_		)
        self.assertTrue(torch.allclose(lowercase_,output + 1,atol=1E-5		)		)

        # Attaching a hook to a model when it already has one replaces, does not chain
        A__          =   PostForwardHook()
        add_hook_to_module(lowercase_,lowercase_		)
        A__          =   test_model(lowercase_		)
        self.assertTrue(torch.allclose(lowercase_,output + 1,atol=1E-5		)		)

        # You need to use the sequential hook to chain two or more hooks
        A__          =   SequentialHook(PostForwardHook(),PostForwardHook()		)
        add_hook_to_module(lowercase_,lowercase_		)

        A__          =   test_model(lowercase_		)
        assert torch.allclose(lowercase_,output + 2,atol=1E-5		)


    def 						snake_case__     (      self :       Dict		)->    Union[str, Any]:





        '''simple docstring'''







        A__          =   ModelForTest()
        A__          =   torch.randn(2,3		)
        A__          =   test_model(lowercase_		)

        A__          =   PostForwardHook()
        add_hook_to_module(lowercase_,lowercase_		)
        A__          =   test_model(lowercase_		)
        self.assertTrue(torch.allclose(lowercase_,output + 1		)		)
        self.assertTrue(outputa.requires_grad		)

        A__          =   True
        A__          =   test_model(lowercase_		)
        self.assertFalse(outputa.requires_grad		)


    @require_multi_gpu
    def 						snake_case__     (      self :       Optional[int]		)->    Any:





        '''simple docstring'''







        A__          =   ModelForTest()
        # Everything is on CPU
        self.assertEqual(model.lineara.weight.device,torch.device('cpu'		)		)
        self.assertEqual(model.batchnorm.weight.device,torch.device('cpu'		)		)
        self.assertEqual(model.lineara.weight.device,torch.device('cpu'		)		)

        # This will move each submodule on different devices
        add_hook_to_module(model.lineara,AlignDevicesHook(execution_device=0		)		)
        add_hook_to_module(model.batchnorm,AlignDevicesHook(execution_device=0		)		)
        add_hook_to_module(model.lineara,AlignDevicesHook(execution_device=1		)		)

        self.assertEqual(model.lineara.weight.device,torch.device(0		)		)
        self.assertEqual(model.batchnorm.weight.device,torch.device(0		)		)
        self.assertEqual(model.batchnorm.running_mean.device,torch.device(0		)		)
        self.assertEqual(model.lineara.weight.device,torch.device(1		)		)

        # We can still make a forward pass. The input does not need to be on any particular device
        A__          =   torch.randn(2,3		)
        A__          =   model(lowercase_		)
        self.assertEqual(output.device,torch.device(1		)		)

        # We can add a general hook to put back output on same device as input.
        add_hook_to_module(lowercase_,AlignDevicesHook(io_same_device=lowercase_		)		)
        A__          =   torch.randn(2,3		).to(0		)
        A__          =   model(lowercase_		)
        self.assertEqual(output.device,torch.device(0		)		)


    def 						snake_case__     (      self :       Dict		)->    Optional[Any]:





        '''simple docstring'''







        A__          =   ModelForTest()

        # Everything is on CPU
        self.assertEqual(model.lineara.weight.device,torch.device('cpu'		)		)
        self.assertEqual(model.batchnorm.weight.device,torch.device('cpu'		)		)
        self.assertEqual(model.lineara.weight.device,torch.device('cpu'		)		)

        # This will move each submodule on different devices
        A__          =   {'execution_device': 0 if torch.cuda.is_available() else 'cpu', 'offload': True}

        add_hook_to_module(model.lineara,AlignDevicesHook(**lowercase_		)		)
        add_hook_to_module(model.batchnorm,AlignDevicesHook(**lowercase_		)		)
        add_hook_to_module(model.lineara,AlignDevicesHook(**lowercase_		)		)

        # Parameters have been offloaded, so on the meta device
        self.assertEqual(model.lineara.weight.device,torch.device('meta'		)		)
        self.assertEqual(model.batchnorm.weight.device,torch.device('meta'		)		)
        self.assertEqual(model.lineara.weight.device,torch.device('meta'		)		)
        # Buffers are not included in the offload by default, so are on the execution device
        A__          =   torch.device(hook_kwargs['execution_device']		)
        self.assertEqual(model.batchnorm.running_mean.device,lowercase_		)

        A__          =   torch.randn(2,3		)
        A__          =   model(lowercase_		)
        self.assertEqual(output.device,lowercase_		)

        # Removing hooks loads back the weights in the model.
        remove_hook_from_module(model.lineara		)
        remove_hook_from_module(model.batchnorm		)
        remove_hook_from_module(model.lineara		)
        self.assertEqual(model.lineara.weight.device,torch.device('cpu'		)		)
        self.assertEqual(model.batchnorm.weight.device,torch.device('cpu'		)		)
        self.assertEqual(model.lineara.weight.device,torch.device('cpu'		)		)

        # Now test with buffers included in the offload
        A__          =   {
            'execution_device': 0 if torch.cuda.is_available() else 'cpu',
            'offload': True,
            'offload_buffers': True,
        }

        add_hook_to_module(model.lineara,AlignDevicesHook(**lowercase_		)		)
        add_hook_to_module(model.batchnorm,AlignDevicesHook(**lowercase_		)		)
        add_hook_to_module(model.lineara,AlignDevicesHook(**lowercase_		)		)

        # Parameters have been offloaded, so on the meta device, buffers included
        self.assertEqual(model.lineara.weight.device,torch.device('meta'		)		)
        self.assertEqual(model.batchnorm.weight.device,torch.device('meta'		)		)
        self.assertEqual(model.lineara.weight.device,torch.device('meta'		)		)
        self.assertEqual(model.batchnorm.running_mean.device,torch.device('meta'		)		)

        A__          =   torch.randn(2,3		)
        A__          =   model(lowercase_		)
        self.assertEqual(output.device,lowercase_		)

        # Removing hooks loads back the weights in the model.
        remove_hook_from_module(model.lineara		)
        remove_hook_from_module(model.batchnorm		)
        remove_hook_from_module(model.lineara		)
        self.assertEqual(model.lineara.weight.device,torch.device('cpu'		)		)
        self.assertEqual(model.batchnorm.weight.device,torch.device('cpu'		)		)
        self.assertEqual(model.lineara.weight.device,torch.device('cpu'		)		)


    def 						snake_case__     (      self :       Tuple		)->    Any:





        '''simple docstring'''







        A__          =   ModelForTest()

        # Everything is on CPU
        self.assertEqual(model.lineara.weight.device,torch.device('cpu'		)		)
        self.assertEqual(model.batchnorm.weight.device,torch.device('cpu'		)		)
        self.assertEqual(model.lineara.weight.device,torch.device('cpu'		)		)

        # This will move each submodule on different devices
        A__          =   0 if torch.cuda.is_available() else 'cpu'
        attach_align_device_hook(lowercase_,execution_device=lowercase_,offload=lowercase_		)

        # Parameters have been offloaded, so on the meta device
        self.assertEqual(model.lineara.weight.device,torch.device('meta'		)		)
        self.assertEqual(model.batchnorm.weight.device,torch.device('meta'		)		)
        self.assertEqual(model.lineara.weight.device,torch.device('meta'		)		)
        # Buffers are not included in the offload by default, so are on the execution device
        A__          =   torch.device(lowercase_		)
        self.assertEqual(model.batchnorm.running_mean.device,lowercase_		)

        A__          =   torch.randn(2,3		)
        A__          =   model(lowercase_		)
        self.assertEqual(output.device,lowercase_		)

        # Removing hooks loads back the weights in the model.
        remove_hook_from_submodules(lowercase_		)
        self.assertEqual(model.lineara.weight.device,torch.device('cpu'		)		)
        self.assertEqual(model.batchnorm.weight.device,torch.device('cpu'		)		)
        self.assertEqual(model.lineara.weight.device,torch.device('cpu'		)		)

        # Now test with buffers included in the offload
        attach_align_device_hook(lowercase_,execution_device=lowercase_,offload=lowercase_,offload_buffers=lowercase_		)

        # Parameters have been offloaded, so on the meta device, buffers included
        self.assertEqual(model.lineara.weight.device,torch.device('meta'		)		)
        self.assertEqual(model.batchnorm.weight.device,torch.device('meta'		)		)
        self.assertEqual(model.lineara.weight.device,torch.device('meta'		)		)
        self.assertEqual(model.batchnorm.running_mean.device,torch.device('meta'		)		)

        A__          =   torch.randn(2,3		)
        A__          =   model(lowercase_		)
        self.assertEqual(output.device,lowercase_		)

        # Removing hooks loads back the weights in the model.
        remove_hook_from_submodules(lowercase_		)
        self.assertEqual(model.lineara.weight.device,torch.device('cpu'		)		)
        self.assertEqual(model.batchnorm.weight.device,torch.device('cpu'		)		)
        self.assertEqual(model.lineara.weight.device,torch.device('cpu'		)		)


    def 						snake_case__     (      self :       List[Any]		)->    Union[str, Any]:





        '''simple docstring'''







        A__          =   ModelForTest()

        # Everything is on CPU
        self.assertEqual(model.lineara.weight.device,torch.device('cpu'		)		)
        self.assertEqual(model.batchnorm.weight.device,torch.device('cpu'		)		)
        self.assertEqual(model.lineara.weight.device,torch.device('cpu'		)		)

        # This will move each submodule on different devices
        A__          =   0 if torch.cuda.is_available() else 'cpu'
        attach_align_device_hook(
            lowercase_,execution_device=lowercase_,offload=lowercase_,weights_map=model.state_dict()		)

        # Parameters have been offloaded, so on the meta device
        self.assertEqual(model.lineara.weight.device,torch.device('meta'		)		)
        self.assertEqual(model.batchnorm.weight.device,torch.device('meta'		)		)
        self.assertEqual(model.lineara.weight.device,torch.device('meta'		)		)
        # Buffers are not included in the offload by default, so are on the execution device
        A__          =   torch.device(lowercase_		)
        self.assertEqual(model.batchnorm.running_mean.device,lowercase_		)

        A__          =   torch.randn(2,3		)
        A__          =   model(lowercase_		)
        self.assertEqual(output.device,lowercase_		)

        # Removing hooks loads back the weights in the model.
        remove_hook_from_submodules(lowercase_		)
        self.assertEqual(model.lineara.weight.device,torch.device('cpu'		)		)
        self.assertEqual(model.batchnorm.weight.device,torch.device('cpu'		)		)
        self.assertEqual(model.lineara.weight.device,torch.device('cpu'		)		)

        # Now test with buffers included in the offload
        attach_align_device_hook(
            lowercase_,execution_device=lowercase_,offload=lowercase_,weights_map=model.state_dict(),offload_buffers=lowercase_,)

        # Parameters have been offloaded, so on the meta device, buffers included
        self.assertEqual(model.lineara.weight.device,torch.device('meta'		)		)
        self.assertEqual(model.batchnorm.weight.device,torch.device('meta'		)		)
        self.assertEqual(model.lineara.weight.device,torch.device('meta'		)		)
        self.assertEqual(model.batchnorm.running_mean.device,torch.device('meta'		)		)

        A__          =   torch.randn(2,3		)
        A__          =   model(lowercase_		)
        self.assertEqual(output.device,lowercase_		)

        # Removing hooks loads back the weights in the model.
        remove_hook_from_submodules(lowercase_		)
        self.assertEqual(model.lineara.weight.device,torch.device('cpu'		)		)
        self.assertEqual(model.batchnorm.weight.device,torch.device('cpu'		)		)
        self.assertEqual(model.lineara.weight.device,torch.device('cpu'		)		)


'''simple docstring'''




import pickle
import shutil
import tempfile
import unittest

from transformers import SPIECE_UNDERLINE, XLMRobertaTokenizer, XLMRobertaTokenizerFast
from transformers.testing_utils import get_tests_dir, require_sentencepiece, require_tokenizers, slow
from transformers.utils import cached_property

from ...test_tokenization_common import TokenizerTesterMixin


__lowercase          =  get_tests_dir('''fixtures/test_sentencepiece.model''')






@require_sentencepiece
@require_tokenizers
class 				a__(					lowerCAmelCase__ ,						unittest.TestCase	):


			'''simple docstring'''






			UpperCAmelCase_			:     Tuple      =       XLMRobertaTokenizer
			UpperCAmelCase_			:     int      =       XLMRobertaTokenizerFast
			UpperCAmelCase_			:     List[str]      =       True
			UpperCAmelCase_			:     Optional[int]      =       True



			def 							a_       (   self):




					"""simple docstring"""


					super().setUp()

					# We have a SentencePiece fixture for testing
					lowerCAmelCase   					=  XLMRobertaTokenizer(__lowerCAmelCase       ,    keep_accents=__lowerCAmelCase)
					tokenizer.save_pretrained(self.tmpdirname)



			def 							a_       (   self):




					"""simple docstring"""


					lowerCAmelCase   					=  """<pad>"""
					lowerCAmelCase   					=  1

					self.assertEqual(self.get_tokenizer()._convert_token_to_id(__lowerCAmelCase)       ,    __lowerCAmelCase)
					self.assertEqual(self.get_tokenizer()._convert_id_to_token(__lowerCAmelCase)       ,    __lowerCAmelCase)



			def 							a_       (   self):




					"""simple docstring"""


					lowerCAmelCase   					=  list(self.get_tokenizer().get_vocab().keys())

					self.assertEqual(vocab_keys[0]       ,    """<s>""")
					self.assertEqual(vocab_keys[1]       ,    """<pad>""")
					self.assertEqual(vocab_keys[-1]       ,    """<mask>""")
					self.assertEqual(len(__lowerCAmelCase)       ,    1002)



			def 							a_       (   self):




					"""simple docstring"""


					self.assertEqual(self.get_tokenizer().vocab_size       ,    1002)



			def 							a_       (   self):




					"""simple docstring"""


					lowerCAmelCase   					=  XLMRobertaTokenizer(__lowerCAmelCase       ,    keep_accents=__lowerCAmelCase)

					lowerCAmelCase   					=  tokenizer.tokenize("""This is a test""")
					self.assertListEqual(__lowerCAmelCase       ,    ["""▁This""", """▁is""", """▁a""", """▁t""", """est"""])

					self.assertListEqual(
					    tokenizer.convert_tokens_to_ids(__lowerCAmelCase)       ,    [value + tokenizer.fairseq_offset for value in [285, 46, 10, 170, 382]]       ,    )

					lowerCAmelCase   					=  tokenizer.tokenize("""I was born in 92000, and this is falsé.""")
					self.assertListEqual(
					    __lowerCAmelCase       ,    [
					        SPIECE_UNDERLINE + """I""",
					        SPIECE_UNDERLINE + """was""",
					        SPIECE_UNDERLINE + """b""",
					        """or""",
					        """n""",
					        SPIECE_UNDERLINE + """in""",
					        SPIECE_UNDERLINE + """""",
					        """9""",
					        """2""",
					        """0""",
					        """0""",
					        """0""",
					        """,""",
					        SPIECE_UNDERLINE + """and""",
					        SPIECE_UNDERLINE + """this""",
					        SPIECE_UNDERLINE + """is""",
					        SPIECE_UNDERLINE + """f""",
					        """al""",
					        """s""",
					        """é""",
					        """.""",
					    ]       ,    )
					lowerCAmelCase   					=  tokenizer.convert_tokens_to_ids(__lowerCAmelCase)
					self.assertListEqual(
					    __lowerCAmelCase       ,    [
					        value + tokenizer.fairseq_offset
					        for value in [8, 21, 84, 55, 24, 19, 7, 2, 602, 347, 347, 347, 3, 12, 66, 46, 72, 80, 6, 2, 4]
					        #                                       ^ unk: 2 + 1 = 3                  unk: 2 + 1 = 3 ^
					    ]       ,    )

					lowerCAmelCase   					=  tokenizer.convert_ids_to_tokens(__lowerCAmelCase)
					self.assertListEqual(
					    __lowerCAmelCase       ,    [
					        SPIECE_UNDERLINE + """I""",
					        SPIECE_UNDERLINE + """was""",
					        SPIECE_UNDERLINE + """b""",
					        """or""",
					        """n""",
					        SPIECE_UNDERLINE + """in""",
					        SPIECE_UNDERLINE + """""",
					        """<unk>""",
					        """2""",
					        """0""",
					        """0""",
					        """0""",
					        """,""",
					        SPIECE_UNDERLINE + """and""",
					        SPIECE_UNDERLINE + """this""",
					        SPIECE_UNDERLINE + """is""",
					        SPIECE_UNDERLINE + """f""",
					        """al""",
					        """s""",
					        """<unk>""",
					        """.""",
					    ]       ,    )



			def 							a_       (   self):




					"""simple docstring"""


					if not self.test_slow_tokenizer:
							# as we don't have a slow version, we can't compare the outputs between slow and fast versions
							return

					lowerCAmelCase   					=  (self.rust_tokenizer_class, """hf-internal-testing/tiny-xlm-roberta""", {})
					for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
							with self.subTest(f"{tokenizer.__class__.__name__} ({pretrained_name})"):
									lowerCAmelCase   					=  self.rust_tokenizer_class.from_pretrained(__lowerCAmelCase       ,    **__lowerCAmelCase)
									lowerCAmelCase   					=  self.tokenizer_class.from_pretrained(__lowerCAmelCase       ,    **__lowerCAmelCase)

									lowerCAmelCase   					=  tempfile.mkdtemp()

									lowerCAmelCase   					=  tokenizer_r.save_pretrained(__lowerCAmelCase)
									lowerCAmelCase   					=  tokenizer_p.save_pretrained(__lowerCAmelCase)

									# Checks it save with the same files + the tokenizer.json file for the fast one
									self.assertTrue(any("""tokenizer.json""" in f for f in tokenizer_r_files))
									lowerCAmelCase   					=  tuple(f for f in tokenizer_r_files if """tokenizer.json""" not in f)
									self.assertSequenceEqual(__lowerCAmelCase       ,    __lowerCAmelCase)

									# Checks everything loads correctly in the same way
									lowerCAmelCase   					=  tokenizer_r.from_pretrained(__lowerCAmelCase)
									lowerCAmelCase   					=  tokenizer_p.from_pretrained(__lowerCAmelCase)

									# Check special tokens are set accordingly on Rust and Python
									for key in tokenizer_pp.special_tokens_map:
											self.assertTrue(hasattr(__lowerCAmelCase       ,    __lowerCAmelCase))
											# self.assertEqual(getattr(tokenizer_rp, key), getattr(tokenizer_pp, key))
											# self.assertEqual(getattr(tokenizer_rp, key + "_id"), getattr(tokenizer_pp, key + "_id"))

									shutil.rmtree(__lowerCAmelCase)

									# Save tokenizer rust, legacy_format=True
									lowerCAmelCase   					=  tempfile.mkdtemp()

									lowerCAmelCase   					=  tokenizer_r.save_pretrained(__lowerCAmelCase       ,    legacy_format=__lowerCAmelCase)
									lowerCAmelCase   					=  tokenizer_p.save_pretrained(__lowerCAmelCase)

									# Checks it save with the same files
									self.assertSequenceEqual(__lowerCAmelCase       ,    __lowerCAmelCase)

									# Checks everything loads correctly in the same way
									lowerCAmelCase   					=  tokenizer_r.from_pretrained(__lowerCAmelCase)
									lowerCAmelCase   					=  tokenizer_p.from_pretrained(__lowerCAmelCase)

									# Check special tokens are set accordingly on Rust and Python
									for key in tokenizer_pp.special_tokens_map:
											self.assertTrue(hasattr(__lowerCAmelCase       ,    __lowerCAmelCase))

									shutil.rmtree(__lowerCAmelCase)

									# Save tokenizer rust, legacy_format=False
									lowerCAmelCase   					=  tempfile.mkdtemp()

									lowerCAmelCase   					=  tokenizer_r.save_pretrained(__lowerCAmelCase       ,    legacy_format=__lowerCAmelCase)
									lowerCAmelCase   					=  tokenizer_p.save_pretrained(__lowerCAmelCase)

									# Checks it saved the tokenizer.json file
									self.assertTrue(any("""tokenizer.json""" in f for f in tokenizer_r_files))

									# Checks everything loads correctly in the same way
									lowerCAmelCase   					=  tokenizer_r.from_pretrained(__lowerCAmelCase)
									lowerCAmelCase   					=  tokenizer_p.from_pretrained(__lowerCAmelCase)

									# Check special tokens are set accordingly on Rust and Python
									for key in tokenizer_pp.special_tokens_map:
											self.assertTrue(hasattr(__lowerCAmelCase       ,    __lowerCAmelCase))

									shutil.rmtree(__lowerCAmelCase)



			@cached_property
			def 							a_       (   self):




					"""simple docstring"""


					return XLMRobertaTokenizer.from_pretrained("""xlm-roberta-base""")



			def 							a_       (   self):




					"""simple docstring"""


					with tempfile.NamedTemporaryFile() as f:
							shutil.copyfile(__lowerCAmelCase       ,    f.name)
							lowerCAmelCase   					=  XLMRobertaTokenizer(f.name       ,    keep_accents=__lowerCAmelCase)
							lowerCAmelCase   					=  pickle.dumps(__lowerCAmelCase)
					pickle.loads(__lowerCAmelCase)



			def 							a_       (   self):




					"""simple docstring"""


					if not self.test_rust_tokenizer:
							return

					lowerCAmelCase   					=  self.get_tokenizer()
					lowerCAmelCase   					=  self.get_rust_tokenizer()

					lowerCAmelCase   					=  """I was born in 92000, and this is falsé."""

					lowerCAmelCase   					=  tokenizer.tokenize(__lowerCAmelCase)
					lowerCAmelCase   					=  rust_tokenizer.tokenize(__lowerCAmelCase)
					self.assertListEqual(__lowerCAmelCase       ,    __lowerCAmelCase)

					lowerCAmelCase   					=  tokenizer.encode(__lowerCAmelCase       ,    add_special_tokens=__lowerCAmelCase)
					lowerCAmelCase   					=  rust_tokenizer.encode(__lowerCAmelCase       ,    add_special_tokens=__lowerCAmelCase)
					self.assertListEqual(__lowerCAmelCase       ,    __lowerCAmelCase)

					lowerCAmelCase   					=  self.get_rust_tokenizer()
					lowerCAmelCase   					=  tokenizer.encode(__lowerCAmelCase)
					lowerCAmelCase   					=  rust_tokenizer.encode(__lowerCAmelCase)
					self.assertListEqual(__lowerCAmelCase       ,    __lowerCAmelCase)



			@slow
			def 							a_       (   self):




					"""simple docstring"""


					lowerCAmelCase   					=  """Hello World!"""
					lowerCAmelCase   					=  [0, 35378, 6661, 38, 2]
					# xlmr = torch.hub.load('pytorch/fairseq', 'xlmr.base')  # xlmr.large has same tokenizer
					# xlmr.eval()
					# xlmr.encode(symbols)

					self.assertListEqual(__lowerCAmelCase       ,    self.big_tokenizer.encode(__lowerCAmelCase))



			@slow
			def 							a_       (   self):




					"""simple docstring"""


					lowerCAmelCase   					=  (
					    """This is a very long text with a lot of weird characters, such as: . , ~ ? ( ) \" [ ] ! : - . Also we will"""
					    """ add words that should not exsist and be tokenized to <unk>, such as saoneuhaoesuth"""
					)
					lowerCAmelCase   					=  [
					    0,
					    3293,
					    83,
					    10,
					    4552,
					    4989,
					    7986,
					    678,
					    10,
					    5915,
					    111,
					    179459,
					    124850,
					    4,
					    6044,
					    237,
					    12,
					    6,
					    5,
					    6,
					    4,
					    6780,
					    705,
					    15,
					    1388,
					    44,
					    378,
					    10114,
					    711,
					    152,
					    20,
					    6,
					    5,
					    22376,
					    642,
					    1221,
					    15190,
					    34153,
					    450,
					    5608,
					    959,
					    1119,
					    57702,
					    136,
					    186,
					    47,
					    1098,
					    29367,
					    47,
					    # 4426, # What fairseq tokenizes from "<unk>": "_<"
					    # 3678, # What fairseq tokenizes from "<unk>": "unk"
					    # 2740, # What fairseq tokenizes from "<unk>": ">"
					    3,  # What we tokenize from "<unk>": "<unk>"
					    6,  # Residue from the tokenization: an extra sentencepiece underline
					    4,
					    6044,
					    237,
					    6284,
					    50901,
					    528,
					    31,
					    90,
					    34,
					    927,
					    2,
					]
					# xlmr = torch.hub.load('pytorch/fairseq', 'xlmr.base')  # xlmr.large has same tokenizer
					# xlmr.eval()
					# xlmr.encode(symbols)

					self.assertListEqual(__lowerCAmelCase       ,    self.big_tokenizer.encode(__lowerCAmelCase))







			@slow
			def 							a_       (   self):




					"""simple docstring"""


					lowerCAmelCase   					=  {"""input_ids""": [[0, 11062, 82772, 7, 15, 82772, 538, 51529, 237, 17198, 1290, 206, 9, 215175, 1314, 136, 17198, 1290, 206, 9, 56359, 42, 122009, 9, 16466, 16, 87344, 4537, 9, 4717, 78381, 6, 159958, 7, 15, 24480, 618, 4, 527, 22693, 5428, 4, 2777, 24480, 9874, 4, 43523, 594, 4, 803, 18392, 33189, 18, 4, 43523, 24447, 12399, 100, 24955, 83658, 9626, 144057, 15, 839, 22335, 16, 136, 24955, 83658, 83479, 15, 39102, 724, 16, 678, 645, 2789, 1328, 4589, 42, 122009, 115774, 23, 805, 1328, 46876, 7, 136, 53894, 1940, 42227, 41159, 17721, 823, 425, 4, 27512, 98722, 206, 136, 5531, 4970, 919, 17336, 5, 2], [0, 20080, 618, 83, 82775, 47, 479, 9, 1517, 73, 53894, 333, 80581, 110117, 18811, 5256, 1295, 51, 152526, 297, 7986, 390, 124416, 538, 35431, 214, 98, 15044, 25737, 136, 7108, 43701, 23, 756, 135355, 7, 5, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [0, 581, 63773, 119455, 6, 147797, 88203, 7, 645, 70, 21, 3285, 10269, 5, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]], """attention_mask""": [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]]}  # noqa: E501
					# fmt: on

					self.tokenizer_integration_test_util(
					    expected_encoding=__lowerCAmelCase       ,    model_name="""xlm-roberta-base"""       ,    revision="""d9d8a8ea5eb94b1c6654ae9249df7793cd2933d3"""       ,    )




