'''simple docstring'''




import copy
import inspect
import unittest

from transformers import PretrainedConfig, SwiftFormerConfig
from transformers.testing_utils import (
    require_torch,
    require_vision,
    slow,
    torch_device,
)
from transformers.utils import cached_property, is_torch_available, is_vision_available

from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, floats_tensor, ids_tensor
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
						import torch
						from torch import nn

						from transformers import SwiftFormerForImageClassification, SwiftFormerModel
						from transformers.models.swiftformer.modeling_swiftformer import SWIFTFORMER_PRETRAINED_MODEL_ARCHIVE_LIST


if is_vision_available():
						from PIL import Image

						from transformers import ViTImageProcessor






class 				a__:


			'''simple docstring'''






			def __init__(   self       ,    __lowerCAmelCase       ,    __lowerCAmelCase=13       ,    __lowerCAmelCase=3       ,    __lowerCAmelCase=True       ,    __lowerCAmelCase=True       ,    __lowerCAmelCase=0.1       ,    __lowerCAmelCase=0.1       ,    __lowerCAmelCase=224       ,    __lowerCAmelCase=1000       ,    __lowerCAmelCase=[3, 3, 6, 4]       ,    __lowerCAmelCase=[48, 56, 112, 220]       ,    ):




					"""simple docstring"""


					lowerCAmelCase   					=  parent
					lowerCAmelCase   					=  batch_size
					lowerCAmelCase   					=  num_channels
					lowerCAmelCase   					=  is_training
					lowerCAmelCase   					=  use_labels
					lowerCAmelCase   					=  hidden_dropout_prob
					lowerCAmelCase   					=  attention_probs_dropout_prob
					lowerCAmelCase   					=  num_labels
					lowerCAmelCase   					=  image_size
					lowerCAmelCase   					=  layer_depths
					lowerCAmelCase   					=  embed_dims



			def 							a_       (   self):




					"""simple docstring"""


					lowerCAmelCase   					=  floats_tensor([self.batch_size, self.num_channels, self.image_size, self.image_size])

					lowerCAmelCase   					=  None
					if self.use_labels:
							lowerCAmelCase   					=  ids_tensor([self.batch_size]       ,    self.num_labels)

					lowerCAmelCase   					=  self.get_config()

					return config, pixel_values, labels



			def 							a_       (   self):




					"""simple docstring"""


					return SwiftFormerConfig(
					    depths=self.layer_depths       ,    embed_dims=self.embed_dims       ,    mlp_ratio=4       ,    downsamples=[True, True, True, True]       ,    hidden_act="""gelu"""       ,    num_labels=self.num_labels       ,    down_patch_size=3       ,    down_stride=2       ,    down_pad=1       ,    drop_rate=0.0       ,    drop_path_rate=0.0       ,    use_layer_scale=__lowerCAmelCase       ,    layer_scale_init_value=1E-5       ,    )



			def 							a_       (   self       ,    __lowerCAmelCase       ,    __lowerCAmelCase       ,    __lowerCAmelCase):




					"""simple docstring"""


					lowerCAmelCase   					=  SwiftFormerModel(config=__lowerCAmelCase)
					model.to(__lowerCAmelCase)
					model.eval()
					lowerCAmelCase   					=  model(__lowerCAmelCase)
					self.parent.assertEqual(result.last_hidden_state.shape       ,    (self.batch_size, self.embed_dims[-1], 7, 7))



			def 							a_       (   self       ,    __lowerCAmelCase       ,    __lowerCAmelCase       ,    __lowerCAmelCase):




					"""simple docstring"""


					lowerCAmelCase   					=  self.num_labels
					lowerCAmelCase   					=  SwiftFormerForImageClassification(__lowerCAmelCase)
					model.to(__lowerCAmelCase)
					model.eval()
					lowerCAmelCase   					=  model(__lowerCAmelCase       ,    labels=__lowerCAmelCase)
					self.parent.assertEqual(result.logits.shape       ,    (self.batch_size, self.num_labels))

					lowerCAmelCase   					=  SwiftFormerForImageClassification(__lowerCAmelCase)
					model.to(__lowerCAmelCase)
					model.eval()

					lowerCAmelCase   					=  floats_tensor([self.batch_size, self.num_channels, self.image_size, self.image_size])
					lowerCAmelCase   					=  model(__lowerCAmelCase)
					self.parent.assertEqual(result.logits.shape       ,    (self.batch_size, self.num_labels))







			def 							a_       (   self):




					"""simple docstring"""


					((lowerCAmelCase)					,     (lowerCAmelCase)					,     (lowerCAmelCase))   					=  self.prepare_config_and_inputs()
					lowerCAmelCase   					=  {"""pixel_values""": pixel_values}
					return config, inputs_dict







@require_torch
class 				a__(					lowerCAmelCase__ ,						lowerCAmelCase__ ,						unittest.TestCase	):


			'''simple docstring'''






			UpperCAmelCase_			:     Dict      =       (SwiftFormerModel, SwiftFormerForImageClassification) if is_torch_available() else ()
			UpperCAmelCase_			:     List[str]      =       (
			    {'''feature-extraction''': SwiftFormerModel, '''image-classification''': SwiftFormerForImageClassification}
			    if is_torch_available()
			    else {}
			)

			UpperCAmelCase_			:     Union[str, Any]      =       False
			UpperCAmelCase_			:     Optional[int]      =       False
			UpperCAmelCase_			:     Optional[Any]      =       False
			UpperCAmelCase_			:     Any      =       False
			UpperCAmelCase_			:     Optional[Any]      =       False



			def 							a_       (   self):




					"""simple docstring"""


					lowerCAmelCase   					=  SwiftFormerModelTester(self)
					lowerCAmelCase   					=  ConfigTester(
					    self       ,    config_class=__lowerCAmelCase       ,    has_text_modality=__lowerCAmelCase       ,    hidden_size=37       ,    num_attention_heads=12       ,    num_hidden_layers=12       ,    )



			def 							a_       (   self):




					"""simple docstring"""


					self.config_tester.run_common_tests()



			@unittest.skip(reason="""SwiftFormer does not use inputs_embeds""")
			def 							a_       (   self):




					"""simple docstring"""


					pass



			def 							a_       (   self):




					"""simple docstring"""


					lowerCAmelCase					,     lowerCAmelCase   					=  self.model_tester.prepare_config_and_inputs_for_common()

					for model_class in self.all_model_classes:
							lowerCAmelCase   					=  model_class(__lowerCAmelCase)
							lowerCAmelCase   					=  model.get_output_embeddings()
							self.assertTrue(x is None or isinstance(__lowerCAmelCase       ,    nn.Linear))



			def 							a_       (   self):




					"""simple docstring"""


					lowerCAmelCase					,     lowerCAmelCase   					=  self.model_tester.prepare_config_and_inputs_for_common()

					for model_class in self.all_model_classes:
							lowerCAmelCase   					=  model_class(__lowerCAmelCase)
							lowerCAmelCase   					=  inspect.signature(model.forward)
							# signature.parameters is an OrderedDict => so arg_names order is deterministic
							lowerCAmelCase   					=  [*signature.parameters.keys()]

							lowerCAmelCase   					=  ["""pixel_values"""]
							self.assertListEqual(arg_names[:1]       ,    __lowerCAmelCase)



			def 							a_       (   self):




					"""simple docstring"""


					lowerCAmelCase   					=  self.model_tester.prepare_config_and_inputs()
					self.model_tester.create_and_check_model(*__lowerCAmelCase)



			def 							a_       (   self):




					"""simple docstring"""


					lowerCAmelCase   					=  self.model_tester.prepare_config_and_inputs()
					self.model_tester.create_and_check_for_image_classification(*__lowerCAmelCase)



			@slow
			def 							a_       (   self):




					"""simple docstring"""


					for model_name in SWIFTFORMER_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
							lowerCAmelCase   					=  SwiftFormerModel.from_pretrained(__lowerCAmelCase)
							self.assertIsNotNone(__lowerCAmelCase)



			@unittest.skip(reason="""SwiftFormer does not output attentions""")
			def 							a_       (   self):




					"""simple docstring"""


					pass



			def 							a_       (   self):




					"""simple docstring"""


					def check_hidden_states_output(__lowerCAmelCase       ,    __lowerCAmelCase       ,    __lowerCAmelCase):
							lowerCAmelCase   					=  model_class(__lowerCAmelCase)
							model.to(__lowerCAmelCase)
							model.eval()

							with torch.no_grad():
									lowerCAmelCase   					=  model(**self._prepare_for_class(__lowerCAmelCase       ,    __lowerCAmelCase))

							lowerCAmelCase   					=  outputs.hidden_states

							lowerCAmelCase   					=  8
							self.assertEqual(len(__lowerCAmelCase)       ,    __lowerCAmelCase)  # TODO

							# SwiftFormer's feature maps are of shape (batch_size, embed_dims, height, width)
							# with the width and height being successively divided by 2, after every 2 blocks
							for i in range(len(__lowerCAmelCase)):
									self.assertEqual(
									    hidden_states[i].shape       ,    torch.Size(
									        [
									            self.model_tester.batch_size,
									            self.model_tester.embed_dims[i // 2],
									            (self.model_tester.image_size // 4) // 2 ** (i // 2),
									            (self.model_tester.image_size // 4) // 2 ** (i // 2),
									        ])       ,    )

					lowerCAmelCase					,     lowerCAmelCase   					=  self.model_tester.prepare_config_and_inputs_for_common()

					for model_class in self.all_model_classes:
							lowerCAmelCase   					=  True
							check_hidden_states_output(__lowerCAmelCase       ,    __lowerCAmelCase       ,    __lowerCAmelCase)

							# check that output_hidden_states also work using config
							del inputs_dict["output_hidden_states"]
							lowerCAmelCase   					=  True

							check_hidden_states_output(__lowerCAmelCase       ,    __lowerCAmelCase       ,    __lowerCAmelCase)



			def 							a_       (   self):




					"""simple docstring"""


					def _config_zero_init(__lowerCAmelCase):
							lowerCAmelCase   					=  copy.deepcopy(__lowerCAmelCase)
							for key in configs_no_init.__dict__.keys():
									if "_range" in key or "_std" in key or "initializer_factor" in key or "layer_scale" in key:
											setattr(__lowerCAmelCase       ,    __lowerCAmelCase       ,    1E-1_0)
									if isinstance(getattr(__lowerCAmelCase       ,    __lowerCAmelCase       ,    __lowerCAmelCase)       ,    __lowerCAmelCase):
											lowerCAmelCase   					=  _config_zero_init(getattr(__lowerCAmelCase       ,    __lowerCAmelCase))
											setattr(__lowerCAmelCase       ,    __lowerCAmelCase       ,    __lowerCAmelCase)
							return configs_no_init

					lowerCAmelCase					,     lowerCAmelCase   					=  self.model_tester.prepare_config_and_inputs_for_common()

					lowerCAmelCase   					=  _config_zero_init(__lowerCAmelCase)
					for model_class in self.all_model_classes:
							lowerCAmelCase   					=  model_class(config=__lowerCAmelCase)
							for name, param in model.named_parameters():
									if param.requires_grad:
											self.assertIn(
											    ((param.data.mean() * 1E9) / 1E9).round().item()       ,    [0.0, 1.0]       ,    msg=f"Parameter {name} of model {model_class} seems not properly initialized"       ,    )







			@unittest.skip("""Will be fixed soon by reducing the size of the model used for common tests.""")
			def 							a_       (   self):




					"""simple docstring"""


					pass







def 				snake_case__							(			)    ->	Any:

		'''simple docstring'''


		lowerCAmelCase   					=  Image.open("""./tests/fixtures/tests_samples/COCO/000000039769.png""" )
		return image






@require_torch
@require_vision
class 				a__(					unittest.TestCase	):


			'''simple docstring'''






			@cached_property
			def 							a_       (   self):




					"""simple docstring"""


					return ViTImageProcessor.from_pretrained("""MBZUAI/swiftformer-xs""") if is_vision_available() else None







			@slow
			def 							a_       (   self):




					"""simple docstring"""


					lowerCAmelCase   					=  SwiftFormerForImageClassification.from_pretrained("""MBZUAI/swiftformer-xs""").to(__lowerCAmelCase)

					lowerCAmelCase   					=  self.default_image_processor
					lowerCAmelCase   					=  prepare_img()
					lowerCAmelCase   					=  image_processor(images=__lowerCAmelCase       ,    return_tensors="""pt""").to(__lowerCAmelCase)

					# forward pass
					with torch.no_grad():
							lowerCAmelCase   					=  model(**__lowerCAmelCase)

					# verify the logits
					lowerCAmelCase   					=  torch.Size((1, 1000))
					self.assertEqual(outputs.logits.shape       ,    __lowerCAmelCase)

					lowerCAmelCase   					=  torch.tensor([[-2.1_7_0_3E0_0, 2.1_1_0_7E0_0, -2.0_8_1_1E0_0]]).to(__lowerCAmelCase)
					self.assertTrue(torch.allclose(outputs.logits[0, :3]       ,    __lowerCAmelCase       ,    atol=1E-4))





'''simple docstring'''




import pickle
import shutil
import tempfile
import unittest

from transformers import SPIECE_UNDERLINE, XLMRobertaTokenizer, XLMRobertaTokenizerFast
from transformers.testing_utils import get_tests_dir, require_sentencepiece, require_tokenizers, slow
from transformers.utils import cached_property

from ...test_tokenization_common import TokenizerTesterMixin


__lowercase          =  get_tests_dir('''fixtures/test_sentencepiece.model''')






@require_sentencepiece
@require_tokenizers
class 				a__(					lowerCAmelCase__ ,						unittest.TestCase	):


			'''simple docstring'''






			UpperCAmelCase_			:     Tuple      =       XLMRobertaTokenizer
			UpperCAmelCase_			:     int      =       XLMRobertaTokenizerFast
			UpperCAmelCase_			:     List[str]      =       True
			UpperCAmelCase_			:     Optional[int]      =       True



			def 							a_       (   self):




					"""simple docstring"""


					super().setUp()

					# We have a SentencePiece fixture for testing
					lowerCAmelCase   					=  XLMRobertaTokenizer(__lowerCAmelCase       ,    keep_accents=__lowerCAmelCase)
					tokenizer.save_pretrained(self.tmpdirname)



			def 							a_       (   self):




					"""simple docstring"""


					lowerCAmelCase   					=  """<pad>"""
					lowerCAmelCase   					=  1

					self.assertEqual(self.get_tokenizer()._convert_token_to_id(__lowerCAmelCase)       ,    __lowerCAmelCase)
					self.assertEqual(self.get_tokenizer()._convert_id_to_token(__lowerCAmelCase)       ,    __lowerCAmelCase)



			def 							a_       (   self):




					"""simple docstring"""


					lowerCAmelCase   					=  list(self.get_tokenizer().get_vocab().keys())

					self.assertEqual(vocab_keys[0]       ,    """<s>""")
					self.assertEqual(vocab_keys[1]       ,    """<pad>""")
					self.assertEqual(vocab_keys[-1]       ,    """<mask>""")
					self.assertEqual(len(__lowerCAmelCase)       ,    1002)



			def 							a_       (   self):




					"""simple docstring"""


					self.assertEqual(self.get_tokenizer().vocab_size       ,    1002)



			def 							a_       (   self):




					"""simple docstring"""


					lowerCAmelCase   					=  XLMRobertaTokenizer(__lowerCAmelCase       ,    keep_accents=__lowerCAmelCase)

					lowerCAmelCase   					=  tokenizer.tokenize("""This is a test""")
					self.assertListEqual(__lowerCAmelCase       ,    ["""▁This""", """▁is""", """▁a""", """▁t""", """est"""])

					self.assertListEqual(
					    tokenizer.convert_tokens_to_ids(__lowerCAmelCase)       ,    [value + tokenizer.fairseq_offset for value in [285, 46, 10, 170, 382]]       ,    )

					lowerCAmelCase   					=  tokenizer.tokenize("""I was born in 92000, and this is falsé.""")
					self.assertListEqual(
					    __lowerCAmelCase       ,    [
					        SPIECE_UNDERLINE + """I""",
					        SPIECE_UNDERLINE + """was""",
					        SPIECE_UNDERLINE + """b""",
					        """or""",
					        """n""",
					        SPIECE_UNDERLINE + """in""",
					        SPIECE_UNDERLINE + """""",
					        """9""",
					        """2""",
					        """0""",
					        """0""",
					        """0""",
					        """,""",
					        SPIECE_UNDERLINE + """and""",
					        SPIECE_UNDERLINE + """this""",
					        SPIECE_UNDERLINE + """is""",
					        SPIECE_UNDERLINE + """f""",
					        """al""",
					        """s""",
					        """é""",
					        """.""",
					    ]       ,    )
					lowerCAmelCase   					=  tokenizer.convert_tokens_to_ids(__lowerCAmelCase)
					self.assertListEqual(
					    __lowerCAmelCase       ,    [
					        value + tokenizer.fairseq_offset
					        for value in [8, 21, 84, 55, 24, 19, 7, 2, 602, 347, 347, 347, 3, 12, 66, 46, 72, 80, 6, 2, 4]
					        #                                       ^ unk: 2 + 1 = 3                  unk: 2 + 1 = 3 ^
					    ]       ,    )

					lowerCAmelCase   					=  tokenizer.convert_ids_to_tokens(__lowerCAmelCase)
					self.assertListEqual(
					    __lowerCAmelCase       ,    [
					        SPIECE_UNDERLINE + """I""",
					        SPIECE_UNDERLINE + """was""",
					        SPIECE_UNDERLINE + """b""",
					        """or""",
					        """n""",
					        SPIECE_UNDERLINE + """in""",
					        SPIECE_UNDERLINE + """""",
					        """<unk>""",
					        """2""",
					        """0""",
					        """0""",
					        """0""",
					        """,""",
					        SPIECE_UNDERLINE + """and""",
					        SPIECE_UNDERLINE + """this""",
					        SPIECE_UNDERLINE + """is""",
					        SPIECE_UNDERLINE + """f""",
					        """al""",
					        """s""",
					        """<unk>""",
					        """.""",
					    ]       ,    )



			def 							a_       (   self):




					"""simple docstring"""


					if not self.test_slow_tokenizer:
							# as we don't have a slow version, we can't compare the outputs between slow and fast versions
							return

					lowerCAmelCase   					=  (self.rust_tokenizer_class, """hf-internal-testing/tiny-xlm-roberta""", {})
					for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
							with self.subTest(f"{tokenizer.__class__.__name__} ({pretrained_name})"):
									lowerCAmelCase   					=  self.rust_tokenizer_class.from_pretrained(__lowerCAmelCase       ,    **__lowerCAmelCase)
									lowerCAmelCase   					=  self.tokenizer_class.from_pretrained(__lowerCAmelCase       ,    **__lowerCAmelCase)

									lowerCAmelCase   					=  tempfile.mkdtemp()

									lowerCAmelCase   					=  tokenizer_r.save_pretrained(__lowerCAmelCase)
									lowerCAmelCase   					=  tokenizer_p.save_pretrained(__lowerCAmelCase)

									# Checks it save with the same files + the tokenizer.json file for the fast one
									self.assertTrue(any("""tokenizer.json""" in f for f in tokenizer_r_files))
									lowerCAmelCase   					=  tuple(f for f in tokenizer_r_files if """tokenizer.json""" not in f)
									self.assertSequenceEqual(__lowerCAmelCase       ,    __lowerCAmelCase)

									# Checks everything loads correctly in the same way
									lowerCAmelCase   					=  tokenizer_r.from_pretrained(__lowerCAmelCase)
									lowerCAmelCase   					=  tokenizer_p.from_pretrained(__lowerCAmelCase)

									# Check special tokens are set accordingly on Rust and Python
									for key in tokenizer_pp.special_tokens_map:
											self.assertTrue(hasattr(__lowerCAmelCase       ,    __lowerCAmelCase))
											# self.assertEqual(getattr(tokenizer_rp, key), getattr(tokenizer_pp, key))
											# self.assertEqual(getattr(tokenizer_rp, key + "_id"), getattr(tokenizer_pp, key + "_id"))

									shutil.rmtree(__lowerCAmelCase)

									# Save tokenizer rust, legacy_format=True
									lowerCAmelCase   					=  tempfile.mkdtemp()

									lowerCAmelCase   					=  tokenizer_r.save_pretrained(__lowerCAmelCase       ,    legacy_format=__lowerCAmelCase)
									lowerCAmelCase   					=  tokenizer_p.save_pretrained(__lowerCAmelCase)

									# Checks it save with the same files
									self.assertSequenceEqual(__lowerCAmelCase       ,    __lowerCAmelCase)

									# Checks everything loads correctly in the same way
									lowerCAmelCase   					=  tokenizer_r.from_pretrained(__lowerCAmelCase)
									lowerCAmelCase   					=  tokenizer_p.from_pretrained(__lowerCAmelCase)

									# Check special tokens are set accordingly on Rust and Python
									for key in tokenizer_pp.special_tokens_map:
											self.assertTrue(hasattr(__lowerCAmelCase       ,    __lowerCAmelCase))

									shutil.rmtree(__lowerCAmelCase)

									# Save tokenizer rust, legacy_format=False
									lowerCAmelCase   					=  tempfile.mkdtemp()

									lowerCAmelCase   					=  tokenizer_r.save_pretrained(__lowerCAmelCase       ,    legacy_format=__lowerCAmelCase)
									lowerCAmelCase   					=  tokenizer_p.save_pretrained(__lowerCAmelCase)

									# Checks it saved the tokenizer.json file
									self.assertTrue(any("""tokenizer.json""" in f for f in tokenizer_r_files))

									# Checks everything loads correctly in the same way
									lowerCAmelCase   					=  tokenizer_r.from_pretrained(__lowerCAmelCase)
									lowerCAmelCase   					=  tokenizer_p.from_pretrained(__lowerCAmelCase)

									# Check special tokens are set accordingly on Rust and Python
									for key in tokenizer_pp.special_tokens_map:
											self.assertTrue(hasattr(__lowerCAmelCase       ,    __lowerCAmelCase))

									shutil.rmtree(__lowerCAmelCase)



			@cached_property
			def 							a_       (   self):




					"""simple docstring"""


					return XLMRobertaTokenizer.from_pretrained("""xlm-roberta-base""")



			def 							a_       (   self):




					"""simple docstring"""


					with tempfile.NamedTemporaryFile() as f:
							shutil.copyfile(__lowerCAmelCase       ,    f.name)
							lowerCAmelCase   					=  XLMRobertaTokenizer(f.name       ,    keep_accents=__lowerCAmelCase)
							lowerCAmelCase   					=  pickle.dumps(__lowerCAmelCase)
					pickle.loads(__lowerCAmelCase)



			def 							a_       (   self):




					"""simple docstring"""


					if not self.test_rust_tokenizer:
							return

					lowerCAmelCase   					=  self.get_tokenizer()
					lowerCAmelCase   					=  self.get_rust_tokenizer()

					lowerCAmelCase   					=  """I was born in 92000, and this is falsé."""

					lowerCAmelCase   					=  tokenizer.tokenize(__lowerCAmelCase)
					lowerCAmelCase   					=  rust_tokenizer.tokenize(__lowerCAmelCase)
					self.assertListEqual(__lowerCAmelCase       ,    __lowerCAmelCase)

					lowerCAmelCase   					=  tokenizer.encode(__lowerCAmelCase       ,    add_special_tokens=__lowerCAmelCase)
					lowerCAmelCase   					=  rust_tokenizer.encode(__lowerCAmelCase       ,    add_special_tokens=__lowerCAmelCase)
					self.assertListEqual(__lowerCAmelCase       ,    __lowerCAmelCase)

					lowerCAmelCase   					=  self.get_rust_tokenizer()
					lowerCAmelCase   					=  tokenizer.encode(__lowerCAmelCase)
					lowerCAmelCase   					=  rust_tokenizer.encode(__lowerCAmelCase)
					self.assertListEqual(__lowerCAmelCase       ,    __lowerCAmelCase)



			@slow
			def 							a_       (   self):




					"""simple docstring"""


					lowerCAmelCase   					=  """Hello World!"""
					lowerCAmelCase   					=  [0, 35378, 6661, 38, 2]
					# xlmr = torch.hub.load('pytorch/fairseq', 'xlmr.base')  # xlmr.large has same tokenizer
					# xlmr.eval()
					# xlmr.encode(symbols)

					self.assertListEqual(__lowerCAmelCase       ,    self.big_tokenizer.encode(__lowerCAmelCase))



			@slow
			def 							a_       (   self):




					"""simple docstring"""


					lowerCAmelCase   					=  (
					    """This is a very long text with a lot of weird characters, such as: . , ~ ? ( ) \" [ ] ! : - . Also we will"""
					    """ add words that should not exsist and be tokenized to <unk>, such as saoneuhaoesuth"""
					)
					lowerCAmelCase   					=  [
					    0,
					    3293,
					    83,
					    10,
					    4552,
					    4989,
					    7986,
					    678,
					    10,
					    5915,
					    111,
					    179459,
					    124850,
					    4,
					    6044,
					    237,
					    12,
					    6,
					    5,
					    6,
					    4,
					    6780,
					    705,
					    15,
					    1388,
					    44,
					    378,
					    10114,
					    711,
					    152,
					    20,
					    6,
					    5,
					    22376,
					    642,
					    1221,
					    15190,
					    34153,
					    450,
					    5608,
					    959,
					    1119,
					    57702,
					    136,
					    186,
					    47,
					    1098,
					    29367,
					    47,
					    # 4426, # What fairseq tokenizes from "<unk>": "_<"
					    # 3678, # What fairseq tokenizes from "<unk>": "unk"
					    # 2740, # What fairseq tokenizes from "<unk>": ">"
					    3,  # What we tokenize from "<unk>": "<unk>"
					    6,  # Residue from the tokenization: an extra sentencepiece underline
					    4,
					    6044,
					    237,
					    6284,
					    50901,
					    528,
					    31,
					    90,
					    34,
					    927,
					    2,
					]
					# xlmr = torch.hub.load('pytorch/fairseq', 'xlmr.base')  # xlmr.large has same tokenizer
					# xlmr.eval()
					# xlmr.encode(symbols)

					self.assertListEqual(__lowerCAmelCase       ,    self.big_tokenizer.encode(__lowerCAmelCase))







			@slow
			def 							a_       (   self):




					"""simple docstring"""


					lowerCAmelCase   					=  {"""input_ids""": [[0, 11062, 82772, 7, 15, 82772, 538, 51529, 237, 17198, 1290, 206, 9, 215175, 1314, 136, 17198, 1290, 206, 9, 56359, 42, 122009, 9, 16466, 16, 87344, 4537, 9, 4717, 78381, 6, 159958, 7, 15, 24480, 618, 4, 527, 22693, 5428, 4, 2777, 24480, 9874, 4, 43523, 594, 4, 803, 18392, 33189, 18, 4, 43523, 24447, 12399, 100, 24955, 83658, 9626, 144057, 15, 839, 22335, 16, 136, 24955, 83658, 83479, 15, 39102, 724, 16, 678, 645, 2789, 1328, 4589, 42, 122009, 115774, 23, 805, 1328, 46876, 7, 136, 53894, 1940, 42227, 41159, 17721, 823, 425, 4, 27512, 98722, 206, 136, 5531, 4970, 919, 17336, 5, 2], [0, 20080, 618, 83, 82775, 47, 479, 9, 1517, 73, 53894, 333, 80581, 110117, 18811, 5256, 1295, 51, 152526, 297, 7986, 390, 124416, 538, 35431, 214, 98, 15044, 25737, 136, 7108, 43701, 23, 756, 135355, 7, 5, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [0, 581, 63773, 119455, 6, 147797, 88203, 7, 645, 70, 21, 3285, 10269, 5, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]], """attention_mask""": [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]]}  # noqa: E501
					# fmt: on

					self.tokenizer_integration_test_util(
					    expected_encoding=__lowerCAmelCase       ,    model_name="""xlm-roberta-base"""       ,    revision="""d9d8a8ea5eb94b1c6654ae9249df7793cd2933d3"""       ,    )




