





"""simple docstring"""





import importlib
import json
import os
from collections import OrderedDict
from typing import Dict, Optional, Union

# Build the list of all image processors
from ...configuration_utils import PretrainedConfig
from ...dynamic_module_utils import get_class_from_dynamic_module, resolve_trust_remote_code
from ...image_processing_utils import ImageProcessingMixin
from ...utils import CONFIG_NAME, IMAGE_PROCESSOR_NAME, get_file_from_repo, logging
from .auto_factory import _LazyAutoMapping
from .configuration_auto import (
    CONFIG_MAPPING_NAMES,
    AutoConfig,
    model_type_to_module_name,
    replace_list_option_in_docstrings,
)


SCREAMING_SNAKE_CASE__					=   logging.get_logger(__name__)

SCREAMING_SNAKE_CASE__					=   OrderedDict(
    [
        ("align", "EfficientNetImageProcessor"),
        ("beit", "BeitImageProcessor"),
        ("bit", "BitImageProcessor"),
        ("blip", "BlipImageProcessor"),
        ("blip-2", "BlipImageProcessor"),
        ("bridgetower", "BridgeTowerImageProcessor"),
        ("chinese_clip", "ChineseCLIPImageProcessor"),
        ("clip", "CLIPImageProcessor"),
        ("clipseg", "ViTImageProcessor"),
        ("conditional_detr", "ConditionalDetrImageProcessor"),
        ("convnext", "ConvNextImageProcessor"),
        ("convnextv2", "ConvNextImageProcessor"),
        ("cvt", "ConvNextImageProcessor"),
        ("data2vec-vision", "BeitImageProcessor"),
        ("deformable_detr", "DeformableDetrImageProcessor"),
        ("deit", "DeiTImageProcessor"),
        ("deta", "DetaImageProcessor"),
        ("detr", "DetrImageProcessor"),
        ("dinat", "ViTImageProcessor"),
        ("donut-swin", "DonutImageProcessor"),
        ("dpt", "DPTImageProcessor"),
        ("efficientformer", "EfficientFormerImageProcessor"),
        ("efficientnet", "EfficientNetImageProcessor"),
        ("flava", "FlavaImageProcessor"),
        ("focalnet", "BitImageProcessor"),
        ("git", "CLIPImageProcessor"),
        ("glpn", "GLPNImageProcessor"),
        ("groupvit", "CLIPImageProcessor"),
        ("imagegpt", "ImageGPTImageProcessor"),
        ("instructblip", "BlipImageProcessor"),
        ("layoutlmv2", "LayoutLMv2ImageProcessor"),
        ("layoutlmv3", "LayoutLMv3ImageProcessor"),
        ("levit", "LevitImageProcessor"),
        ("mask2former", "Mask2FormerImageProcessor"),
        ("maskformer", "MaskFormerImageProcessor"),
        ("mgp-str", "ViTImageProcessor"),
        ("mobilenet_v1", "MobileNetV1ImageProcessor"),
        ("mobilenet_v2", "MobileNetV2ImageProcessor"),
        ("mobilevit", "MobileViTImageProcessor"),
        ("mobilevit", "MobileViTImageProcessor"),
        ("mobilevitv2", "MobileViTImageProcessor"),
        ("nat", "ViTImageProcessor"),
        ("oneformer", "OneFormerImageProcessor"),
        ("owlvit", "OwlViTImageProcessor"),
        ("perceiver", "PerceiverImageProcessor"),
        ("pix2struct", "Pix2StructImageProcessor"),
        ("poolformer", "PoolFormerImageProcessor"),
        ("regnet", "ConvNextImageProcessor"),
        ("resnet", "ConvNextImageProcessor"),
        ("sam", "SamImageProcessor"),
        ("segformer", "SegformerImageProcessor"),
        ("swiftformer", "ViTImageProcessor"),
        ("swin", "ViTImageProcessor"),
        ("swin2sr", "Swin2SRImageProcessor"),
        ("swinv2", "ViTImageProcessor"),
        ("table-transformer", "DetrImageProcessor"),
        ("timesformer", "VideoMAEImageProcessor"),
        ("tvlt", "TvltImageProcessor"),
        ("upernet", "SegformerImageProcessor"),
        ("van", "ConvNextImageProcessor"),
        ("videomae", "VideoMAEImageProcessor"),
        ("vilt", "ViltImageProcessor"),
        ("vit", "ViTImageProcessor"),
        ("vit_hybrid", "ViTHybridImageProcessor"),
        ("vit_mae", "ViTImageProcessor"),
        ("vit_msn", "ViTImageProcessor"),
        ("xclip", "CLIPImageProcessor"),
        ("yolos", "YolosImageProcessor"),
    ]
)

SCREAMING_SNAKE_CASE__					=   _LazyAutoMapping(CONFIG_MAPPING_NAMES, IMAGE_PROCESSOR_MAPPING_NAMES)


def 			UpperCAmelCase__							(  SCREAMING_SNAKE_CASE     :						str			):







      '''simple docstring'''




      for module_name, extractors in IMAGE_PROCESSOR_MAPPING_NAMES.items():
            if class_name in extractors:
                  lowerCAmelCase  						=       model_type_to_module_name(SCREAMING_SNAKE_CASE			)

                  lowerCAmelCase  						=       importlib.import_module(F'.{module_name}'							,						"""transformers.models"""			)
                  try:
                        return getattr(SCREAMING_SNAKE_CASE							,						SCREAMING_SNAKE_CASE			)
                  except AttributeError:
                        continue

      for _, extractor in IMAGE_PROCESSOR_MAPPING._extra_content.items():
            if getattr(SCREAMING_SNAKE_CASE							,						"""__name__"""							,						SCREAMING_SNAKE_CASE			) == class_name:
                  return extractor

    # We did not fine the class, but maybe it's because a dep is missing. In that case, the class will be in the main
    # init and we return the proper dummy to get an appropriate error message.
      lowerCAmelCase  						=       importlib.import_module("""transformers"""			)
      if hasattr(SCREAMING_SNAKE_CASE							,						SCREAMING_SNAKE_CASE			):
            return getattr(SCREAMING_SNAKE_CASE							,						SCREAMING_SNAKE_CASE			)

      return None


def 			UpperCAmelCase__							(  SCREAMING_SNAKE_CASE     :						Union[str, os.PathLike]							,						SCREAMING_SNAKE_CASE     :						Optional[Union[str, os.PathLike]] = None							,						SCREAMING_SNAKE_CASE     :						bool = False							,						SCREAMING_SNAKE_CASE     :						bool = False							,						SCREAMING_SNAKE_CASE     :						Optional[Dict[str, str]] = None							,						SCREAMING_SNAKE_CASE     :						Optional[Union[bool, str]] = None							,						SCREAMING_SNAKE_CASE     :						Optional[str] = None							,						SCREAMING_SNAKE_CASE     :						bool = False							,						**SCREAMING_SNAKE_CASE     :						List[str]							,						):







      '''simple docstring'''




      lowerCAmelCase  						=       get_file_from_repo(
          SCREAMING_SNAKE_CASE							,						SCREAMING_SNAKE_CASE							,						cache_dir=SCREAMING_SNAKE_CASE							,						force_download=SCREAMING_SNAKE_CASE							,						resume_download=SCREAMING_SNAKE_CASE							,						proxies=SCREAMING_SNAKE_CASE							,						use_auth_token=SCREAMING_SNAKE_CASE							,						revision=SCREAMING_SNAKE_CASE							,						local_files_only=SCREAMING_SNAKE_CASE							,						)
      if resolved_config_file is None:
            logger.info(
                """Could not locate the image processor configuration file, will try to use the model config instead."""			)
            return {}

      with open(SCREAMING_SNAKE_CASE							,						encoding="""utf-8"""			) as reader:
            return json.load(SCREAMING_SNAKE_CASE			)






class 		lowercase					:





      def __init__(       self )		->     str:
            raise EnvironmentError(
                """AutoImageProcessor is designed to be instantiated """
                """using the `AutoImageProcessor.from_pretrained(pretrained_model_name_or_path)` method.""" )





      @classmethod
      @replace_list_option_in_docstrings(lowercase )
      def     _snake_case			(       cls      ,						lowercase      ,						**lowercase )		->     Union[str, Any]:
            lowerCAmelCase  						=       kwargs.pop("""config"""      ,						lowercase )
            lowerCAmelCase  						=       kwargs.pop("""trust_remote_code"""      ,						lowercase )
            lowerCAmelCase  						=       True

            lowerCAmelCase						,		lowerCAmelCase  						=       ImageProcessingMixin.get_image_processor_dict(lowercase      ,						**lowercase )
            lowerCAmelCase  						=       config_dict.get("""image_processor_type"""      ,						lowercase )
            lowerCAmelCase  						=       None
            if "AutoImageProcessor" in config_dict.get("""auto_map"""      ,						{} ):
                  lowerCAmelCase  						=       config_dict["""auto_map"""]["""AutoImageProcessor"""]

            # If we still don't have the image processor class, check if we're loading from a previous feature extractor config
            # and if so, infer the image processor class from there.
            if image_processor_class is None and image_processor_auto_map is None:
                  lowerCAmelCase  						=       config_dict.pop("""feature_extractor_type"""      ,						lowercase )
                  if feature_extractor_class is not None:
                        logger.warning(
                            """Could not find image processor class in the image processor config or the model config. Loading"""
                            """ based on pattern matching with the model's feature extractor configuration.""" )
                        lowerCAmelCase  						=       feature_extractor_class.replace("""FeatureExtractor"""      ,						"""ImageProcessor""" )
                  if "AutoFeatureExtractor" in config_dict.get("""auto_map"""      ,						{} ):
                        lowerCAmelCase  						=       config_dict["""auto_map"""]["""AutoFeatureExtractor"""]
                        lowerCAmelCase  						=       feature_extractor_auto_map.replace("""FeatureExtractor"""      ,						"""ImageProcessor""" )
                        logger.warning(
                            """Could not find image processor auto map in the image processor config or the model config."""
                            """ Loading based on pattern matching with the model's feature extractor configuration.""" )

        # If we don't find the image processor class in the image processor config, let's try the model config.
            if image_processor_class is None and image_processor_auto_map is None:
                  if not isinstance(lowercase      ,						lowercase ):
                        lowerCAmelCase  						=       AutoConfig.from_pretrained(lowercase      ,						**lowercase )
                  # It could be in `config.image_processor_type``
                  lowerCAmelCase  						=       getattr(lowercase      ,						"""image_processor_type"""      ,						lowercase )
                  if hasattr(lowercase      ,						"""auto_map""" ) and "AutoImageProcessor" in config.auto_map:
                        lowerCAmelCase  						=       config.auto_map["""AutoImageProcessor"""]

            if image_processor_class is not None:
                  lowerCAmelCase  						=       image_processor_class_from_name(lowercase )

            lowerCAmelCase  						=       image_processor_auto_map is not None
            lowerCAmelCase  						=       image_processor_class is not None or type(lowercase ) in IMAGE_PROCESSOR_MAPPING
            lowerCAmelCase  						=       resolve_trust_remote_code(
                lowercase      ,						lowercase      ,						lowercase      ,						lowercase )

            if has_remote_code and trust_remote_code:
                  lowerCAmelCase  						=       get_class_from_dynamic_module(
                      lowercase      ,						lowercase      ,						**lowercase )
                  lowerCAmelCase  						=       kwargs.pop("""code_revision"""      ,						lowercase )
                  if os.path.isdir(lowercase ):
                        image_processor_class.register_for_auto_class()
                  return image_processor_class.from_dict(lowercase      ,						**lowercase )
            elif image_processor_class is not None:
                  return image_processor_class.from_dict(lowercase      ,						**lowercase )
            # Last try: we use the IMAGE_PROCESSOR_MAPPING.
            elif type(lowercase ) in IMAGE_PROCESSOR_MAPPING:
                  lowerCAmelCase  						=       IMAGE_PROCESSOR_MAPPING[type(lowercase )]
                  return image_processor_class.from_dict(lowercase      ,						**lowercase )

            raise ValueError(
                f'Unrecognized image processor in {pretrained_model_name_or_path}. Should have a '
                f'`image_processor_type` key in its {IMAGE_PROCESSOR_NAME} of {CONFIG_NAME}, or one of the following '
                f'`model_type` keys in its {CONFIG_NAME}: {", ".join(c for c in IMAGE_PROCESSOR_MAPPING_NAMES.keys() )}' )





      @staticmethod
      def     _snake_case			(       lowercase      ,						lowercase )		->     Optional[Any]:
            IMAGE_PROCESSOR_MAPPING.register(lowercase      ,						lowercase )












import pickle
import shutil
import tempfile
import unittest

from transformers import SPIECE_UNDERLINE, XLMRobertaTokenizer, XLMRobertaTokenizerFast
from transformers.testing_utils import get_tests_dir, require_sentencepiece, require_tokenizers, slow
from transformers.utils import cached_property

from ...test_tokenization_common import TokenizerTesterMixin


__lowerCAmelCase		:				str       			=		get_tests_dir('fixtures/test_sentencepiece.model')




@require_sentencepiece
@require_tokenizers
class     snake_case__							(_UpperCamelCase    ,       unittest.TestCase   ):
				"""simple docstring"""



				SCREAMING_SNAKE_CASE_       :							Dict							 =					XLMRobertaTokenizer
				SCREAMING_SNAKE_CASE_       :							int							 =					XLMRobertaTokenizerFast
				SCREAMING_SNAKE_CASE_       :							str							 =					True
				SCREAMING_SNAKE_CASE_       :							Optional[Any]							 =					True





				def 							__UpperCAmelCase							(      self				:     int							)					->     List[str]:
							super().setUp()

							# We have a SentencePiece fixture for testing
							a    =			XLMRobertaTokenizer(__lowerCamelCase					,      keep_accents=__lowerCamelCase							)
							tokenizer.save_pretrained(self.tmpdirname							)





				def 							__UpperCAmelCase							(      self				:     List[str]							)					->     Any:
							a    =			"<pad>"
							a    =			1

							self.assertEqual(self.get_tokenizer()._convert_token_to_id(__lowerCamelCase							)					,      __lowerCamelCase							)
							self.assertEqual(self.get_tokenizer()._convert_id_to_token(__lowerCamelCase							)					,      __lowerCamelCase							)





				def 							__UpperCAmelCase							(      self				:     Dict							)					->     Optional[Any]:
							a    =			list(self.get_tokenizer().get_vocab().keys()							)

							self.assertEqual(vocab_keys[0]					,      "<s>"							)
							self.assertEqual(vocab_keys[1]					,      "<pad>"							)
							self.assertEqual(vocab_keys[-1]					,      "<mask>"							)
							self.assertEqual(len(__lowerCamelCase							)					,      10_02							)





				def 							__UpperCAmelCase							(      self				:     List[Any]							)					->     Any:
							self.assertEqual(self.get_tokenizer().vocab_size					,      10_02							)





				def 							__UpperCAmelCase							(      self				:     Dict							)					->     List[str]:
							a    =			XLMRobertaTokenizer(__lowerCamelCase					,      keep_accents=__lowerCamelCase							)

							a    =			tokenizer.tokenize("This is a test"							)
							self.assertListEqual(__lowerCamelCase					,      ["▁This", "▁is", "▁a", "▁t", "est"]							)

							self.assertListEqual(
							    tokenizer.convert_tokens_to_ids(__lowerCamelCase							)					,      [value + tokenizer.fairseq_offset for value in [2_85, 46, 10, 1_70, 3_82]]					,      )

							a    =			tokenizer.tokenize("I was born in 92000, and this is falsé."							)
							self.assertListEqual(
							    __lowerCamelCase					,      [
							        SPIECE_UNDERLINE + "I",
							        SPIECE_UNDERLINE + "was",
							        SPIECE_UNDERLINE + "b",
							        "or",
							        "n",
							        SPIECE_UNDERLINE + "in",
							        SPIECE_UNDERLINE + "",
							        "9",
							        "2",
							        "0",
							        "0",
							        "0",
							        ",",
							        SPIECE_UNDERLINE + "and",
							        SPIECE_UNDERLINE + "this",
							        SPIECE_UNDERLINE + "is",
							        SPIECE_UNDERLINE + "f",
							        "al",
							        "s",
							        "é",
							        ".",
							    ]					,      )
							a    =			tokenizer.convert_tokens_to_ids(__lowerCamelCase							)
							self.assertListEqual(
							    __lowerCamelCase					,      [
							        value + tokenizer.fairseq_offset
							        for value in [8, 21, 84, 55, 24, 19, 7, 2, 6_02, 3_47, 3_47, 3_47, 3, 12, 66, 46, 72, 80, 6, 2, 4]
							        #                                       ^ unk: 2 + 1 = 3                  unk: 2 + 1 = 3 ^
							    ]					,      )

							a    =			tokenizer.convert_ids_to_tokens(__lowerCamelCase							)
							self.assertListEqual(
							    __lowerCamelCase					,      [
							        SPIECE_UNDERLINE + "I",
							        SPIECE_UNDERLINE + "was",
							        SPIECE_UNDERLINE + "b",
							        "or",
							        "n",
							        SPIECE_UNDERLINE + "in",
							        SPIECE_UNDERLINE + "",
							        "<unk>",
							        "2",
							        "0",
							        "0",
							        "0",
							        ",",
							        SPIECE_UNDERLINE + "and",
							        SPIECE_UNDERLINE + "this",
							        SPIECE_UNDERLINE + "is",
							        SPIECE_UNDERLINE + "f",
							        "al",
							        "s",
							        "<unk>",
							        ".",
							    ]					,      )





				def 							__UpperCAmelCase							(      self				:     Optional[Any]							)					->     List[str]:
							if not self.test_slow_tokenizer:
										# as we don't have a slow version, we can't compare the outputs between slow and fast versions
										return

							a    =			(self.rust_tokenizer_class, "hf-internal-testing/tiny-xlm-roberta", {})
							for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
										with self.subTest(f"""{tokenizer.__class__.__name__} ({pretrained_name})"""							):
													a    =			self.rust_tokenizer_class.from_pretrained(__lowerCamelCase					,      **__lowerCamelCase							)
													a    =			self.tokenizer_class.from_pretrained(__lowerCamelCase					,      **__lowerCamelCase							)

													a    =			tempfile.mkdtemp()

													a    =			tokenizer_r.save_pretrained(__lowerCamelCase							)
													a    =			tokenizer_p.save_pretrained(__lowerCamelCase							)

													# Checks it save with the same files + the tokenizer.json file for the fast one
													self.assertTrue(any("tokenizer.json" in f for f in tokenizer_r_files							)							)
													a    =			tuple(f for f in tokenizer_r_files if "tokenizer.json" not in f							)
													self.assertSequenceEqual(__lowerCamelCase					,      __lowerCamelCase							)

													# Checks everything loads correctly in the same way
													a    =			tokenizer_r.from_pretrained(__lowerCamelCase							)
													a    =			tokenizer_p.from_pretrained(__lowerCamelCase							)

													# Check special tokens are set accordingly on Rust and Python
													for key in tokenizer_pp.special_tokens_map:
																self.assertTrue(hasattr(__lowerCamelCase					,      __lowerCamelCase							)							)
																# self.assertEqual(getattr(tokenizer_rp, key), getattr(tokenizer_pp, key))
																# self.assertEqual(getattr(tokenizer_rp, key + "_id"), getattr(tokenizer_pp, key + "_id"))

													shutil.rmtree(__lowerCamelCase							)

													# Save tokenizer rust, legacy_format=True
													a    =			tempfile.mkdtemp()

													a    =			tokenizer_r.save_pretrained(__lowerCamelCase					,      legacy_format=__lowerCamelCase							)
													a    =			tokenizer_p.save_pretrained(__lowerCamelCase							)

													# Checks it save with the same files
													self.assertSequenceEqual(__lowerCamelCase					,      __lowerCamelCase							)

													# Checks everything loads correctly in the same way
													a    =			tokenizer_r.from_pretrained(__lowerCamelCase							)
													a    =			tokenizer_p.from_pretrained(__lowerCamelCase							)

													# Check special tokens are set accordingly on Rust and Python
													for key in tokenizer_pp.special_tokens_map:
																self.assertTrue(hasattr(__lowerCamelCase					,      __lowerCamelCase							)							)

													shutil.rmtree(__lowerCamelCase							)

													# Save tokenizer rust, legacy_format=False
													a    =			tempfile.mkdtemp()

													a    =			tokenizer_r.save_pretrained(__lowerCamelCase					,      legacy_format=__lowerCamelCase							)
													a    =			tokenizer_p.save_pretrained(__lowerCamelCase							)

													# Checks it saved the tokenizer.json file
													self.assertTrue(any("tokenizer.json" in f for f in tokenizer_r_files							)							)

													# Checks everything loads correctly in the same way
													a    =			tokenizer_r.from_pretrained(__lowerCamelCase							)
													a    =			tokenizer_p.from_pretrained(__lowerCamelCase							)

													# Check special tokens are set accordingly on Rust and Python
													for key in tokenizer_pp.special_tokens_map:
																self.assertTrue(hasattr(__lowerCamelCase					,      __lowerCamelCase							)							)

													shutil.rmtree(__lowerCamelCase							)





				@cached_property
				def 							__UpperCAmelCase							(      self				:     List[str]							)					->     Union[str, Any]:
							return XLMRobertaTokenizer.from_pretrained("xlm-roberta-base"							)





				def 							__UpperCAmelCase							(      self				:     List[Any]							)					->     List[Any]:
							with tempfile.NamedTemporaryFile() as f:
										shutil.copyfile(__lowerCamelCase					,      f.name							)
										a    =			XLMRobertaTokenizer(f.name					,      keep_accents=__lowerCamelCase							)
										a    =			pickle.dumps(__lowerCamelCase							)
							pickle.loads(__lowerCamelCase							)





				def 							__UpperCAmelCase							(      self				:     int							)					->     str:
							if not self.test_rust_tokenizer:
										return

							a    =			self.get_tokenizer()
							a    =			self.get_rust_tokenizer()

							a    =			"I was born in 92000, and this is falsé."

							a    =			tokenizer.tokenize(__lowerCamelCase							)
							a    =			rust_tokenizer.tokenize(__lowerCamelCase							)
							self.assertListEqual(__lowerCamelCase					,      __lowerCamelCase							)

							a    =			tokenizer.encode(__lowerCamelCase					,      add_special_tokens=__lowerCamelCase							)
							a    =			rust_tokenizer.encode(__lowerCamelCase					,      add_special_tokens=__lowerCamelCase							)
							self.assertListEqual(__lowerCamelCase					,      __lowerCamelCase							)

							a    =			self.get_rust_tokenizer()
							a    =			tokenizer.encode(__lowerCamelCase							)
							a    =			rust_tokenizer.encode(__lowerCamelCase							)
							self.assertListEqual(__lowerCamelCase					,      __lowerCamelCase							)





				@slow
				def 							__UpperCAmelCase							(      self				:     Dict							)					->     Any:
							a    =			"Hello World!"
							a    =			[0, 3_53_78, 66_61, 38, 2]
							# xlmr = torch.hub.load('pytorch/fairseq', 'xlmr.base')  # xlmr.large has same tokenizer
							# xlmr.eval()
							# xlmr.encode(symbols)

							self.assertListEqual(__lowerCamelCase					,      self.big_tokenizer.encode(__lowerCamelCase							)							)





				@slow
				def 							__UpperCAmelCase							(      self				:     Tuple							)					->     int:
							a    =			(
							    "This is a very long text with a lot of weird characters, such as: . , ~ ? ( ) \" [ ] ! : - . Also we will"
							    " add words that should not exsist and be tokenized to <unk>, such as saoneuhaoesuth"
							)
							a    =			[
							    0,
							    32_93,
							    83,
							    10,
							    45_52,
							    49_89,
							    79_86,
							    6_78,
							    10,
							    59_15,
							    1_11,
							    17_94_59,
							    12_48_50,
							    4,
							    60_44,
							    2_37,
							    12,
							    6,
							    5,
							    6,
							    4,
							    67_80,
							    7_05,
							    15,
							    13_88,
							    44,
							    3_78,
							    1_01_14,
							    7_11,
							    1_52,
							    20,
							    6,
							    5,
							    2_23_76,
							    6_42,
							    12_21,
							    1_51_90,
							    3_41_53,
							    4_50,
							    56_08,
							    9_59,
							    11_19,
							    5_77_02,
							    1_36,
							    1_86,
							    47,
							    10_98,
							    2_93_67,
							    47,
							    # 4426, # What fairseq tokenizes from "<unk>": "_<"
							    # 3678, # What fairseq tokenizes from "<unk>": "unk"
							    # 2740, # What fairseq tokenizes from "<unk>": ">"
							    3,  # What we tokenize from "<unk>": "<unk>"
							    6,  # Residue from the tokenization: an extra sentencepiece underline
							    4,
							    60_44,
							    2_37,
							    62_84,
							    5_09_01,
							    5_28,
							    31,
							    90,
							    34,
							    9_27,
							    2,
							]
							# xlmr = torch.hub.load('pytorch/fairseq', 'xlmr.base')  # xlmr.large has same tokenizer
							# xlmr.eval()
							# xlmr.encode(symbols)

							self.assertListEqual(__lowerCamelCase					,      self.big_tokenizer.encode(__lowerCamelCase							)							)





				@slow
				def 							__UpperCAmelCase							(      self				:     Optional[Any]							)					->     List[Any]:
							# fmt: off
							a    =			{"input_ids": [[0, 1_10_62, 8_27_72, 7, 15, 8_27_72, 5_38, 5_15_29, 2_37, 1_71_98, 12_90, 2_06, 9, 21_51_75, 13_14, 1_36, 1_71_98, 12_90, 2_06, 9, 5_63_59, 42, 12_20_09, 9, 1_64_66, 16, 8_73_44, 45_37, 9, 47_17, 7_83_81, 6, 15_99_58, 7, 15, 2_44_80, 6_18, 4, 5_27, 2_26_93, 54_28, 4, 27_77, 2_44_80, 98_74, 4, 4_35_23, 5_94, 4, 8_03, 1_83_92, 3_31_89, 18, 4, 4_35_23, 2_44_47, 1_23_99, 1_00, 2_49_55, 8_36_58, 96_26, 14_40_57, 15, 8_39, 2_23_35, 16, 1_36, 2_49_55, 8_36_58, 8_34_79, 15, 3_91_02, 7_24, 16, 6_78, 6_45, 27_89, 13_28, 45_89, 42, 12_20_09, 11_57_74, 23, 8_05, 13_28, 4_68_76, 7, 1_36, 5_38_94, 19_40, 4_22_27, 4_11_59, 1_77_21, 8_23, 4_25, 4, 2_75_12, 9_87_22, 2_06, 1_36, 55_31, 49_70, 9_19, 1_73_36, 5, 2], [0, 2_00_80, 6_18, 83, 8_27_75, 47, 4_79, 9, 15_17, 73, 5_38_94, 3_33, 8_05_81, 11_01_17, 1_88_11, 52_56, 12_95, 51, 15_25_26, 2_97, 79_86, 3_90, 12_44_16, 5_38, 3_54_31, 2_14, 98, 1_50_44, 2_57_37, 1_36, 71_08, 4_37_01, 23, 7_56, 13_53_55, 7, 5, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [0, 5_81, 6_37_73, 11_94_55, 6, 14_77_97, 8_82_03, 7, 6_45, 70, 21, 32_85, 1_02_69, 5, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]], "attention_mask": [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]]}  # noqa: E501
							# fmt: on

							self.tokenizer_integration_test_util(
							    expected_encoding=__lowerCamelCase					,      model_name="xlm-roberta-base"					,      revision="d9d8a8ea5eb94b1c6654ae9249df7793cd2933d3"					,      )






