




"""simple docstring"""






import importlib
import json
import os
from collections import OrderedDict
from typing import Dict, Optional, Union

# Build the list of all image processors
from ...configuration_utils import PretrainedConfig
from ...dynamic_module_utils import get_class_from_dynamic_module, resolve_trust_remote_code
from ...image_processing_utils import ImageProcessingMixin
from ...utils import CONFIG_NAME, IMAGE_PROCESSOR_NAME, get_file_from_repo, logging
from .auto_factory import _LazyAutoMapping
from .configuration_auto import (
    CONFIG_MAPPING_NAMES,
    AutoConfig,
    model_type_to_module_name,
    replace_list_option_in_docstrings,
)


lowercase__			=		logging.get_logger(__name__)

lowercase__			=		OrderedDict(
    [
        ("""align""", """EfficientNetImageProcessor"""),
        ("""beit""", """BeitImageProcessor"""),
        ("""bit""", """BitImageProcessor"""),
        ("""blip""", """BlipImageProcessor"""),
        ("""blip-2""", """BlipImageProcessor"""),
        ("""bridgetower""", """BridgeTowerImageProcessor"""),
        ("""chinese_clip""", """ChineseCLIPImageProcessor"""),
        ("""clip""", """CLIPImageProcessor"""),
        ("""clipseg""", """ViTImageProcessor"""),
        ("""conditional_detr""", """ConditionalDetrImageProcessor"""),
        ("""convnext""", """ConvNextImageProcessor"""),
        ("""convnextv2""", """ConvNextImageProcessor"""),
        ("""cvt""", """ConvNextImageProcessor"""),
        ("""data2vec-vision""", """BeitImageProcessor"""),
        ("""deformable_detr""", """DeformableDetrImageProcessor"""),
        ("""deit""", """DeiTImageProcessor"""),
        ("""deta""", """DetaImageProcessor"""),
        ("""detr""", """DetrImageProcessor"""),
        ("""dinat""", """ViTImageProcessor"""),
        ("""donut-swin""", """DonutImageProcessor"""),
        ("""dpt""", """DPTImageProcessor"""),
        ("""efficientformer""", """EfficientFormerImageProcessor"""),
        ("""efficientnet""", """EfficientNetImageProcessor"""),
        ("""flava""", """FlavaImageProcessor"""),
        ("""focalnet""", """BitImageProcessor"""),
        ("""git""", """CLIPImageProcessor"""),
        ("""glpn""", """GLPNImageProcessor"""),
        ("""groupvit""", """CLIPImageProcessor"""),
        ("""imagegpt""", """ImageGPTImageProcessor"""),
        ("""instructblip""", """BlipImageProcessor"""),
        ("""layoutlmv2""", """LayoutLMv2ImageProcessor"""),
        ("""layoutlmv3""", """LayoutLMv3ImageProcessor"""),
        ("""levit""", """LevitImageProcessor"""),
        ("""mask2former""", """Mask2FormerImageProcessor"""),
        ("""maskformer""", """MaskFormerImageProcessor"""),
        ("""mgp-str""", """ViTImageProcessor"""),
        ("""mobilenet_v1""", """MobileNetV1ImageProcessor"""),
        ("""mobilenet_v2""", """MobileNetV2ImageProcessor"""),
        ("""mobilevit""", """MobileViTImageProcessor"""),
        ("""mobilevit""", """MobileViTImageProcessor"""),
        ("""mobilevitv2""", """MobileViTImageProcessor"""),
        ("""nat""", """ViTImageProcessor"""),
        ("""oneformer""", """OneFormerImageProcessor"""),
        ("""owlvit""", """OwlViTImageProcessor"""),
        ("""perceiver""", """PerceiverImageProcessor"""),
        ("""pix2struct""", """Pix2StructImageProcessor"""),
        ("""poolformer""", """PoolFormerImageProcessor"""),
        ("""regnet""", """ConvNextImageProcessor"""),
        ("""resnet""", """ConvNextImageProcessor"""),
        ("""sam""", """SamImageProcessor"""),
        ("""segformer""", """SegformerImageProcessor"""),
        ("""swiftformer""", """ViTImageProcessor"""),
        ("""swin""", """ViTImageProcessor"""),
        ("""swin2sr""", """Swin2SRImageProcessor"""),
        ("""swinv2""", """ViTImageProcessor"""),
        ("""table-transformer""", """DetrImageProcessor"""),
        ("""timesformer""", """VideoMAEImageProcessor"""),
        ("""tvlt""", """TvltImageProcessor"""),
        ("""upernet""", """SegformerImageProcessor"""),
        ("""van""", """ConvNextImageProcessor"""),
        ("""videomae""", """VideoMAEImageProcessor"""),
        ("""vilt""", """ViltImageProcessor"""),
        ("""vit""", """ViTImageProcessor"""),
        ("""vit_hybrid""", """ViTHybridImageProcessor"""),
        ("""vit_mae""", """ViTImageProcessor"""),
        ("""vit_msn""", """ViTImageProcessor"""),
        ("""xclip""", """CLIPImageProcessor"""),
        ("""yolos""", """YolosImageProcessor"""),
    ]
)

lowercase__			=		_LazyAutoMapping(CONFIG_MAPPING_NAMES, IMAGE_PROCESSOR_MAPPING_NAMES)







def 	_snake_case     (							lowercase__						):
				for module_name, extractors in IMAGE_PROCESSOR_MAPPING_NAMES.items():
								if class_name in extractors:
												_lowerCamelCase	:     Optional[int]       =       model_type_to_module_name(lowercase__						)

												_lowerCamelCase	:     Any       =       importlib.import_module(f'''.{module_name}'''				,       'transformers.models'						)
												try:
																return getattr(lowercase__				,       lowercase__						)
												except AttributeError:
																continue

				for _, extractor in IMAGE_PROCESSOR_MAPPING._extra_content.items():
								if getattr(lowercase__				,       '__name__'				,       lowercase__						) == class_name:
												return extractor

    # We did not fine the class, but maybe it's because a dep is missing. In that case, the class will be in the main
    # init and we return the proper dummy to get an appropriate error message.
				_lowerCamelCase	:     Tuple       =       importlib.import_module('transformers'						)
				if hasattr(lowercase__				,       lowercase__						):
								return getattr(lowercase__				,       lowercase__						)

				return None







def 	_snake_case     (							lowercase__				,       lowercase__ = None				,       lowercase__ = False				,       lowercase__ = False				,       lowercase__ = None				,       lowercase__ = None				,       lowercase__ = None				,       lowercase__ = False				,       **lowercase__				,       ):
				_lowerCamelCase	:     Optional[int]       =       get_file_from_repo(
				    lowercase__				,       lowercase__				,       cache_dir=lowercase__				,       force_download=lowercase__				,       resume_download=lowercase__				,       proxies=lowercase__				,       use_auth_token=lowercase__				,       revision=lowercase__				,       local_files_only=lowercase__				,       )
				if resolved_config_file is None:
								logger.info(
								    'Could not locate the image processor configuration file, will try to use the model config instead.'						)
								return {}

				with open(lowercase__				,       encoding='utf-8'						) as reader:
								return json.load(lowercase__						)






class lowerCAmelCase__      :






			'''simple docstring'''



			def __init__(    self    ):
							raise EnvironmentError(
							    'AutoImageProcessor is designed to be instantiated '
							    'using the `AutoImageProcessor.from_pretrained(pretrained_model_name_or_path)` method.'    )
			@classmethod
			@replace_list_option_in_docstrings(lowercase    )
			def    A_       (    cls    , lowercase    , **lowercase    ):
							_lowerCamelCase	:     int       =       kwargs.pop('config'    , lowercase    )
							_lowerCamelCase	:     Optional[int]       =       kwargs.pop('trust_remote_code'    , lowercase    )
							_lowerCamelCase	:     Optional[int]       =       True

							_lowerCamelCase,					_lowerCamelCase	:     Any       =       ImageProcessingMixin.get_image_processor_dict(lowercase    , **lowercase    )
							_lowerCamelCase	:     Tuple       =       config_dict.get('image_processor_type'    , lowercase    )
							_lowerCamelCase	:     str       =       None
							if "AutoImageProcessor" in config_dict.get('auto_map'    , {}    ):
											_lowerCamelCase	:     Any       =       config_dict['auto_map']['AutoImageProcessor']

							# If we still don't have the image processor class, check if we're loading from a previous feature extractor config
							# and if so, infer the image processor class from there.
							if image_processor_class is None and image_processor_auto_map is None:
											_lowerCamelCase	:     Optional[int]       =       config_dict.pop('feature_extractor_type'    , lowercase    )
											if feature_extractor_class is not None:
															logger.warning(
															    'Could not find image processor class in the image processor config or the model config. Loading'
															    ' based on pattern matching with the model\'s feature extractor configuration.'    )
															_lowerCamelCase	:     Tuple       =       feature_extractor_class.replace('FeatureExtractor'    , 'ImageProcessor'    )
											if "AutoFeatureExtractor" in config_dict.get('auto_map'    , {}    ):
															_lowerCamelCase	:     Dict       =       config_dict['auto_map']['AutoFeatureExtractor']
															_lowerCamelCase	:     str       =       feature_extractor_auto_map.replace('FeatureExtractor'    , 'ImageProcessor'    )
															logger.warning(
															    'Could not find image processor auto map in the image processor config or the model config.'
															    ' Loading based on pattern matching with the model\'s feature extractor configuration.'    )

        # If we don't find the image processor class in the image processor config, let's try the model config.
							if image_processor_class is None and image_processor_auto_map is None:
											if not isinstance(lowercase    , lowercase    ):
															_lowerCamelCase	:     Union[str, Any]       =       AutoConfig.from_pretrained(lowercase    , **lowercase    )
											# It could be in `config.image_processor_type``
											_lowerCamelCase	:     List[Any]       =       getattr(lowercase    , 'image_processor_type'    , lowercase    )
											if hasattr(lowercase    , 'auto_map'    ) and "AutoImageProcessor" in config.auto_map:
															_lowerCamelCase	:     List[str]       =       config.auto_map['AutoImageProcessor']

							if image_processor_class is not None:
											_lowerCamelCase	:     Union[str, Any]       =       image_processor_class_from_name(lowercase    )

							_lowerCamelCase	:     Union[str, Any]       =       image_processor_auto_map is not None
							_lowerCamelCase	:     Optional[Any]       =       image_processor_class is not None or type(lowercase    ) in IMAGE_PROCESSOR_MAPPING
							_lowerCamelCase	:     Union[str, Any]       =       resolve_trust_remote_code(
							    lowercase    , lowercase    , lowercase    , lowercase    )

							if has_remote_code and trust_remote_code:
											_lowerCamelCase	:     Optional[Any]       =       get_class_from_dynamic_module(
											    lowercase    , lowercase    , **lowercase    )
											_lowerCamelCase	:     int       =       kwargs.pop('code_revision'    , lowercase    )
											if os.path.isdir(lowercase    ):
															image_processor_class.register_for_auto_class()
											return image_processor_class.from_dict(lowercase    , **lowercase    )
							elif image_processor_class is not None:
											return image_processor_class.from_dict(lowercase    , **lowercase    )
							# Last try: we use the IMAGE_PROCESSOR_MAPPING.
							elif type(lowercase    ) in IMAGE_PROCESSOR_MAPPING:
											_lowerCamelCase	:     Union[str, Any]       =       IMAGE_PROCESSOR_MAPPING[type(lowercase    )]
											return image_processor_class.from_dict(lowercase    , **lowercase    )

							raise ValueError(
							    F'''Unrecognized image processor in {pretrained_model_name_or_path}. Should have a '''
							    F'''`image_processor_type` key in its {IMAGE_PROCESSOR_NAME} of {CONFIG_NAME}, or one of the following '''
							    F'''`model_type` keys in its {CONFIG_NAME}: {', '.join(c for c in IMAGE_PROCESSOR_MAPPING_NAMES.keys()    )}'''    )






			@staticmethod
			def    A_       (    lowercase    , lowercase    ):
							IMAGE_PROCESSOR_MAPPING.register(lowercase    , lowercase    )





"""simple docstring"""






import itertools
import json
import os
import unittest

from transformers import AddedToken, RobertaTokenizer, RobertaTokenizerFast
from transformers.models.roberta.tokenization_roberta import VOCAB_FILES_NAMES
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin






@require_tokenizers
class lowerCAmelCase__      (	lowercase,      unittest.TestCase		):






			'''simple docstring'''



			lowerCamelCase__ =     RobertaTokenizer
			lowerCamelCase__ =     RobertaTokenizerFast
			lowerCamelCase__ =     True
			lowerCamelCase__ =     {"""cls_token""": """<s>"""}
			def    A_       (    self    ):
							super().setUp()

							# Adapted from Sennrich et al. 2015 and https://github.com/rsennrich/subword-nmt
							_lowerCamelCase	:     Optional[Any]       =       [
							    'l',
							    'o',
							    'w',
							    'e',
							    'r',
							    's',
							    't',
							    'i',
							    'd',
							    'n',
							    '\u0120',
							    '\u0120l',
							    '\u0120n',
							    '\u0120lo',
							    '\u0120low',
							    'er',
							    '\u0120lowest',
							    '\u0120newer',
							    '\u0120wider',
							    '<unk>',
							]
							_lowerCamelCase	:     Optional[int]       =       dict(zip(lowercase    , range(len(lowercase    )    )    )    )
							_lowerCamelCase	:     Optional[int]       =       ['#version: 0.2', '\u0120 l', '\u0120l o', '\u0120lo w', 'e r', '']
							_lowerCamelCase	:     Optional[Any]       =       {'unk_token': '<unk>'}

							_lowerCamelCase	:     Optional[int]       =       os.path.join(self.tmpdirname    , VOCAB_FILES_NAMES['vocab_file']    )
							_lowerCamelCase	:     Optional[Any]       =       os.path.join(self.tmpdirname    , VOCAB_FILES_NAMES['merges_file']    )
							with open(self.vocab_file    , 'w'    , encoding='utf-8'    ) as fp:
											fp.write(json.dumps(lowercase    ) + '\n'    )
							with open(self.merges_file    , 'w'    , encoding='utf-8'    ) as fp:
											fp.write('\n'.join(lowercase    )    )
			def    A_       (    self    , **lowercase    ):
							kwargs.update(self.special_tokens_map    )
							return self.tokenizer_class.from_pretrained(self.tmpdirname    , **lowercase    )
			def    A_       (    self    , **lowercase    ):
							kwargs.update(self.special_tokens_map    )
							return RobertaTokenizerFast.from_pretrained(self.tmpdirname    , **lowercase    )
			def    A_       (    self    , lowercase    ):
							_lowerCamelCase	:     Any       =       'lower newer'
							_lowerCamelCase	:     Union[str, Any]       =       'lower newer'
							return input_text, output_text
			def    A_       (    self    ):
							_lowerCamelCase	:     Dict       =       self.tokenizer_class(self.vocab_file    , self.merges_file    , **self.special_tokens_map    )
							_lowerCamelCase	:     List[str]       =       'lower newer'
							_lowerCamelCase	:     Tuple       =       ['l', 'o', 'w', 'er', '\u0120', 'n', 'e', 'w', 'er']
							_lowerCamelCase	:     List[str]       =       tokenizer.tokenize(lowercase    )  # , add_prefix_space=True)
							self.assertListEqual(lowercase    , lowercase    )

							_lowerCamelCase	:     List[Any]       =       tokens + [tokenizer.unk_token]
							_lowerCamelCase	:     str       =       [0, 1, 2, 15, 10, 9, 3, 2, 15, 19]
							self.assertListEqual(tokenizer.convert_tokens_to_ids(lowercase    )    , lowercase    )
			def    A_       (    self    ):
							_lowerCamelCase	:     Union[str, Any]       =       self.get_tokenizer()

							self.assertListEqual(tokenizer.encode('Hello world!'    , add_special_tokens=lowercase    )    , [0, 31414, 232, 328, 2]    )
							self.assertListEqual(
							    tokenizer.encode('Hello world! cécé herlolip 418'    , add_special_tokens=lowercase    )    , [0, 31414, 232, 328, 740, 1140, 12695, 69, 46078, 1588, 2]    , )
			@slow
			def    A_       (    self    ):
							_lowerCamelCase	:     Dict       =       self.tokenizer_class.from_pretrained('roberta-base'    )

							_lowerCamelCase	:     Tuple       =       tokenizer.encode('sequence builders'    , add_special_tokens=lowercase    )
							_lowerCamelCase	:     int       =       tokenizer.encode('multi-sequence build'    , add_special_tokens=lowercase    )

							_lowerCamelCase	:     List[Any]       =       tokenizer.encode(
							    'sequence builders'    , add_special_tokens=lowercase    , add_prefix_space=lowercase    )
							_lowerCamelCase	:     List[str]       =       tokenizer.encode(
							    'sequence builders'    , 'multi-sequence build'    , add_special_tokens=lowercase    , add_prefix_space=lowercase    )

							_lowerCamelCase	:     Dict       =       tokenizer.build_inputs_with_special_tokens(lowercase    )
							_lowerCamelCase	:     Optional[int]       =       tokenizer.build_inputs_with_special_tokens(lowercase    , lowercase    )

							assert encoded_sentence == encoded_text_from_decode
							assert encoded_pair == encoded_pair_from_decode
			def    A_       (    self    ):
							_lowerCamelCase	:     Tuple       =       self.get_tokenizer()

							_lowerCamelCase	:     str       =       'Encode this sequence.'
							_lowerCamelCase	:     Optional[int]       =       tokenizer.byte_encoder[' '.encode('utf-8'    )[0]]

							# Testing encoder arguments
							_lowerCamelCase	:     Optional[int]       =       tokenizer.encode(lowercase    , add_special_tokens=lowercase    , add_prefix_space=lowercase    )
							_lowerCamelCase	:     Union[str, Any]       =       tokenizer.convert_ids_to_tokens(encoded[0]    )[0]
							self.assertNotEqual(lowercase    , lowercase    )

							_lowerCamelCase	:     List[str]       =       tokenizer.encode(lowercase    , add_special_tokens=lowercase    , add_prefix_space=lowercase    )
							_lowerCamelCase	:     List[Any]       =       tokenizer.convert_ids_to_tokens(encoded[0]    )[0]
							self.assertEqual(lowercase    , lowercase    )

							tokenizer.add_special_tokens({'bos_token': '<s>'}    )
							_lowerCamelCase	:     Optional[Any]       =       tokenizer.encode(lowercase    , add_special_tokens=lowercase    )
							_lowerCamelCase	:     Union[str, Any]       =       tokenizer.convert_ids_to_tokens(encoded[1]    )[0]
							self.assertNotEqual(lowercase    , lowercase    )

							# Testing spaces after special tokens
							_lowerCamelCase	:     List[Any]       =       '<mask>'
							tokenizer.add_special_tokens(
							    {'mask_token': AddedToken(lowercase    , lstrip=lowercase    , rstrip=lowercase    )}    )  # mask token has a left space
							_lowerCamelCase	:     List[str]       =       tokenizer.convert_tokens_to_ids(lowercase    )

							_lowerCamelCase	:     Optional[int]       =       'Encode <mask> sequence'
							_lowerCamelCase	:     int       =       'Encode <mask>sequence'

							_lowerCamelCase	:     str       =       tokenizer.encode(lowercase    )
							_lowerCamelCase	:     Tuple       =       encoded.index(lowercase    )
							_lowerCamelCase	:     int       =       tokenizer.convert_ids_to_tokens(encoded[mask_loc + 1]    )[0]
							self.assertEqual(lowercase    , lowercase    )

							_lowerCamelCase	:     Any       =       tokenizer.encode(lowercase    )
							_lowerCamelCase	:     Union[str, Any]       =       encoded.index(lowercase    )
							_lowerCamelCase	:     int       =       tokenizer.convert_ids_to_tokens(encoded[mask_loc + 1]    )[0]
							self.assertNotEqual(lowercase    , lowercase    )
			def    A_       (    self    ):
							pass
			def    A_       (    self    ):
							for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
											with self.subTest(F'''{tokenizer.__class__.__name__} ({pretrained_name})'''    ):
															_lowerCamelCase	:     Optional[Any]       =       self.rust_tokenizer_class.from_pretrained(lowercase    , **lowercase    )
															_lowerCamelCase	:     Dict       =       self.tokenizer_class.from_pretrained(lowercase    , **lowercase    )
															_lowerCamelCase	:     List[str]       =       'A, <mask> AllenNLP sentence.'
															_lowerCamelCase	:     List[Any]       =       tokenizer_r.encode_plus(lowercase    , add_special_tokens=lowercase    , return_token_type_ids=lowercase    )
															_lowerCamelCase	:     Union[str, Any]       =       tokenizer_p.encode_plus(lowercase    , add_special_tokens=lowercase    , return_token_type_ids=lowercase    )

															# token_type_ids should put 0 everywhere
															self.assertEqual(sum(tokens_r['token_type_ids']    )    , sum(tokens_p['token_type_ids']    )    )

															# attention_mask should put 1 everywhere, so sum over length should be 1
															self.assertEqual(
															    sum(tokens_r['attention_mask']    ) / len(tokens_r['attention_mask']    )    , sum(tokens_p['attention_mask']    ) / len(tokens_p['attention_mask']    )    , )

															_lowerCamelCase	:     List[Any]       =       tokenizer_r.convert_ids_to_tokens(tokens_r['input_ids']    )
															_lowerCamelCase	:     Tuple       =       tokenizer_p.convert_ids_to_tokens(tokens_p['input_ids']    )

															# Rust correctly handles the space before the mask while python doesnt
															self.assertSequenceEqual(tokens_p['input_ids']    , [0, 250, 6, 50264, 3823, 487, 21992, 3645, 4, 2]    )
															self.assertSequenceEqual(tokens_r['input_ids']    , [0, 250, 6, 50264, 3823, 487, 21992, 3645, 4, 2]    )

															self.assertSequenceEqual(
															    lowercase    , ['<s>', 'A', ',', '<mask>', 'ĠAllen', 'N', 'LP', 'Ġsentence', '.', '</s>']    )
															self.assertSequenceEqual(
															    lowercase    , ['<s>', 'A', ',', '<mask>', 'ĠAllen', 'N', 'LP', 'Ġsentence', '.', '</s>']    )
			def    A_       (    self    ):
							for trim_offsets, add_prefix_space in itertools.product([True, False]    , repeat=2    ):
											_lowerCamelCase	:     Union[str, Any]       =       self.rust_tokenizer_class.from_pretrained(
											    self.tmpdirname    , use_fast=lowercase    , add_prefix_space=lowercase    , trim_offsets=lowercase    )

											_lowerCamelCase	:     Optional[int]       =       json.loads(tokenizer_r.backend_tokenizer.pre_tokenizer.__getstate__()    )
											_lowerCamelCase	:     Dict       =       json.loads(tokenizer_r.backend_tokenizer.post_processor.__getstate__()    )

											self.assertEqual(pre_tokenizer_state['add_prefix_space']    , lowercase    )

											self.assertEqual(post_processor_state['add_prefix_space']    , lowercase    )
											self.assertEqual(post_processor_state['trim_offsets']    , lowercase    )






			def    A_       (    self    ):
							# Test which aims to verify that the offsets are well adapted to the argument `add_prefix_space` and
							# `trim_offsets`
							for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
											with self.subTest(F'''{tokenizer.__class__.__name__} ({pretrained_name})'''    ):
															_lowerCamelCase	:     Dict       =       'hello'  # `hello` is a token in the vocabulary of `pretrained_name`
															_lowerCamelCase	:     Tuple       =       F'''{text_of_1_token} {text_of_1_token}'''

															_lowerCamelCase	:     Dict       =       self.rust_tokenizer_class.from_pretrained(
															    lowercase    , use_fast=lowercase    , add_prefix_space=lowercase    , trim_offsets=lowercase    )
															_lowerCamelCase	:     List[Any]       =       tokenizer_r(lowercase    , return_offsets_mapping=lowercase    , add_special_tokens=lowercase    )
															self.assertEqual(encoding.offset_mapping[0]    , (0, len(lowercase    ))    )
															self.assertEqual(
															    encoding.offset_mapping[1]    , (len(lowercase    ) + 1, len(lowercase    ) + 1 + len(lowercase    ))    , )

															_lowerCamelCase	:     int       =       self.rust_tokenizer_class.from_pretrained(
															    lowercase    , use_fast=lowercase    , add_prefix_space=lowercase    , trim_offsets=lowercase    )
															_lowerCamelCase	:     Union[str, Any]       =       tokenizer_r(lowercase    , return_offsets_mapping=lowercase    , add_special_tokens=lowercase    )
															self.assertEqual(encoding.offset_mapping[0]    , (0, len(lowercase    ))    )
															self.assertEqual(
															    encoding.offset_mapping[1]    , (len(lowercase    ) + 1, len(lowercase    ) + 1 + len(lowercase    ))    , )

															_lowerCamelCase	:     Union[str, Any]       =       self.rust_tokenizer_class.from_pretrained(
															    lowercase    , use_fast=lowercase    , add_prefix_space=lowercase    , trim_offsets=lowercase    )
															_lowerCamelCase	:     List[str]       =       tokenizer_r(lowercase    , return_offsets_mapping=lowercase    , add_special_tokens=lowercase    )
															self.assertEqual(encoding.offset_mapping[0]    , (0, len(lowercase    ))    )
															self.assertEqual(
															    encoding.offset_mapping[1]    , (len(lowercase    ), len(lowercase    ) + 1 + len(lowercase    ))    , )

															_lowerCamelCase	:     Dict       =       self.rust_tokenizer_class.from_pretrained(
															    lowercase    , use_fast=lowercase    , add_prefix_space=lowercase    , trim_offsets=lowercase    )
															_lowerCamelCase	:     int       =       tokenizer_r(lowercase    , return_offsets_mapping=lowercase    , add_special_tokens=lowercase    )
															self.assertEqual(encoding.offset_mapping[0]    , (0, len(lowercase    ))    )
															self.assertEqual(
															    encoding.offset_mapping[1]    , (len(lowercase    ), len(lowercase    ) + 1 + len(lowercase    ))    , )

															_lowerCamelCase	:     int       =       F''' {text}'''

															# tokenizer_r = self.rust_tokenizer_class.from_pretrained(
															#     pretrained_name, use_fast=True, add_prefix_space=True, trim_offsets=True
															# )
															# encoding = tokenizer_r(text, return_offsets_mapping=True, add_special_tokens=False)
															# self.assertEqual(encoding.offset_mapping[0], (1, 1 + len(text_of_1_token)))
															# self.assertEqual(
															#     encoding.offset_mapping[1],
															#     (1 + len(text_of_1_token) + 1, 1 + len(text_of_1_token) + 1 + len(text_of_1_token)),
															# )

															_lowerCamelCase	:     Optional[Any]       =       self.rust_tokenizer_class.from_pretrained(
															    lowercase    , use_fast=lowercase    , add_prefix_space=lowercase    , trim_offsets=lowercase    )
															_lowerCamelCase	:     Optional[int]       =       tokenizer_r(lowercase    , return_offsets_mapping=lowercase    , add_special_tokens=lowercase    )
															self.assertEqual(encoding.offset_mapping[0]    , (1, 1 + len(lowercase    ))    )
															self.assertEqual(
															    encoding.offset_mapping[1]    , (1 + len(lowercase    ) + 1, 1 + len(lowercase    ) + 1 + len(lowercase    ))    , )

															_lowerCamelCase	:     Optional[int]       =       self.rust_tokenizer_class.from_pretrained(
															    lowercase    , use_fast=lowercase    , add_prefix_space=lowercase    , trim_offsets=lowercase    )
															_lowerCamelCase	:     Union[str, Any]       =       tokenizer_r(lowercase    , return_offsets_mapping=lowercase    , add_special_tokens=lowercase    )
															self.assertEqual(encoding.offset_mapping[0]    , (0, 1 + len(lowercase    ))    )
															self.assertEqual(
															    encoding.offset_mapping[1]    , (1 + len(lowercase    ), 1 + len(lowercase    ) + 1 + len(lowercase    ))    , )

															_lowerCamelCase	:     Union[str, Any]       =       self.rust_tokenizer_class.from_pretrained(
															    lowercase    , use_fast=lowercase    , add_prefix_space=lowercase    , trim_offsets=lowercase    )
															_lowerCamelCase	:     Optional[Any]       =       tokenizer_r(lowercase    , return_offsets_mapping=lowercase    , add_special_tokens=lowercase    )
															self.assertEqual(encoding.offset_mapping[0]    , (0, 1 + len(lowercase    ))    )
															self.assertEqual(
															    encoding.offset_mapping[1]    , (1 + len(lowercase    ), 1 + len(lowercase    ) + 1 + len(lowercase    ))    , )