



'''simple docstring'''




from typing import Dict, List, Optional, Union

import numpy as np

from transformers.utils import is_vision_available
from transformers.utils.generic import TensorType

from ...image_processing_utils import BaseImageProcessor, BatchFeature, get_size_dict
from ...image_transforms import (
    center_crop,
    get_resize_output_image_size,
    normalize,
    rescale,
    resize,
    to_channel_dimension_format,
)
from ...image_utils import (
    IMAGENET_STANDARD_MEAN,
    IMAGENET_STANDARD_STD,
    ChannelDimension,
    ImageInput,
    PILImageResampling,
    is_valid_image,
    to_numpy_array,
    valid_images,
)
from ...utils import logging


if is_vision_available():
							import PIL

a   :			str					  =    logging.get_logger(__name__)



def       __lowerCamelCase  (					_lowercase		)      ->			List[List[ImageInput]]:
							if isinstance(_lowercase	,				(list, tuple)		) and isinstance(videos[0]	,				(list, tuple)		) and is_valid_image(videos[0][0]		):
														return videos

							elif isinstance(_lowercase	,				(list, tuple)		) and is_valid_image(videos[0]		):
														return [videos]

							elif is_valid_image(_lowercase		):
														return [[videos]]

							raise ValueError(F'''Could not make batched video from {videos}'''		)


class 			UpperCamelCase_ (							__magic_name__			):
	lowercase          = ['pixel_values']





	def __init__(  self		,		A = True		,		A = None		,		A = PILImageResampling.BILINEAR		,		A = True		,		A = None		,		A = True		,		A = 1 / 255		,		A = True		,		A = True		,		A = None		,		A = None		,		**A		,		)			->			None:
								super().__init__(**A    )
								UpperCAmelCase     :		Union[str, Any]							     =		size if size is not None else {"""shortest_edge""": 256}
								UpperCAmelCase     :		Union[str, Any]							     =		get_size_dict(A		,		default_to_square=A    )
								UpperCAmelCase     :		Union[str, Any]							     =		crop_size if crop_size is not None else {"""height""": 224, """width""": 224}
								UpperCAmelCase     :		List[Any]							     =		get_size_dict(A		,		param_name="""crop_size"""    )

								UpperCAmelCase     :		Optional[int]							     =		do_resize
								UpperCAmelCase     :		Optional[int]							     =		size
								UpperCAmelCase     :		List[Any]							     =		do_center_crop
								UpperCAmelCase     :		Tuple							     =		crop_size
								UpperCAmelCase     :		Any							     =		resample
								UpperCAmelCase     :		Union[str, Any]							     =		do_rescale
								UpperCAmelCase     :		Optional[Any]							     =		rescale_factor
								UpperCAmelCase     :		Dict							     =		offset
								UpperCAmelCase     :		Optional[Any]							     =		do_normalize
								UpperCAmelCase     :		List[str]							     =		image_mean if image_mean is not None else IMAGENET_STANDARD_MEAN
								UpperCAmelCase     :		Dict							     =		image_std if image_std is not None else IMAGENET_STANDARD_STD





	def  _lowercase(  self		,		A		,		A		,		A = PILImageResampling.BILINEAR		,		A = None		,		**A		,		)			->			np.ndarray:
								UpperCAmelCase     :		Any							     =		get_size_dict(A		,		default_to_square=A    )
								if "shortest_edge" in size:
															UpperCAmelCase     :		Union[str, Any]							     =		get_resize_output_image_size(A		,		size["""shortest_edge"""]		,		default_to_square=A    )
								elif "height" in size and "width" in size:
															UpperCAmelCase     :		List[Any]							     =		(size["""height"""], size["""width"""])
								else:
															raise ValueError(f'''Size must have \'height\' and \'width\' or \'shortest_edge\' as keys. Got {size.keys()}'''    )
								return resize(A		,		size=A		,		resample=A		,		data_format=A		,		**A    )





	def  _lowercase(  self		,		A		,		A		,		A = None		,		**A		,		)			->			np.ndarray:
								UpperCAmelCase     :		str							     =		get_size_dict(A    )
								if "height" not in size or "width" not in size:
															raise ValueError(f'''Size must have \'height\' and \'width\' as keys. Got {size.keys()}'''    )
								return center_crop(A		,		size=(size["""height"""], size["""width"""])		,		data_format=A		,		**A    )





	def  _lowercase(  self		,		A		,		A		,		A = True		,		A = None		,		**A		,		)			->			str:
								UpperCAmelCase     :		Any							     =		image.astype(np.floataa    )
								if offset:
															UpperCAmelCase     :		Optional[Any]							     =		image - (scale / 2)
								return rescale(A		,		scale=A		,		data_format=A		,		**A    )





	def  _lowercase(  self		,		A		,		A		,		A		,		A = None		,		**A		,		)			->			np.ndarray:
								return normalize(A		,		mean=A		,		std=A		,		data_format=A		,		**A    )





	def  _lowercase(  self		,		A		,		A = None		,		A = None		,		A = None		,		A = None		,		A = None		,		A = None		,		A = None		,		A = None		,		A = None		,		A = None		,		A = None		,		A = ChannelDimension.FIRST		,		)			->			np.ndarray:
								if do_resize and size is None or resample is None:
															raise ValueError("""Size and resample must be specified if do_resize is True."""    )

								if do_center_crop and crop_size is None:
															raise ValueError("""Crop size must be specified if do_center_crop is True."""    )

								if do_rescale and rescale_factor is None:
															raise ValueError("""Rescale factor must be specified if do_rescale is True."""    )

								if do_normalize and (image_mean is None or image_std is None):
															raise ValueError("""Image mean and std must be specified if do_normalize is True."""    )

								if offset and not do_rescale:
															raise ValueError("""For offset, do_rescale must also be set to True."""    )

								# All transformations expect numpy arrays.
								UpperCAmelCase     :		Tuple							     =		to_numpy_array(A    )

								if do_resize:
															UpperCAmelCase     :		str							     =		self.resize(image=A		,		size=A		,		resample=A    )

								if do_center_crop:
															UpperCAmelCase     :		Any							     =		self.center_crop(A		,		size=A    )

								if do_rescale:
															UpperCAmelCase     :		Any							     =		self.rescale(image=A		,		scale=A		,		offset=A    )

								if do_normalize:
															UpperCAmelCase     :		Union[str, Any]							     =		self.normalize(image=A		,		mean=A		,		std=A    )

								UpperCAmelCase     :		List[str]							     =		to_channel_dimension_format(A		,		A    )
								return image






	def  _lowercase(  self		,		A		,		A = None		,		A = None		,		A = None		,		A = None		,		A = None		,		A = None		,		A = None		,		A = None		,		A = None		,		A = None		,		A = None		,		A = None		,		A = ChannelDimension.FIRST		,		**A		,		)			->			PIL.Image.Image:
								UpperCAmelCase     :		int							     =		do_resize if do_resize is not None else self.do_resize
								UpperCAmelCase     :		int							     =		resample if resample is not None else self.resample
								UpperCAmelCase     :		str							     =		do_center_crop if do_center_crop is not None else self.do_center_crop
								UpperCAmelCase     :		str							     =		do_rescale if do_rescale is not None else self.do_rescale
								UpperCAmelCase     :		str							     =		rescale_factor if rescale_factor is not None else self.rescale_factor
								UpperCAmelCase     :		Optional[int]							     =		offset if offset is not None else self.offset
								UpperCAmelCase     :		List[str]							     =		do_normalize if do_normalize is not None else self.do_normalize
								UpperCAmelCase     :		Optional[Any]							     =		image_mean if image_mean is not None else self.image_mean
								UpperCAmelCase     :		Tuple							     =		image_std if image_std is not None else self.image_std

								UpperCAmelCase     :		str							     =		size if size is not None else self.size
								UpperCAmelCase     :		Tuple							     =		get_size_dict(A		,		default_to_square=A    )
								UpperCAmelCase     :		Any							     =		crop_size if crop_size is not None else self.crop_size
								UpperCAmelCase     :		Tuple							     =		get_size_dict(A		,		param_name="""crop_size"""    )

								if not valid_images(A    ):
															raise ValueError(
															    """Invalid image type. Must be of type PIL.Image.Image, numpy.ndarray, """
															    """torch.Tensor, tf.Tensor or jax.ndarray."""    )

								UpperCAmelCase     :		List[Any]							     =		make_batched(A    )

								UpperCAmelCase     :		Tuple							     =		[
								    [
								        self._preprocess_image(
								            image=A		,		do_resize=A		,		size=A		,		resample=A		,		do_center_crop=A		,		crop_size=A		,		do_rescale=A		,		rescale_factor=A		,		offset=A		,		do_normalize=A		,		image_mean=A		,		image_std=A		,		data_format=A		,		)
								        for img in video
								    ]
								    for video in videos
								]

								UpperCAmelCase     :		str							     =		{"""pixel_values""": videos}
								return BatchFeature(data=A		,		tensor_type=A    )






'''simple docstring'''




import itertools
import json
import os
import unittest

from transformers import AddedToken, LongformerTokenizer, LongformerTokenizerFast
from transformers.models.longformer.tokenization_longformer import VOCAB_FILES_NAMES
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin

@require_tokenizers
class 			UpperCamelCase_ (							__magic_name__      ,     unittest.TestCase			):
	lowercase          = LongformerTokenizer
	lowercase          = True
	lowercase          = LongformerTokenizerFast
	lowercase          = True





	def  _lowercase(  self    )			->			List[Any]:
								super().setUp()

								# Adapted from Sennrich et al. 2015 and https://github.com/rsennrich/subword-nmt
								UpperCAmelCase     :		List[str]							     =		[
								    """l""",
								    """o""",
								    """w""",
								    """e""",
								    """r""",
								    """s""",
								    """t""",
								    """i""",
								    """d""",
								    """n""",
								    """\u0120""",
								    """\u0120l""",
								    """\u0120n""",
								    """\u0120lo""",
								    """\u0120low""",
								    """er""",
								    """\u0120lowest""",
								    """\u0120newer""",
								    """\u0120wider""",
								    """<unk>""",
								]
								UpperCAmelCase     :		int							     =		dict(zip(A		,		range(len(A    )    )    )    )
								UpperCAmelCase     :		Any							     =		["""#version: 0.2""", """\u0120 l""", """\u0120l o""", """\u0120lo w""", """e r""", """"""]
								UpperCAmelCase     :		Dict							     =		{"""unk_token""": """<unk>"""}

								UpperCAmelCase     :		int							     =		os.path.join(self.tmpdirname		,		VOCAB_FILES_NAMES["""vocab_file"""]    )
								UpperCAmelCase     :		Optional[int]							     =		os.path.join(self.tmpdirname		,		VOCAB_FILES_NAMES["""merges_file"""]    )
								with open(self.vocab_file		,		"""w"""		,		encoding="""utf-8"""    ) as fp:
															fp.write(json.dumps(A    ) + """\n"""    )
								with open(self.merges_file		,		"""w"""		,		encoding="""utf-8"""    ) as fp:
															fp.write("""\n""".join(A    )    )





	def  _lowercase(  self		,		**A    )			->			Optional[Any]:
								kwargs.update(self.special_tokens_map    )
								return self.tokenizer_class.from_pretrained(self.tmpdirname		,		**A    )





	def  _lowercase(  self		,		**A    )			->			int:
								kwargs.update(self.special_tokens_map    )
								return self.rust_tokenizer_class.from_pretrained(self.tmpdirname		,		**A    )





	def  _lowercase(  self		,		A    )			->			Optional[int]:
								UpperCAmelCase     :		Optional[Any]							     =		"""lower newer"""
								UpperCAmelCase     :		Optional[int]							     =		"""lower newer"""
								return input_text, output_text





	def  _lowercase(  self    )			->			Optional[Any]:
								UpperCAmelCase     :		Tuple							     =		self.tokenizer_class(self.vocab_file		,		self.merges_file		,		**self.special_tokens_map    )
								UpperCAmelCase     :		Dict							     =		"""lower newer"""
								UpperCAmelCase     :		int							     =		["""l""", """o""", """w""", """er""", """\u0120""", """n""", """e""", """w""", """er"""]
								UpperCAmelCase     :		Tuple							     =		tokenizer.tokenize(A    )  # , add_prefix_space=True)
								self.assertListEqual(A		,		A    )

								UpperCAmelCase     :		Any							     =		tokens + [tokenizer.unk_token]
								UpperCAmelCase     :		Tuple							     =		[0, 1, 2, 15, 10, 9, 3, 2, 15, 19]
								self.assertListEqual(tokenizer.convert_tokens_to_ids(A    )		,		A    )





	def  _lowercase(  self    )			->			Union[str, Any]:
								UpperCAmelCase     :		str							     =		self.get_tokenizer()

								self.assertListEqual(tokenizer.encode("""Hello world!"""		,		add_special_tokens=A    )		,		[0, 31414, 232, 328, 2]    )
								self.assertListEqual(
								    tokenizer.encode("""Hello world! cécé herlolip 418"""		,		add_special_tokens=A    )		,		[0, 31414, 232, 328, 740, 1140, 12695, 69, 46078, 1588, 2]		,		)





	@slow
	def  _lowercase(  self    )			->			Optional[int]:
								UpperCAmelCase     :		Any							     =		self.tokenizer_class.from_pretrained("""allenai/longformer-base-4096"""    )

								UpperCAmelCase     :		List[Any]							     =		tokenizer.encode("""sequence builders"""		,		add_special_tokens=A    )
								UpperCAmelCase     :		Optional[Any]							     =		tokenizer.encode("""multi-sequence build"""		,		add_special_tokens=A    )

								UpperCAmelCase     :		List[str]							     =		tokenizer.encode(
								    """sequence builders"""		,		add_special_tokens=A		,		add_prefix_space=A    )
								UpperCAmelCase     :		List[str]							     =		tokenizer.encode(
								    """sequence builders"""		,		"""multi-sequence build"""		,		add_special_tokens=A		,		add_prefix_space=A    )

								UpperCAmelCase     :		Union[str, Any]							     =		tokenizer.build_inputs_with_special_tokens(A    )
								UpperCAmelCase     :		Any							     =		tokenizer.build_inputs_with_special_tokens(A		,		A    )

								assert encoded_sentence == encoded_text_from_decode
								assert encoded_pair == encoded_pair_from_decode





	def  _lowercase(  self    )			->			List[Any]:
								UpperCAmelCase     :		str							     =		self.get_tokenizer()

								UpperCAmelCase     :		List[Any]							     =		"""Encode this sequence."""
								UpperCAmelCase     :		List[str]							     =		tokenizer.byte_encoder[""" """.encode("""utf-8"""    )[0]]

								# Testing encoder arguments
								UpperCAmelCase     :		Union[str, Any]							     =		tokenizer.encode(A		,		add_special_tokens=A		,		add_prefix_space=A    )
								UpperCAmelCase     :		Dict							     =		tokenizer.convert_ids_to_tokens(encoded[0]    )[0]
								self.assertNotEqual(A		,		A    )

								UpperCAmelCase     :		Tuple							     =		tokenizer.encode(A		,		add_special_tokens=A		,		add_prefix_space=A    )
								UpperCAmelCase     :		int							     =		tokenizer.convert_ids_to_tokens(encoded[0]    )[0]
								self.assertEqual(A		,		A    )

								tokenizer.add_special_tokens({"""bos_token""": """<s>"""}    )
								UpperCAmelCase     :		int							     =		tokenizer.encode(A		,		add_special_tokens=A    )
								UpperCAmelCase     :		List[Any]							     =		tokenizer.convert_ids_to_tokens(encoded[1]    )[0]
								self.assertNotEqual(A		,		A    )

								# Testing spaces after special tokens
								UpperCAmelCase     :		Union[str, Any]							     =		"""<mask>"""
								tokenizer.add_special_tokens(
								    {"""mask_token""": AddedToken(A		,		lstrip=A		,		rstrip=A    )}    )  # mask token has a left space
								UpperCAmelCase     :		str							     =		tokenizer.convert_tokens_to_ids(A    )

								UpperCAmelCase     :		Union[str, Any]							     =		"""Encode <mask> sequence"""
								UpperCAmelCase     :		Union[str, Any]							     =		"""Encode <mask>sequence"""

								UpperCAmelCase     :		Union[str, Any]							     =		tokenizer.encode(A    )
								UpperCAmelCase     :		Union[str, Any]							     =		encoded.index(A    )
								UpperCAmelCase     :		List[str]							     =		tokenizer.convert_ids_to_tokens(encoded[mask_loc + 1]    )[0]
								self.assertEqual(A		,		A    )

								UpperCAmelCase     :		Tuple							     =		tokenizer.encode(A    )
								UpperCAmelCase     :		Optional[int]							     =		encoded.index(A    )
								UpperCAmelCase     :		Optional[int]							     =		tokenizer.convert_ids_to_tokens(encoded[mask_loc + 1]    )[0]
								self.assertNotEqual(A		,		A    )





	def  _lowercase(  self    )			->			Optional[int]:
								pass





	def  _lowercase(  self    )			->			Any:
								for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
															with self.subTest(f'''{tokenizer.__class__.__name__} ({pretrained_name})'''    ):
																						UpperCAmelCase     :		str							     =		self.rust_tokenizer_class.from_pretrained(A		,		**A    )
																						UpperCAmelCase     :		int							     =		self.tokenizer_class.from_pretrained(A		,		**A    )
																						UpperCAmelCase     :		Dict							     =		"""A, <mask> AllenNLP sentence."""
																						UpperCAmelCase     :		Dict							     =		tokenizer_r.encode_plus(A		,		add_special_tokens=A		,		return_token_type_ids=A    )
																						UpperCAmelCase     :		Tuple							     =		tokenizer_p.encode_plus(A		,		add_special_tokens=A		,		return_token_type_ids=A    )

																						# token_type_ids should put 0 everywhere
																						self.assertEqual(sum(tokens_r["""token_type_ids"""]    )		,		sum(tokens_p["""token_type_ids"""]    )    )

																						# attention_mask should put 1 everywhere, so sum over length should be 1
																						self.assertEqual(
																						    sum(tokens_r["""attention_mask"""]    ) / len(tokens_r["""attention_mask"""]    )		,		sum(tokens_p["""attention_mask"""]    ) / len(tokens_p["""attention_mask"""]    )		,		)

																						UpperCAmelCase     :		List[Any]							     =		tokenizer_r.convert_ids_to_tokens(tokens_r["""input_ids"""]    )
																						UpperCAmelCase     :		int							     =		tokenizer_p.convert_ids_to_tokens(tokens_p["""input_ids"""]    )

																						# Rust correctly handles the space before the mask while python doesnt
																						self.assertSequenceEqual(tokens_p["""input_ids"""]		,		[0, 250, 6, 50264, 3823, 487, 21992, 3645, 4, 2]    )
																						self.assertSequenceEqual(tokens_r["""input_ids"""]		,		[0, 250, 6, 50264, 3823, 487, 21992, 3645, 4, 2]    )

																						self.assertSequenceEqual(
																						    A		,		["""<s>""", """A""", """,""", """<mask>""", """ĠAllen""", """N""", """LP""", """Ġsentence""", """.""", """</s>"""]    )
																						self.assertSequenceEqual(
																						    A		,		["""<s>""", """A""", """,""", """<mask>""", """ĠAllen""", """N""", """LP""", """Ġsentence""", """.""", """</s>"""]    )





	def  _lowercase(  self    )			->			List[Any]:
								for trim_offsets, add_prefix_space in itertools.product([True, False]		,		repeat=2    ):
															UpperCAmelCase     :		Optional[int]							     =		self.rust_tokenizer_class.from_pretrained(
															    self.tmpdirname		,		use_fast=A		,		add_prefix_space=A		,		trim_offsets=A    )

															UpperCAmelCase     :		Optional[int]							     =		json.loads(tokenizer_r.backend_tokenizer.pre_tokenizer.__getstate__()    )
															UpperCAmelCase     :		Optional[int]							     =		json.loads(tokenizer_r.backend_tokenizer.post_processor.__getstate__()    )

															self.assertEqual(pre_tokenizer_state["""add_prefix_space"""]		,		A    )

															self.assertEqual(post_processor_state["""add_prefix_space"""]		,		A    )
															self.assertEqual(post_processor_state["""trim_offsets"""]		,		A    )






	def  _lowercase(  self    )			->			Optional[Any]:
								# Test which aims to verify that the offsets are well adapted to the argument `add_prefix_space` and
								# `trim_offsets`
								for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
															with self.subTest(f'''{tokenizer.__class__.__name__} ({pretrained_name})'''    ):
																						UpperCAmelCase     :		Union[str, Any]							     =		"""hello"""  # `hello` is a token in the vocabulary of `pretrained_name`
																						UpperCAmelCase     :		int							     =		f'''{text_of_1_token} {text_of_1_token}'''

																						UpperCAmelCase     :		List[Any]							     =		self.rust_tokenizer_class.from_pretrained(
																						    A		,		use_fast=A		,		add_prefix_space=A		,		trim_offsets=A    )
																						UpperCAmelCase     :		str							     =		tokenizer_r(A		,		return_offsets_mapping=A		,		add_special_tokens=A    )
																						self.assertEqual(encoding.offset_mapping[0]		,		(0, len(A    ))    )
																						self.assertEqual(
																						    encoding.offset_mapping[1]		,		(len(A    ) + 1, len(A    ) + 1 + len(A    ))		,		)

																						UpperCAmelCase     :		Optional[Any]							     =		self.rust_tokenizer_class.from_pretrained(
																						    A		,		use_fast=A		,		add_prefix_space=A		,		trim_offsets=A    )
																						UpperCAmelCase     :		Dict							     =		tokenizer_r(A		,		return_offsets_mapping=A		,		add_special_tokens=A    )
																						self.assertEqual(encoding.offset_mapping[0]		,		(0, len(A    ))    )
																						self.assertEqual(
																						    encoding.offset_mapping[1]		,		(len(A    ) + 1, len(A    ) + 1 + len(A    ))		,		)

																						UpperCAmelCase     :		int							     =		self.rust_tokenizer_class.from_pretrained(
																						    A		,		use_fast=A		,		add_prefix_space=A		,		trim_offsets=A    )
																						UpperCAmelCase     :		List[Any]							     =		tokenizer_r(A		,		return_offsets_mapping=A		,		add_special_tokens=A    )
																						self.assertEqual(encoding.offset_mapping[0]		,		(0, len(A    ))    )
																						self.assertEqual(
																						    encoding.offset_mapping[1]		,		(len(A    ), len(A    ) + 1 + len(A    ))		,		)

																						UpperCAmelCase     :		Any							     =		self.rust_tokenizer_class.from_pretrained(
																						    A		,		use_fast=A		,		add_prefix_space=A		,		trim_offsets=A    )
																						UpperCAmelCase     :		str							     =		tokenizer_r(A		,		return_offsets_mapping=A		,		add_special_tokens=A    )
																						self.assertEqual(encoding.offset_mapping[0]		,		(0, len(A    ))    )
																						self.assertEqual(
																						    encoding.offset_mapping[1]		,		(len(A    ), len(A    ) + 1 + len(A    ))		,		)

																						UpperCAmelCase     :		Optional[Any]							     =		f''' {text}'''

																						# tokenizer_r = self.rust_tokenizer_class.from_pretrained(
																						#     pretrained_name, use_fast=True, add_prefix_space=True, trim_offsets=True
																						# )
																						# encoding = tokenizer_r(text, return_offsets_mapping=True, add_special_tokens=False)
																						# self.assertEqual(encoding.offset_mapping[0], (1, 1 + len(text_of_1_token)))
																						# self.assertEqual(
																						#     encoding.offset_mapping[1],
																						#     (1 + len(text_of_1_token) + 1, 1 + len(text_of_1_token) + 1 + len(text_of_1_token)),
																						# )

																						UpperCAmelCase     :		Any							     =		self.rust_tokenizer_class.from_pretrained(
																						    A		,		use_fast=A		,		add_prefix_space=A		,		trim_offsets=A    )
																						UpperCAmelCase     :		str							     =		tokenizer_r(A		,		return_offsets_mapping=A		,		add_special_tokens=A    )
																						self.assertEqual(encoding.offset_mapping[0]		,		(1, 1 + len(A    ))    )
																						self.assertEqual(
																						    encoding.offset_mapping[1]		,		(1 + len(A    ) + 1, 1 + len(A    ) + 1 + len(A    ))		,		)

																						UpperCAmelCase     :		Optional[int]							     =		self.rust_tokenizer_class.from_pretrained(
																						    A		,		use_fast=A		,		add_prefix_space=A		,		trim_offsets=A    )
																						UpperCAmelCase     :		Union[str, Any]							     =		tokenizer_r(A		,		return_offsets_mapping=A		,		add_special_tokens=A    )
																						self.assertEqual(encoding.offset_mapping[0]		,		(0, 1 + len(A    ))    )
																						self.assertEqual(
																						    encoding.offset_mapping[1]		,		(1 + len(A    ), 1 + len(A    ) + 1 + len(A    ))		,		)

																						UpperCAmelCase     :		Union[str, Any]							     =		self.rust_tokenizer_class.from_pretrained(
																						    A		,		use_fast=A		,		add_prefix_space=A		,		trim_offsets=A    )
																						UpperCAmelCase     :		Optional[int]							     =		tokenizer_r(A		,		return_offsets_mapping=A		,		add_special_tokens=A    )
																						self.assertEqual(encoding.offset_mapping[0]		,		(0, 1 + len(A    ))    )
																						self.assertEqual(
																						    encoding.offset_mapping[1]		,		(1 + len(A    ), 1 + len(A    ) + 1 + len(A    ))		,		)

