




'''simple docstring'''





import torch

from diffusers import DDPMScheduler

from .test_schedulers import SchedulerCommonTest
class 		SCREAMING_SNAKE_CASE			(     snake_case							):




				'''simple docstring'''





				__UpperCamelCase     				=   (DDPMScheduler,)







				def        _UpperCamelCase							(   self  ,						**SCREAMING_SNAKE_CASE__				):
							'''simple docstring'''







							snake_case:       Dict        =       {
							    'num_train_timesteps': 10_00,
							    'beta_start': 0.00_01,
							    'beta_end': 0.02,
							    'beta_schedule': 'linear',
							    'variance_type': 'fixed_small',
							    'clip_sample': True,
							}

							config.update(**SCREAMING_SNAKE_CASE__				)
							return config







				def        _UpperCamelCase							(   self				):
							'''simple docstring'''







							for timesteps in [1, 5, 1_00, 10_00]:
										self.check_over_configs(num_train_timesteps=SCREAMING_SNAKE_CASE__				)







				def        _UpperCamelCase							(   self				):
							'''simple docstring'''







							for beta_start, beta_end in zip([0.00_01, 0.0_01, 0.01, 0.1]  ,						[0.0_02, 0.02, 0.2, 2]				):
										self.check_over_configs(beta_start=SCREAMING_SNAKE_CASE__  ,						beta_end=SCREAMING_SNAKE_CASE__				)







				def        _UpperCamelCase							(   self				):
							'''simple docstring'''







							for schedule in ["linear", "squaredcos_cap_v2"]:
										self.check_over_configs(beta_schedule=SCREAMING_SNAKE_CASE__				)







				def        _UpperCamelCase							(   self				):
							'''simple docstring'''







							for variance in ["fixed_small", "fixed_large", "other"]:
										self.check_over_configs(variance_type=SCREAMING_SNAKE_CASE__				)







				def        _UpperCamelCase							(   self				):
							'''simple docstring'''







							for clip_sample in [True, False]:
										self.check_over_configs(clip_sample=SCREAMING_SNAKE_CASE__				)







				def        _UpperCamelCase							(   self				):
							'''simple docstring'''







							self.check_over_configs(thresholding=SCREAMING_SNAKE_CASE__				)
							for threshold in [0.5, 1.0, 2.0]:
										for prediction_type in ["epsilon", "sample", "v_prediction"]:
													self.check_over_configs(
													    thresholding=SCREAMING_SNAKE_CASE__  ,						prediction_type=SCREAMING_SNAKE_CASE__  ,						sample_max_value=SCREAMING_SNAKE_CASE__  ,						)







				def        _UpperCamelCase							(   self				):
							'''simple docstring'''







							for prediction_type in ["epsilon", "sample", "v_prediction"]:
										self.check_over_configs(prediction_type=SCREAMING_SNAKE_CASE__				)







				def        _UpperCamelCase							(   self				):
							'''simple docstring'''







							for t in [0, 5_00, 9_99]:
										self.check_over_forward(time_step=SCREAMING_SNAKE_CASE__				)







				def        _UpperCamelCase							(   self				):
							'''simple docstring'''







							snake_case:       Optional[Any]        =       self.scheduler_classes[0]
							snake_case:       List[str]        =       self.get_scheduler_config()
							snake_case:       Optional[int]        =       scheduler_class(**SCREAMING_SNAKE_CASE__				)

							assert torch.sum(torch.abs(scheduler._get_variance(0				) - 0.0				)				) < 1E-5
							assert torch.sum(torch.abs(scheduler._get_variance(4_87				) - 0.0_09_79				)				) < 1E-5
							assert torch.sum(torch.abs(scheduler._get_variance(9_99				) - 0.02				)				) < 1E-5







				def        _UpperCamelCase							(   self				):
							'''simple docstring'''







							snake_case:       Union[str, Any]        =       self.scheduler_classes[0]
							snake_case:       Dict        =       self.get_scheduler_config()
							snake_case:       int        =       scheduler_class(**SCREAMING_SNAKE_CASE__				)

							snake_case:       Optional[int]        =       len(SCREAMING_SNAKE_CASE__				)

							snake_case:       List[str]        =       self.dummy_model()
							snake_case:       Any        =       self.dummy_sample_deter
							snake_case:       List[str]        =       torch.manual_seed(0				)

							for t in reversed(range(SCREAMING_SNAKE_CASE__				)				):
										# 1. predict noise residual
										snake_case:       Tuple        =       model(SCREAMING_SNAKE_CASE__  ,						SCREAMING_SNAKE_CASE__				)

										# 2. predict previous mean of sample x_t-1
										snake_case:       Dict        =       scheduler.step(SCREAMING_SNAKE_CASE__  ,						SCREAMING_SNAKE_CASE__  ,						SCREAMING_SNAKE_CASE__  ,						generator=SCREAMING_SNAKE_CASE__				).prev_sample

										# if t > 0:
										#     noise = self.dummy_sample_deter
										#     variance = scheduler.get_variance(t) ** (0.5) * noise
										#
										# sample = pred_prev_sample + variance
										snake_case:       Optional[int]        =       pred_prev_sample

							snake_case:       List[Any]        =       torch.sum(torch.abs(SCREAMING_SNAKE_CASE__				)				)
							snake_case:       Tuple        =       torch.mean(torch.abs(SCREAMING_SNAKE_CASE__				)				)

							assert abs(result_sum.item() - 2_58.96_06				) < 1E-2
							assert abs(result_mean.item() - 0.33_72				) < 1E-3







				def        _UpperCamelCase							(   self				):
							'''simple docstring'''







							snake_case:       Any        =       self.scheduler_classes[0]
							snake_case:       Optional[int]        =       self.get_scheduler_config(prediction_type='v_prediction'				)
							snake_case:       Optional[Any]        =       scheduler_class(**SCREAMING_SNAKE_CASE__				)

							snake_case:       Tuple        =       len(SCREAMING_SNAKE_CASE__				)

							snake_case:       Union[str, Any]        =       self.dummy_model()
							snake_case:       Optional[int]        =       self.dummy_sample_deter
							snake_case:       str        =       torch.manual_seed(0				)

							for t in reversed(range(SCREAMING_SNAKE_CASE__				)				):
										# 1. predict noise residual
										snake_case:       str        =       model(SCREAMING_SNAKE_CASE__  ,						SCREAMING_SNAKE_CASE__				)

										# 2. predict previous mean of sample x_t-1
										snake_case:       Optional[Any]        =       scheduler.step(SCREAMING_SNAKE_CASE__  ,						SCREAMING_SNAKE_CASE__  ,						SCREAMING_SNAKE_CASE__  ,						generator=SCREAMING_SNAKE_CASE__				).prev_sample

										# if t > 0:
										#     noise = self.dummy_sample_deter
										#     variance = scheduler.get_variance(t) ** (0.5) * noise
										#
										# sample = pred_prev_sample + variance
										snake_case:       str        =       pred_prev_sample

							snake_case:       List[Any]        =       torch.sum(torch.abs(SCREAMING_SNAKE_CASE__				)				)
							snake_case:       List[str]        =       torch.mean(torch.abs(SCREAMING_SNAKE_CASE__				)				)

							assert abs(result_sum.item() - 2_02.02_96				) < 1E-2
							assert abs(result_mean.item() - 0.26_31				) < 1E-3







				def        _UpperCamelCase							(   self				):
							'''simple docstring'''







							snake_case:       str        =       self.scheduler_classes[0]
							snake_case:       Optional[int]        =       self.get_scheduler_config()
							snake_case:       Union[str, Any]        =       scheduler_class(**SCREAMING_SNAKE_CASE__				)

							snake_case:       Optional[Any]        =       [1_00, 87, 50, 1, 0]

							scheduler.set_timesteps(timesteps=SCREAMING_SNAKE_CASE__				)

							snake_case:       Union[str, Any]        =       scheduler.timesteps

							for i, timestep in enumerate(SCREAMING_SNAKE_CASE__				):
										if i == len(SCREAMING_SNAKE_CASE__				) - 1:
													snake_case:       Dict        =       -1
										else:
													snake_case:       Dict        =       timesteps[i + 1]

										snake_case:       Union[str, Any]        =       scheduler.previous_timestep(SCREAMING_SNAKE_CASE__				)
										snake_case:       Optional[int]        =       prev_t.item()

										self.assertEqual(SCREAMING_SNAKE_CASE__  ,						SCREAMING_SNAKE_CASE__				)







				def        _UpperCamelCase							(   self				):
							'''simple docstring'''







							snake_case:       Any        =       self.scheduler_classes[0]
							snake_case:       Any        =       self.get_scheduler_config()
							snake_case:       Optional[Any]        =       scheduler_class(**SCREAMING_SNAKE_CASE__				)

							snake_case:       Dict        =       [1_00, 87, 50, 51, 0]

							with self.assertRaises(SCREAMING_SNAKE_CASE__  ,						msg='`custom_timesteps` must be in descending order.'				):
										scheduler.set_timesteps(timesteps=SCREAMING_SNAKE_CASE__				)







				def        _UpperCamelCase							(   self				):
							'''simple docstring'''







							snake_case:       Optional[int]        =       self.scheduler_classes[0]
							snake_case:       List[Any]        =       self.get_scheduler_config()
							snake_case:       int        =       scheduler_class(**SCREAMING_SNAKE_CASE__				)

							snake_case:       Tuple        =       [1_00, 87, 50, 1, 0]
							snake_case:       Optional[Any]        =       len(SCREAMING_SNAKE_CASE__				)

							with self.assertRaises(SCREAMING_SNAKE_CASE__  ,						msg='Can only pass one of `num_inference_steps` or `custom_timesteps`.'				):
										scheduler.set_timesteps(num_inference_steps=SCREAMING_SNAKE_CASE__  ,						timesteps=SCREAMING_SNAKE_CASE__				)







				def        _UpperCamelCase							(   self				):
							'''simple docstring'''







							snake_case:       Any        =       self.scheduler_classes[0]
							snake_case:       str        =       self.get_scheduler_config()
							snake_case:       Dict        =       scheduler_class(**SCREAMING_SNAKE_CASE__				)

							snake_case:       str        =       [scheduler.config.num_train_timesteps]

							with self.assertRaises(
							    SCREAMING_SNAKE_CASE__  ,						msg='`timesteps` must start before `self.config.train_timesteps`: {scheduler.config.num_train_timesteps}}'  ,						):
										scheduler.set_timesteps(timesteps=SCREAMING_SNAKE_CASE__				)





"""simple docstring"""






import itertools
import json
import os
import unittest

from transformers import AddedToken, RobertaTokenizer, RobertaTokenizerFast
from transformers.models.roberta.tokenization_roberta import VOCAB_FILES_NAMES
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin






@require_tokenizers
class lowerCAmelCase__      (	lowercase,      unittest.TestCase		):






			'''simple docstring'''



			lowerCamelCase__ =     RobertaTokenizer
			lowerCamelCase__ =     RobertaTokenizerFast
			lowerCamelCase__ =     True
			lowerCamelCase__ =     {"""cls_token""": """<s>"""}
			def    A_       (    self    ):
							super().setUp()

							# Adapted from Sennrich et al. 2015 and https://github.com/rsennrich/subword-nmt
							_lowerCamelCase	:     Optional[Any]       =       [
							    'l',
							    'o',
							    'w',
							    'e',
							    'r',
							    's',
							    't',
							    'i',
							    'd',
							    'n',
							    '\u0120',
							    '\u0120l',
							    '\u0120n',
							    '\u0120lo',
							    '\u0120low',
							    'er',
							    '\u0120lowest',
							    '\u0120newer',
							    '\u0120wider',
							    '<unk>',
							]
							_lowerCamelCase	:     Optional[int]       =       dict(zip(lowercase    , range(len(lowercase    )    )    )    )
							_lowerCamelCase	:     Optional[int]       =       ['#version: 0.2', '\u0120 l', '\u0120l o', '\u0120lo w', 'e r', '']
							_lowerCamelCase	:     Optional[Any]       =       {'unk_token': '<unk>'}

							_lowerCamelCase	:     Optional[int]       =       os.path.join(self.tmpdirname    , VOCAB_FILES_NAMES['vocab_file']    )
							_lowerCamelCase	:     Optional[Any]       =       os.path.join(self.tmpdirname    , VOCAB_FILES_NAMES['merges_file']    )
							with open(self.vocab_file    , 'w'    , encoding='utf-8'    ) as fp:
											fp.write(json.dumps(lowercase    ) + '\n'    )
							with open(self.merges_file    , 'w'    , encoding='utf-8'    ) as fp:
											fp.write('\n'.join(lowercase    )    )
			def    A_       (    self    , **lowercase    ):
							kwargs.update(self.special_tokens_map    )
							return self.tokenizer_class.from_pretrained(self.tmpdirname    , **lowercase    )
			def    A_       (    self    , **lowercase    ):
							kwargs.update(self.special_tokens_map    )
							return RobertaTokenizerFast.from_pretrained(self.tmpdirname    , **lowercase    )
			def    A_       (    self    , lowercase    ):
							_lowerCamelCase	:     Any       =       'lower newer'
							_lowerCamelCase	:     Union[str, Any]       =       'lower newer'
							return input_text, output_text
			def    A_       (    self    ):
							_lowerCamelCase	:     Dict       =       self.tokenizer_class(self.vocab_file    , self.merges_file    , **self.special_tokens_map    )
							_lowerCamelCase	:     List[str]       =       'lower newer'
							_lowerCamelCase	:     Tuple       =       ['l', 'o', 'w', 'er', '\u0120', 'n', 'e', 'w', 'er']
							_lowerCamelCase	:     List[str]       =       tokenizer.tokenize(lowercase    )  # , add_prefix_space=True)
							self.assertListEqual(lowercase    , lowercase    )

							_lowerCamelCase	:     List[Any]       =       tokens + [tokenizer.unk_token]
							_lowerCamelCase	:     str       =       [0, 1, 2, 15, 10, 9, 3, 2, 15, 19]
							self.assertListEqual(tokenizer.convert_tokens_to_ids(lowercase    )    , lowercase    )
			def    A_       (    self    ):
							_lowerCamelCase	:     Union[str, Any]       =       self.get_tokenizer()

							self.assertListEqual(tokenizer.encode('Hello world!'    , add_special_tokens=lowercase    )    , [0, 31414, 232, 328, 2]    )
							self.assertListEqual(
							    tokenizer.encode('Hello world! cécé herlolip 418'    , add_special_tokens=lowercase    )    , [0, 31414, 232, 328, 740, 1140, 12695, 69, 46078, 1588, 2]    , )
			@slow
			def    A_       (    self    ):
							_lowerCamelCase	:     Dict       =       self.tokenizer_class.from_pretrained('roberta-base'    )

							_lowerCamelCase	:     Tuple       =       tokenizer.encode('sequence builders'    , add_special_tokens=lowercase    )
							_lowerCamelCase	:     int       =       tokenizer.encode('multi-sequence build'    , add_special_tokens=lowercase    )

							_lowerCamelCase	:     List[Any]       =       tokenizer.encode(
							    'sequence builders'    , add_special_tokens=lowercase    , add_prefix_space=lowercase    )
							_lowerCamelCase	:     List[str]       =       tokenizer.encode(
							    'sequence builders'    , 'multi-sequence build'    , add_special_tokens=lowercase    , add_prefix_space=lowercase    )

							_lowerCamelCase	:     Dict       =       tokenizer.build_inputs_with_special_tokens(lowercase    )
							_lowerCamelCase	:     Optional[int]       =       tokenizer.build_inputs_with_special_tokens(lowercase    , lowercase    )

							assert encoded_sentence == encoded_text_from_decode
							assert encoded_pair == encoded_pair_from_decode
			def    A_       (    self    ):
							_lowerCamelCase	:     Tuple       =       self.get_tokenizer()

							_lowerCamelCase	:     str       =       'Encode this sequence.'
							_lowerCamelCase	:     Optional[int]       =       tokenizer.byte_encoder[' '.encode('utf-8'    )[0]]

							# Testing encoder arguments
							_lowerCamelCase	:     Optional[int]       =       tokenizer.encode(lowercase    , add_special_tokens=lowercase    , add_prefix_space=lowercase    )
							_lowerCamelCase	:     Union[str, Any]       =       tokenizer.convert_ids_to_tokens(encoded[0]    )[0]
							self.assertNotEqual(lowercase    , lowercase    )

							_lowerCamelCase	:     List[str]       =       tokenizer.encode(lowercase    , add_special_tokens=lowercase    , add_prefix_space=lowercase    )
							_lowerCamelCase	:     List[Any]       =       tokenizer.convert_ids_to_tokens(encoded[0]    )[0]
							self.assertEqual(lowercase    , lowercase    )

							tokenizer.add_special_tokens({'bos_token': '<s>'}    )
							_lowerCamelCase	:     Optional[Any]       =       tokenizer.encode(lowercase    , add_special_tokens=lowercase    )
							_lowerCamelCase	:     Union[str, Any]       =       tokenizer.convert_ids_to_tokens(encoded[1]    )[0]
							self.assertNotEqual(lowercase    , lowercase    )

							# Testing spaces after special tokens
							_lowerCamelCase	:     List[Any]       =       '<mask>'
							tokenizer.add_special_tokens(
							    {'mask_token': AddedToken(lowercase    , lstrip=lowercase    , rstrip=lowercase    )}    )  # mask token has a left space
							_lowerCamelCase	:     List[str]       =       tokenizer.convert_tokens_to_ids(lowercase    )

							_lowerCamelCase	:     Optional[int]       =       'Encode <mask> sequence'
							_lowerCamelCase	:     int       =       'Encode <mask>sequence'

							_lowerCamelCase	:     str       =       tokenizer.encode(lowercase    )
							_lowerCamelCase	:     Tuple       =       encoded.index(lowercase    )
							_lowerCamelCase	:     int       =       tokenizer.convert_ids_to_tokens(encoded[mask_loc + 1]    )[0]
							self.assertEqual(lowercase    , lowercase    )

							_lowerCamelCase	:     Any       =       tokenizer.encode(lowercase    )
							_lowerCamelCase	:     Union[str, Any]       =       encoded.index(lowercase    )
							_lowerCamelCase	:     int       =       tokenizer.convert_ids_to_tokens(encoded[mask_loc + 1]    )[0]
							self.assertNotEqual(lowercase    , lowercase    )
			def    A_       (    self    ):
							pass
			def    A_       (    self    ):
							for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
											with self.subTest(F'''{tokenizer.__class__.__name__} ({pretrained_name})'''    ):
															_lowerCamelCase	:     Optional[Any]       =       self.rust_tokenizer_class.from_pretrained(lowercase    , **lowercase    )
															_lowerCamelCase	:     Dict       =       self.tokenizer_class.from_pretrained(lowercase    , **lowercase    )
															_lowerCamelCase	:     List[str]       =       'A, <mask> AllenNLP sentence.'
															_lowerCamelCase	:     List[Any]       =       tokenizer_r.encode_plus(lowercase    , add_special_tokens=lowercase    , return_token_type_ids=lowercase    )
															_lowerCamelCase	:     Union[str, Any]       =       tokenizer_p.encode_plus(lowercase    , add_special_tokens=lowercase    , return_token_type_ids=lowercase    )

															# token_type_ids should put 0 everywhere
															self.assertEqual(sum(tokens_r['token_type_ids']    )    , sum(tokens_p['token_type_ids']    )    )

															# attention_mask should put 1 everywhere, so sum over length should be 1
															self.assertEqual(
															    sum(tokens_r['attention_mask']    ) / len(tokens_r['attention_mask']    )    , sum(tokens_p['attention_mask']    ) / len(tokens_p['attention_mask']    )    , )

															_lowerCamelCase	:     List[Any]       =       tokenizer_r.convert_ids_to_tokens(tokens_r['input_ids']    )
															_lowerCamelCase	:     Tuple       =       tokenizer_p.convert_ids_to_tokens(tokens_p['input_ids']    )

															# Rust correctly handles the space before the mask while python doesnt
															self.assertSequenceEqual(tokens_p['input_ids']    , [0, 250, 6, 50264, 3823, 487, 21992, 3645, 4, 2]    )
															self.assertSequenceEqual(tokens_r['input_ids']    , [0, 250, 6, 50264, 3823, 487, 21992, 3645, 4, 2]    )

															self.assertSequenceEqual(
															    lowercase    , ['<s>', 'A', ',', '<mask>', 'ĠAllen', 'N', 'LP', 'Ġsentence', '.', '</s>']    )
															self.assertSequenceEqual(
															    lowercase    , ['<s>', 'A', ',', '<mask>', 'ĠAllen', 'N', 'LP', 'Ġsentence', '.', '</s>']    )
			def    A_       (    self    ):
							for trim_offsets, add_prefix_space in itertools.product([True, False]    , repeat=2    ):
											_lowerCamelCase	:     Union[str, Any]       =       self.rust_tokenizer_class.from_pretrained(
											    self.tmpdirname    , use_fast=lowercase    , add_prefix_space=lowercase    , trim_offsets=lowercase    )

											_lowerCamelCase	:     Optional[int]       =       json.loads(tokenizer_r.backend_tokenizer.pre_tokenizer.__getstate__()    )
											_lowerCamelCase	:     Dict       =       json.loads(tokenizer_r.backend_tokenizer.post_processor.__getstate__()    )

											self.assertEqual(pre_tokenizer_state['add_prefix_space']    , lowercase    )

											self.assertEqual(post_processor_state['add_prefix_space']    , lowercase    )
											self.assertEqual(post_processor_state['trim_offsets']    , lowercase    )






			def    A_       (    self    ):
							# Test which aims to verify that the offsets are well adapted to the argument `add_prefix_space` and
							# `trim_offsets`
							for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
											with self.subTest(F'''{tokenizer.__class__.__name__} ({pretrained_name})'''    ):
															_lowerCamelCase	:     Dict       =       'hello'  # `hello` is a token in the vocabulary of `pretrained_name`
															_lowerCamelCase	:     Tuple       =       F'''{text_of_1_token} {text_of_1_token}'''

															_lowerCamelCase	:     Dict       =       self.rust_tokenizer_class.from_pretrained(
															    lowercase    , use_fast=lowercase    , add_prefix_space=lowercase    , trim_offsets=lowercase    )
															_lowerCamelCase	:     List[Any]       =       tokenizer_r(lowercase    , return_offsets_mapping=lowercase    , add_special_tokens=lowercase    )
															self.assertEqual(encoding.offset_mapping[0]    , (0, len(lowercase    ))    )
															self.assertEqual(
															    encoding.offset_mapping[1]    , (len(lowercase    ) + 1, len(lowercase    ) + 1 + len(lowercase    ))    , )

															_lowerCamelCase	:     int       =       self.rust_tokenizer_class.from_pretrained(
															    lowercase    , use_fast=lowercase    , add_prefix_space=lowercase    , trim_offsets=lowercase    )
															_lowerCamelCase	:     Union[str, Any]       =       tokenizer_r(lowercase    , return_offsets_mapping=lowercase    , add_special_tokens=lowercase    )
															self.assertEqual(encoding.offset_mapping[0]    , (0, len(lowercase    ))    )
															self.assertEqual(
															    encoding.offset_mapping[1]    , (len(lowercase    ) + 1, len(lowercase    ) + 1 + len(lowercase    ))    , )

															_lowerCamelCase	:     Union[str, Any]       =       self.rust_tokenizer_class.from_pretrained(
															    lowercase    , use_fast=lowercase    , add_prefix_space=lowercase    , trim_offsets=lowercase    )
															_lowerCamelCase	:     List[str]       =       tokenizer_r(lowercase    , return_offsets_mapping=lowercase    , add_special_tokens=lowercase    )
															self.assertEqual(encoding.offset_mapping[0]    , (0, len(lowercase    ))    )
															self.assertEqual(
															    encoding.offset_mapping[1]    , (len(lowercase    ), len(lowercase    ) + 1 + len(lowercase    ))    , )

															_lowerCamelCase	:     Dict       =       self.rust_tokenizer_class.from_pretrained(
															    lowercase    , use_fast=lowercase    , add_prefix_space=lowercase    , trim_offsets=lowercase    )
															_lowerCamelCase	:     int       =       tokenizer_r(lowercase    , return_offsets_mapping=lowercase    , add_special_tokens=lowercase    )
															self.assertEqual(encoding.offset_mapping[0]    , (0, len(lowercase    ))    )
															self.assertEqual(
															    encoding.offset_mapping[1]    , (len(lowercase    ), len(lowercase    ) + 1 + len(lowercase    ))    , )

															_lowerCamelCase	:     int       =       F''' {text}'''

															# tokenizer_r = self.rust_tokenizer_class.from_pretrained(
															#     pretrained_name, use_fast=True, add_prefix_space=True, trim_offsets=True
															# )
															# encoding = tokenizer_r(text, return_offsets_mapping=True, add_special_tokens=False)
															# self.assertEqual(encoding.offset_mapping[0], (1, 1 + len(text_of_1_token)))
															# self.assertEqual(
															#     encoding.offset_mapping[1],
															#     (1 + len(text_of_1_token) + 1, 1 + len(text_of_1_token) + 1 + len(text_of_1_token)),
															# )

															_lowerCamelCase	:     Optional[Any]       =       self.rust_tokenizer_class.from_pretrained(
															    lowercase    , use_fast=lowercase    , add_prefix_space=lowercase    , trim_offsets=lowercase    )
															_lowerCamelCase	:     Optional[int]       =       tokenizer_r(lowercase    , return_offsets_mapping=lowercase    , add_special_tokens=lowercase    )
															self.assertEqual(encoding.offset_mapping[0]    , (1, 1 + len(lowercase    ))    )
															self.assertEqual(
															    encoding.offset_mapping[1]    , (1 + len(lowercase    ) + 1, 1 + len(lowercase    ) + 1 + len(lowercase    ))    , )

															_lowerCamelCase	:     Optional[int]       =       self.rust_tokenizer_class.from_pretrained(
															    lowercase    , use_fast=lowercase    , add_prefix_space=lowercase    , trim_offsets=lowercase    )
															_lowerCamelCase	:     Union[str, Any]       =       tokenizer_r(lowercase    , return_offsets_mapping=lowercase    , add_special_tokens=lowercase    )
															self.assertEqual(encoding.offset_mapping[0]    , (0, 1 + len(lowercase    ))    )
															self.assertEqual(
															    encoding.offset_mapping[1]    , (1 + len(lowercase    ), 1 + len(lowercase    ) + 1 + len(lowercase    ))    , )

															_lowerCamelCase	:     Union[str, Any]       =       self.rust_tokenizer_class.from_pretrained(
															    lowercase    , use_fast=lowercase    , add_prefix_space=lowercase    , trim_offsets=lowercase    )
															_lowerCamelCase	:     Optional[Any]       =       tokenizer_r(lowercase    , return_offsets_mapping=lowercase    , add_special_tokens=lowercase    )
															self.assertEqual(encoding.offset_mapping[0]    , (0, 1 + len(lowercase    ))    )
															self.assertEqual(
															    encoding.offset_mapping[1]    , (1 + len(lowercase    ), 1 + len(lowercase    ) + 1 + len(lowercase    ))    , )