






"""simple docstring"""
# DISCLAIMER: This code is strongly influenced by https://github.com/pesser/pytorch_diffusion
# and https://github.com/hojonathanho/diffusion
import math
from dataclasses import dataclass
from typing import List, Optional, Tuple, Union

import numpy as np
import torch

from diffusers.configuration_utils import ConfigMixin, register_to_config
from diffusers.schedulers.scheduling_utils import SchedulerMixin
from diffusers.utils import BaseOutput, deprecate







@dataclass
# Copied from diffusers.schedulers.scheduling_ddpm.DDPMSchedulerOutput with DDPM->DDIM
class 			UpperCamelCase_       (__A     ):
	__magic_name__	      =				42
	__magic_name__	      =				None





def 				snake_case      (   A__	,A__=0.999	,A__="cosine"	,):
			if alpha_transform_type == "cosine":

						def alpha_bar_fn(A__						):
									return math.cos((t + 0.008) / 1.008 * math.pi / 2						) ** 2

			elif alpha_transform_type == "exp":

						def alpha_bar_fn(A__						):
									return math.exp(t * -12.0						)

			else:
						raise ValueError(F"""Unsupported alpha_tranform_type: {alpha_transform_type}"""						)

			UpperCAmelCase_     : List[str]			     =					[]
			for i in range(A__						):
						UpperCAmelCase_     : List[str]			     =					i / num_diffusion_timesteps
						UpperCAmelCase_     : Union[str, Any]			     =					(i + 1) / num_diffusion_timesteps
						betas.append(min(1 - alpha_bar_fn(A__						) / alpha_bar_fn(A__						)	,A__						)						)
			return torch.tensor(A__	,dtype=torch.floataa						)







class 			UpperCamelCase_       (__A				,						__A     ):
	__magic_name__	      =				1





	@register_to_config
	def __init__(      self					: List[Any]	,	lowerCAmelCase_					: int = 1_000	,	lowerCAmelCase_					: float = 0.0_0_0_1	,	lowerCAmelCase_					: float = 0.0_2	,	lowerCAmelCase_					: str = "linear"	,	lowerCAmelCase_					: Optional[Union[np.ndarray, List[float]]] = None	,	lowerCAmelCase_					: bool = True	,	lowerCAmelCase_					: bool = True	,	lowerCAmelCase_					: int = 0	,	lowerCAmelCase_					: str = "epsilon"	,	lowerCAmelCase_					: float = 1.0	,	**lowerCAmelCase_					: Optional[int]	,	)						->			Any:
				if kwargs.get("set_alpha_to_one"	,	lowerCAmelCase_						) is not None:
							UpperCAmelCase_     : Tuple			     =					(
							    "The `set_alpha_to_one` argument is deprecated. Please use `set_alpha_to_zero` instead."
							)
							deprecate("set_alpha_to_one"	,	"1.0.0"	,	lowerCAmelCase_	,	standard_warn=lowerCAmelCase_						)
							UpperCAmelCase_     : Dict			     =					kwargs["set_alpha_to_one"]
				if trained_betas is not None:
							UpperCAmelCase_     : List[Any]			     =					torch.tensor(lowerCAmelCase_	,	dtype=torch.floataa						)
				elif beta_schedule == "linear":
							UpperCAmelCase_     : Any			     =					torch.linspace(lowerCAmelCase_	,	lowerCAmelCase_	,	lowerCAmelCase_	,	dtype=torch.floataa						)
				elif beta_schedule == "scaled_linear":
							# this schedule is very specific to the latent diffusion model.
							UpperCAmelCase_     : int			     =					(
							    torch.linspace(beta_start**0.5	,	beta_end**0.5	,	lowerCAmelCase_	,	dtype=torch.floataa						) ** 2
							)
				elif beta_schedule == "squaredcos_cap_v2":
							# Glide cosine schedule
							UpperCAmelCase_     : Optional[Any]			     =					betas_for_alpha_bar(lowerCAmelCase_						)
				else:
							raise NotImplementedError(f"""{beta_schedule} does is not implemented for {self.__class__}"""						)

				UpperCAmelCase_     : List[str]			     =					1.0 - self.betas
				UpperCAmelCase_     : Dict			     =					torch.cumprod(self.alphas	,	dim=0						)

				# At every step in inverted ddim, we are looking into the next alphas_cumprod
				# For the final step, there is no next alphas_cumprod, and the index is out of bounds
				# `set_alpha_to_zero` decides whether we set this parameter simply to zero
				# in this case, self.step() just output the predicted noise
				# or whether we use the final alpha of the "non-previous" one.
				UpperCAmelCase_     : List[Any]			     =					torch.tensor(0.0						) if set_alpha_to_zero else self.alphas_cumprod[-1]

				# standard deviation of the initial noise distribution
				UpperCAmelCase_     : Tuple			     =					1.0

				# setable values
				UpperCAmelCase_     : str			     =					None
				UpperCAmelCase_     : List[str]			     =					torch.from_numpy(np.arange(0	,	lowerCAmelCase_						).copy().astype(np.intaa						)						)





	def      _SCREAMING_SNAKE_CASE      (      self					: List[Any]	,	lowerCAmelCase_					: torch.FloatTensor	,	lowerCAmelCase_					: Optional[int] = None						)						->			torch.FloatTensor:
				return sample





	def      _SCREAMING_SNAKE_CASE      (      self					: Optional[int]	,	lowerCAmelCase_					: int	,	lowerCAmelCase_					: Union[str, torch.device] = None						)						->			str:

				if num_inference_steps > self.config.num_train_timesteps:
							raise ValueError(
							    f"""`num_inference_steps`: {num_inference_steps} cannot be larger than `self.config.train_timesteps`:"""
							    f""" {self.config.num_train_timesteps} as the unet model trained with this scheduler can only handle"""
							    f""" maximal {self.config.num_train_timesteps} timesteps."""						)

				UpperCAmelCase_     : List[Any]			     =					num_inference_steps
				UpperCAmelCase_     : List[Any]			     =					self.config.num_train_timesteps // self.num_inference_steps
				# creates integer timesteps by multiplying by ratio
				# casting to int to avoid issues when num_inference_step is power of 3
				UpperCAmelCase_     : str			     =					(np.arange(0	,	lowerCAmelCase_						) * step_ratio).round().copy().astype(np.intaa						)
				UpperCAmelCase_     : Dict			     =					torch.from_numpy(lowerCAmelCase_						).to(lowerCAmelCase_						)
				self.timesteps += self.config.steps_offset





	def      _SCREAMING_SNAKE_CASE      (      self					: Any	,	lowerCAmelCase_					: torch.FloatTensor	,	lowerCAmelCase_					: int	,	lowerCAmelCase_					: torch.FloatTensor	,	lowerCAmelCase_					: float = 0.0	,	lowerCAmelCase_					: bool = False	,	lowerCAmelCase_					: Optional[torch.FloatTensor] = None	,	lowerCAmelCase_					: bool = True	,	)						->			Union[DDIMSchedulerOutput, Tuple]:
				# 1. get previous step value (=t+1)
				UpperCAmelCase_     : Dict			     =					timestep + self.config.num_train_timesteps // self.num_inference_steps

				# 2. compute alphas, betas
				# change original implementation to exactly match noise levels for analogous forward process
				UpperCAmelCase_     : Optional[Any]			     =					self.alphas_cumprod[timestep]
				UpperCAmelCase_     : Any			     =					(
				    self.alphas_cumprod[prev_timestep]
				    if prev_timestep < self.config.num_train_timesteps
				    else self.final_alpha_cumprod
				)

				UpperCAmelCase_     : Tuple			     =					1 - alpha_prod_t

				# 3. compute predicted original sample from predicted noise also called
				# "predicted x_0" of formula (12) from https://arxiv.org/pdf/2010.02502.pdf
				if self.config.prediction_type == "epsilon":
							UpperCAmelCase_     : Dict			     =					(sample - beta_prod_t ** 0.5 * model_output) / alpha_prod_t ** 0.5
							UpperCAmelCase_     : int			     =					model_output
				elif self.config.prediction_type == "sample":
							UpperCAmelCase_     : Tuple			     =					model_output
							UpperCAmelCase_     : int			     =					(sample - alpha_prod_t ** 0.5 * pred_original_sample) / beta_prod_t ** 0.5
				elif self.config.prediction_type == "v_prediction":
							UpperCAmelCase_     : Optional[Any]			     =					(alpha_prod_t**0.5) * sample - (beta_prod_t**0.5) * model_output
							UpperCAmelCase_     : Union[str, Any]			     =					(alpha_prod_t**0.5) * model_output + (beta_prod_t**0.5) * sample
				else:
							raise ValueError(
							    f"""prediction_type given as {self.config.prediction_type} must be one of `epsilon`, `sample`, or"""
							    " `v_prediction`"						)

				# 4. Clip or threshold "predicted x_0"
				if self.config.clip_sample:
							UpperCAmelCase_     : Tuple			     =					pred_original_sample.clamp(
							    -self.config.clip_sample_range	,	self.config.clip_sample_range						)

				# 5. compute "direction pointing to x_t" of formula (12) from https://arxiv.org/pdf/2010.02502.pdf
				UpperCAmelCase_     : List[Any]			     =					(1 - alpha_prod_t_prev) ** 0.5 * pred_epsilon

				# 6. compute x_t without "random noise" of formula (12) from https://arxiv.org/pdf/2010.02502.pdf
				UpperCAmelCase_     : Optional[int]			     =					alpha_prod_t_prev ** 0.5 * pred_original_sample + pred_sample_direction

				if not return_dict:
							return (prev_sample, pred_original_sample)
				return DDIMSchedulerOutput(prev_sample=lowerCAmelCase_	,	pred_original_sample=lowerCAmelCase_						)





	def __len__(      self					: Optional[Any]						)						->			Dict:
				return self.config.num_train_timesteps











"""simple docstring"""
import os
import unittest

from transformers import BertTokenizerFast
from transformers.models.bert.tokenization_bert import (
    VOCAB_FILES_NAMES,
    BasicTokenizer,
    BertTokenizer,
    WordpieceTokenizer,
    _is_control,
    _is_punctuation,
    _is_whitespace,
)
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin, filter_non_english







@require_tokenizers
class 			UpperCamelCase_       (__A				,						unittest.TestCase     ):
	__magic_name__	      =				BertTokenizer
	__magic_name__	      =				BertTokenizerFast
	__magic_name__	      =				True
	__magic_name__	      =				True
	__magic_name__	      =				filter_non_english





	def      _SCREAMING_SNAKE_CASE      (      self					: Tuple						)						->			List[Any]:
				super().setUp()

				UpperCAmelCase_     : Tuple			     =					[
				    "[UNK]",
				    "[CLS]",
				    "[SEP]",
				    "[PAD]",
				    "[MASK]",
				    "want",
				    "##want",
				    "##ed",
				    "wa",
				    "un",
				    "runn",
				    "##ing",
				    ",",
				    "low",
				    "lowest",
				]
				UpperCAmelCase_     : Any			     =					os.path.join(self.tmpdirname	,	VOCAB_FILES_NAMES["vocab_file"]						)
				with open(self.vocab_file	,	"w"	,	encoding="utf-8"						) as vocab_writer:
							vocab_writer.write("".join([x + "\n" for x in vocab_tokens]						)						)





	def      _SCREAMING_SNAKE_CASE      (      self					: Any	,	lowerCAmelCase_					: str						)						->			Union[str, Any]:
				UpperCAmelCase_     : Tuple			     =					"UNwant\u00E9d,running"
				UpperCAmelCase_     : Any			     =					"unwanted, running"
				return input_text, output_text





	def      _SCREAMING_SNAKE_CASE      (      self					: Optional[Any]						)						->			List[str]:
				UpperCAmelCase_     : Any			     =					self.tokenizer_class(self.vocab_file						)

				UpperCAmelCase_     : Tuple			     =					tokenizer.tokenize("UNwant\u00E9d,running"						)
				self.assertListEqual(lowerCAmelCase_	,	["un", "##want", "##ed", ",", "runn", "##ing"]						)
				self.assertListEqual(tokenizer.convert_tokens_to_ids(lowerCAmelCase_						)	,	[9, 6, 7, 12, 10, 11]						)





	def      _SCREAMING_SNAKE_CASE      (      self					: List[str]						)						->			Any:
				if not self.test_rust_tokenizer:
							return

				UpperCAmelCase_     : Union[str, Any]			     =					self.get_tokenizer()
				UpperCAmelCase_     : List[Any]			     =					self.get_rust_tokenizer()

				UpperCAmelCase_     : List[Any]			     =					"UNwant\u00E9d,running"

				UpperCAmelCase_     : Union[str, Any]			     =					tokenizer.tokenize(lowerCAmelCase_						)
				UpperCAmelCase_     : int			     =					rust_tokenizer.tokenize(lowerCAmelCase_						)
				self.assertListEqual(lowerCAmelCase_	,	lowerCAmelCase_						)

				UpperCAmelCase_     : Any			     =					tokenizer.encode(lowerCAmelCase_	,	add_special_tokens=lowerCAmelCase_						)
				UpperCAmelCase_     : Optional[int]			     =					rust_tokenizer.encode(lowerCAmelCase_	,	add_special_tokens=lowerCAmelCase_						)
				self.assertListEqual(lowerCAmelCase_	,	lowerCAmelCase_						)

				UpperCAmelCase_     : Optional[Any]			     =					self.get_rust_tokenizer()
				UpperCAmelCase_     : Dict			     =					tokenizer.encode(lowerCAmelCase_						)
				UpperCAmelCase_     : Optional[Any]			     =					rust_tokenizer.encode(lowerCAmelCase_						)
				self.assertListEqual(lowerCAmelCase_	,	lowerCAmelCase_						)

				# With lower casing
				UpperCAmelCase_     : Tuple			     =					self.get_tokenizer(do_lower_case=lowerCAmelCase_						)
				UpperCAmelCase_     : Optional[int]			     =					self.get_rust_tokenizer(do_lower_case=lowerCAmelCase_						)

				UpperCAmelCase_     : List[Any]			     =					"UNwant\u00E9d,running"

				UpperCAmelCase_     : List[Any]			     =					tokenizer.tokenize(lowerCAmelCase_						)
				UpperCAmelCase_     : Tuple			     =					rust_tokenizer.tokenize(lowerCAmelCase_						)
				self.assertListEqual(lowerCAmelCase_	,	lowerCAmelCase_						)

				UpperCAmelCase_     : Union[str, Any]			     =					tokenizer.encode(lowerCAmelCase_	,	add_special_tokens=lowerCAmelCase_						)
				UpperCAmelCase_     : Union[str, Any]			     =					rust_tokenizer.encode(lowerCAmelCase_	,	add_special_tokens=lowerCAmelCase_						)
				self.assertListEqual(lowerCAmelCase_	,	lowerCAmelCase_						)

				UpperCAmelCase_     : Optional[Any]			     =					self.get_rust_tokenizer()
				UpperCAmelCase_     : Union[str, Any]			     =					tokenizer.encode(lowerCAmelCase_						)
				UpperCAmelCase_     : Union[str, Any]			     =					rust_tokenizer.encode(lowerCAmelCase_						)
				self.assertListEqual(lowerCAmelCase_	,	lowerCAmelCase_						)





	def      _SCREAMING_SNAKE_CASE      (      self					: Any						)						->			str:
				UpperCAmelCase_     : Optional[Any]			     =					BasicTokenizer()

				self.assertListEqual(tokenizer.tokenize("ah\u535A\u63A8zz"						)	,	["ah", "\u535A", "\u63A8", "zz"]						)





	def      _SCREAMING_SNAKE_CASE      (      self					: List[str]						)						->			Any:
				UpperCAmelCase_     : Optional[Any]			     =					BasicTokenizer(do_lower_case=lowerCAmelCase_						)

				self.assertListEqual(
				    tokenizer.tokenize(" \tHeLLo!how  \n Are yoU?  "						)	,	["hello", "!", "how", "are", "you", "?"]						)
				self.assertListEqual(tokenizer.tokenize("H\u00E9llo"						)	,	["hello"]						)





	def      _SCREAMING_SNAKE_CASE      (      self					: Tuple						)						->			Any:
				UpperCAmelCase_     : List[str]			     =					BasicTokenizer(do_lower_case=lowerCAmelCase_	,	strip_accents=lowerCAmelCase_						)

				self.assertListEqual(
				    tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "						)	,	["hällo", "!", "how", "are", "you", "?"]						)
				self.assertListEqual(tokenizer.tokenize("H\u00E9llo"						)	,	["h\u00E9llo"]						)





	def      _SCREAMING_SNAKE_CASE      (      self					: int						)						->			int:
				UpperCAmelCase_     : Any			     =					BasicTokenizer(do_lower_case=lowerCAmelCase_	,	strip_accents=lowerCAmelCase_						)

				self.assertListEqual(
				    tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "						)	,	["hallo", "!", "how", "are", "you", "?"]						)
				self.assertListEqual(tokenizer.tokenize("H\u00E9llo"						)	,	["hello"]						)





	def      _SCREAMING_SNAKE_CASE      (      self					: List[str]						)						->			Optional[int]:
				UpperCAmelCase_     : Tuple			     =					BasicTokenizer(do_lower_case=lowerCAmelCase_						)

				self.assertListEqual(
				    tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "						)	,	["hallo", "!", "how", "are", "you", "?"]						)
				self.assertListEqual(tokenizer.tokenize("H\u00E9llo"						)	,	["hello"]						)





	def      _SCREAMING_SNAKE_CASE      (      self					: Union[str, Any]						)						->			str:
				UpperCAmelCase_     : List[str]			     =					BasicTokenizer(do_lower_case=lowerCAmelCase_						)

				self.assertListEqual(
				    tokenizer.tokenize(" \tHeLLo!how  \n Are yoU?  "						)	,	["HeLLo", "!", "how", "Are", "yoU", "?"]						)





	def      _SCREAMING_SNAKE_CASE      (      self					: Optional[Any]						)						->			List[Any]:
				UpperCAmelCase_     : Optional[int]			     =					BasicTokenizer(do_lower_case=lowerCAmelCase_	,	strip_accents=lowerCAmelCase_						)

				self.assertListEqual(
				    tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "						)	,	["HäLLo", "!", "how", "Are", "yoU", "?"]						)





	def      _SCREAMING_SNAKE_CASE      (      self					: str						)						->			Optional[Any]:
				UpperCAmelCase_     : Union[str, Any]			     =					BasicTokenizer(do_lower_case=lowerCAmelCase_	,	strip_accents=lowerCAmelCase_						)

				self.assertListEqual(
				    tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "						)	,	["HaLLo", "!", "how", "Are", "yoU", "?"]						)





	def      _SCREAMING_SNAKE_CASE      (      self					: Union[str, Any]						)						->			int:
				UpperCAmelCase_     : Tuple			     =					BasicTokenizer(do_lower_case=lowerCAmelCase_	,	never_split=["[UNK]"]						)

				self.assertListEqual(
				    tokenizer.tokenize(" \tHeLLo!how  \n Are yoU? [UNK]"						)	,	["HeLLo", "!", "how", "Are", "yoU", "?", "[UNK]"]						)





	def      _SCREAMING_SNAKE_CASE      (      self					: Optional[Any]						)						->			Any:
				UpperCAmelCase_     : Tuple			     =					BasicTokenizer()
				UpperCAmelCase_     : Dict			     =					"a\n'll !!to?'d of, can't."
				UpperCAmelCase_     : List[str]			     =					["a", "'", "ll", "!", "!", "to", "?", "'", "d", "of", ",", "can", "'", "t", "."]
				self.assertListEqual(tokenizer.tokenize(lowerCAmelCase_						)	,	lowerCAmelCase_						)





	def      _SCREAMING_SNAKE_CASE      (      self					: Dict						)						->			int:
				UpperCAmelCase_     : int			     =					["[UNK]", "[CLS]", "[SEP]", "want", "##want", "##ed", "wa", "un", "runn", "##ing"]

				UpperCAmelCase_     : Tuple			     =					{}
				for i, token in enumerate(lowerCAmelCase_						):
							UpperCAmelCase_     : Optional[int]			     =					i
				UpperCAmelCase_     : Optional[Any]			     =					WordpieceTokenizer(vocab=lowerCAmelCase_	,	unk_token="[UNK]"						)

				self.assertListEqual(tokenizer.tokenize(""						)	,	[]						)

				self.assertListEqual(tokenizer.tokenize("unwanted running"						)	,	["un", "##want", "##ed", "runn", "##ing"]						)

				self.assertListEqual(tokenizer.tokenize("unwantedX running"						)	,	["[UNK]", "runn", "##ing"]						)





	def      _SCREAMING_SNAKE_CASE      (      self					: Dict						)						->			Optional[int]:
				self.assertTrue(_is_whitespace(" "						)						)
				self.assertTrue(_is_whitespace("\t"						)						)
				self.assertTrue(_is_whitespace("\r"						)						)
				self.assertTrue(_is_whitespace("\n"						)						)
				self.assertTrue(_is_whitespace("\u00A0"						)						)

				self.assertFalse(_is_whitespace("A"						)						)
				self.assertFalse(_is_whitespace("-"						)						)





	def      _SCREAMING_SNAKE_CASE      (      self					: Any						)						->			Optional[int]:
				self.assertTrue(_is_control("\u0005"						)						)

				self.assertFalse(_is_control("A"						)						)
				self.assertFalse(_is_control(" "						)						)
				self.assertFalse(_is_control("\t"						)						)
				self.assertFalse(_is_control("\r"						)						)





	def      _SCREAMING_SNAKE_CASE      (      self					: int						)						->			List[str]:
				self.assertTrue(_is_punctuation("-"						)						)
				self.assertTrue(_is_punctuation("$"						)						)
				self.assertTrue(_is_punctuation("`"						)						)
				self.assertTrue(_is_punctuation("."						)						)

				self.assertFalse(_is_punctuation("A"						)						)
				self.assertFalse(_is_punctuation(" "						)						)





	def      _SCREAMING_SNAKE_CASE      (      self					: str						)						->			Optional[Any]:
				UpperCAmelCase_     : Dict			     =					self.get_tokenizer()
				UpperCAmelCase_     : List[str]			     =					self.get_rust_tokenizer()

				# Example taken from the issue https://github.com/huggingface/tokenizers/issues/340
				self.assertListEqual([tokenizer.tokenize(lowerCAmelCase_						) for t in ["Test", "\xad", "test"]]	,	[["[UNK]"], [], ["[UNK]"]]						)

				self.assertListEqual(
				    [rust_tokenizer.tokenize(lowerCAmelCase_						) for t in ["Test", "\xad", "test"]]	,	[["[UNK]"], [], ["[UNK]"]]						)





	@slow
	def      _SCREAMING_SNAKE_CASE      (      self					: Optional[Any]						)						->			Optional[Any]:
				UpperCAmelCase_     : List[str]			     =					self.tokenizer_class.from_pretrained("bert-base-uncased"						)

				UpperCAmelCase_     : Any			     =					tokenizer.encode("sequence builders"	,	add_special_tokens=lowerCAmelCase_						)
				UpperCAmelCase_     : Union[str, Any]			     =					tokenizer.encode("multi-sequence build"	,	add_special_tokens=lowerCAmelCase_						)

				UpperCAmelCase_     : Tuple			     =					tokenizer.build_inputs_with_special_tokens(lowerCAmelCase_						)
				UpperCAmelCase_     : Dict			     =					tokenizer.build_inputs_with_special_tokens(lowerCAmelCase_	,	lowerCAmelCase_						)

				assert encoded_sentence == [101] + text + [102]
				assert encoded_pair == [101] + text + [102] + text_a + [102]





	def      _SCREAMING_SNAKE_CASE      (      self					: Any						)						->			str:
				for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
							with self.subTest(f"""{tokenizer.__class__.__name__} ({pretrained_name})"""						):
										UpperCAmelCase_     : str			     =					self.rust_tokenizer_class.from_pretrained(lowerCAmelCase_	,	**lowerCAmelCase_						)

										UpperCAmelCase_     : Union[str, Any]			     =					f"""A, naïve {tokenizer_r.mask_token} AllenNLP sentence."""
										UpperCAmelCase_     : Tuple			     =					tokenizer_r.encode_plus(
										    lowerCAmelCase_	,	return_attention_mask=lowerCAmelCase_	,	return_token_type_ids=lowerCAmelCase_	,	return_offsets_mapping=lowerCAmelCase_	,	add_special_tokens=lowerCAmelCase_	,	)

										UpperCAmelCase_     : Optional[int]			     =					tokenizer_r.do_lower_case if hasattr(lowerCAmelCase_	,	"do_lower_case"						) else False
										UpperCAmelCase_     : List[Any]			     =					(
										    [
										        ((0, 0), tokenizer_r.cls_token),
										        ((0, 1), "A"),
										        ((1, 2), ","),
										        ((3, 5), "na"),
										        ((5, 6), "##ï"),
										        ((6, 8), "##ve"),
										        ((9, 15), tokenizer_r.mask_token),
										        ((16, 21), "Allen"),
										        ((21, 23), "##NL"),
										        ((23, 24), "##P"),
										        ((25, 33), "sentence"),
										        ((33, 34), "."),
										        ((0, 0), tokenizer_r.sep_token),
										    ]
										    if not do_lower_case
										    else [
										        ((0, 0), tokenizer_r.cls_token),
										        ((0, 1), "a"),
										        ((1, 2), ","),
										        ((3, 8), "naive"),
										        ((9, 15), tokenizer_r.mask_token),
										        ((16, 21), "allen"),
										        ((21, 23), "##nl"),
										        ((23, 24), "##p"),
										        ((25, 33), "sentence"),
										        ((33, 34), "."),
										        ((0, 0), tokenizer_r.sep_token),
										    ]
										)

										self.assertEqual(
										    [e[1] for e in expected_results]	,	tokenizer_r.convert_ids_to_tokens(tokens["input_ids"]						)						)
										self.assertEqual([e[0] for e in expected_results]	,	tokens["offset_mapping"]						)





	def      _SCREAMING_SNAKE_CASE      (      self					: Union[str, Any]						)						->			Optional[Any]:
				UpperCAmelCase_     : List[Any]			     =					["的", "人", "有"]
				UpperCAmelCase_     : Tuple			     =					"".join(lowerCAmelCase_						)
				for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
							with self.subTest(f"""{tokenizer.__class__.__name__} ({pretrained_name})"""						):
										UpperCAmelCase_     : Optional[Any]			     =					True
										UpperCAmelCase_     : Any			     =					self.tokenizer_class.from_pretrained(lowerCAmelCase_	,	**lowerCAmelCase_						)
										UpperCAmelCase_     : Optional[Any]			     =					self.rust_tokenizer_class.from_pretrained(lowerCAmelCase_	,	**lowerCAmelCase_						)

										UpperCAmelCase_     : Dict			     =					tokenizer_p.encode(lowerCAmelCase_	,	add_special_tokens=lowerCAmelCase_						)
										UpperCAmelCase_     : Dict			     =					tokenizer_r.encode(lowerCAmelCase_	,	add_special_tokens=lowerCAmelCase_						)

										UpperCAmelCase_     : Union[str, Any]			     =					tokenizer_r.convert_ids_to_tokens(lowerCAmelCase_						)
										UpperCAmelCase_     : Any			     =					tokenizer_p.convert_ids_to_tokens(lowerCAmelCase_						)

										# it is expected that each Chinese character is not preceded by "##"
										self.assertListEqual(lowerCAmelCase_	,	lowerCAmelCase_						)
										self.assertListEqual(lowerCAmelCase_	,	lowerCAmelCase_						)

										UpperCAmelCase_     : List[Any]			     =					False
										UpperCAmelCase_     : Optional[Any]			     =					self.rust_tokenizer_class.from_pretrained(lowerCAmelCase_	,	**lowerCAmelCase_						)
										UpperCAmelCase_     : List[Any]			     =					self.tokenizer_class.from_pretrained(lowerCAmelCase_	,	**lowerCAmelCase_						)

										UpperCAmelCase_     : int			     =					tokenizer_r.encode(lowerCAmelCase_	,	add_special_tokens=lowerCAmelCase_						)
										UpperCAmelCase_     : List[str]			     =					tokenizer_p.encode(lowerCAmelCase_	,	add_special_tokens=lowerCAmelCase_						)

										UpperCAmelCase_     : Union[str, Any]			     =					tokenizer_r.convert_ids_to_tokens(lowerCAmelCase_						)
										UpperCAmelCase_     : List[str]			     =					tokenizer_p.convert_ids_to_tokens(lowerCAmelCase_						)

										# it is expected that only the first Chinese character is not preceded by "##".
										UpperCAmelCase_     : Tuple			     =					[
										    f"""##{token}""" if idx != 0 else token for idx, token in enumerate(lowerCAmelCase_						)
										]
										self.assertListEqual(lowerCAmelCase_	,	lowerCAmelCase_						)
										self.assertListEqual(lowerCAmelCase_	,	lowerCAmelCase_						)



