
import functools
import operator

from ...configuration_utils import PretrainedConfig
from ...utils import logging


lowercase  :					str													=       logging.get_logger(__name__)

lowercase  :					int													=       {
    'asapp/sew-d-tiny-100k': 'https://huggingface.co/asapp/sew-d-tiny-100k/resolve/main/config.json',
    # See all SEW-D models at https://huggingface.co/models?filter=sew-d
}




class  lowerCamelCase__							(				__A):




	'''simple docstring'''




	_A								=     'sew-d'







	def __init__(					self		:Optional[Any]     ,   a		:List[str]=3_2     ,   a		:Dict=7_6_8     ,   a		:List[str]=1_2     ,   a		:List[Any]=1_2     ,   a		:Tuple=3_0_7_2     ,   a		:Any=2     ,   a		:Optional[Any]=5_1_2     ,   a		:List[str]=2_5_6     ,   a		:int=True     ,   a		:Dict=True     ,   a		:str=("p2c", "c2p")     ,   a		:Union[str, Any]="layer_norm"     ,   a		:Union[str, Any]="gelu_python"     ,   a		:Optional[Any]=0.1     ,   a		:int=0.1     ,   a		:Any=0.1     ,   a		:int=0.0     ,   a		:Optional[int]=0.1     ,   a		:List[Any]=0.02     ,   a		:Union[str, Any]=1E-7     ,   a		:List[str]=1E-5     ,   a		:Dict="group"     ,   a		:Dict="gelu"     ,   a		:Optional[int]=(6_4, 1_2_8, 1_2_8, 1_2_8, 1_2_8, 2_5_6, 2_5_6, 2_5_6, 2_5_6, 5_1_2, 5_1_2, 5_1_2, 5_1_2)     ,   a		:Tuple=(5, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1)     ,   a		:Union[str, Any]=(1_0, 3, 1, 3, 1, 3, 1, 3, 1, 2, 1, 2, 1)     ,   a		:str=False     ,   a		:Optional[Any]=1_2_8     ,   a		:Optional[int]=1_6     ,   a		:Optional[int]=True     ,   a		:int=0.05     ,   a		:Any=1_0     ,   a		:Dict=2     ,   a		:List[Any]=0.0     ,   a		:Union[str, Any]=1_0     ,   a		:Dict=0     ,   a		:Optional[Any]="mean"     ,   a		:Tuple=False     ,   a		:List[str]=False     ,   a		:List[str]=2_5_6     ,   a		:Optional[Any]=0     ,   a		:int=1     ,   a		:Union[str, Any]=2     ,   **a		:Optional[int]     ,   )		-> Dict:
								super().__init__(**lowerCAmelCase_     ,   pad_token_id=lowerCAmelCase_     ,   bos_token_id=lowerCAmelCase_     ,   eos_token_id=lowerCAmelCase_						)
								__UpperCamelCase	:   List[str]			   =					hidden_size
								__UpperCamelCase	:   Tuple			   =					feat_extract_norm
								__UpperCamelCase	:   Optional[Any]			   =					feat_extract_activation
								__UpperCamelCase	:   str			   =					list(lowerCAmelCase_						)
								__UpperCamelCase	:   Any			   =					list(lowerCAmelCase_						)
								__UpperCamelCase	:   Optional[Any]			   =					list(lowerCAmelCase_						)
								__UpperCamelCase	:   Tuple			   =					conv_bias
								__UpperCamelCase	:   List[str]			   =					num_conv_pos_embeddings
								__UpperCamelCase	:   Tuple			   =					num_conv_pos_embedding_groups
								__UpperCamelCase	:   Tuple			   =					len(self.conv_dim						)
								__UpperCamelCase	:   Optional[Any]			   =					num_hidden_layers
								__UpperCamelCase	:   Any			   =					intermediate_size
								__UpperCamelCase	:   Tuple			   =					squeeze_factor
								__UpperCamelCase	:   Optional[Any]			   =					max_position_embeddings
								__UpperCamelCase	:   Tuple			   =					position_buckets
								__UpperCamelCase	:   List[str]			   =					share_att_key
								__UpperCamelCase	:   Dict			   =					relative_attention
								__UpperCamelCase	:   Union[str, Any]			   =					norm_rel_ebd
								__UpperCamelCase	:   Dict			   =					list(lowerCAmelCase_						)
								__UpperCamelCase	:   Optional[Any]			   =					hidden_act
								__UpperCamelCase	:   List[Any]			   =					num_attention_heads
								__UpperCamelCase	:   Optional[int]			   =					hidden_dropout
								__UpperCamelCase	:   Union[str, Any]			   =					attention_dropout
								__UpperCamelCase	:   int			   =					activation_dropout
								__UpperCamelCase	:   Tuple			   =					feat_proj_dropout
								__UpperCamelCase	:   Optional[int]			   =					final_dropout
								__UpperCamelCase	:   Tuple			   =					layer_norm_eps
								__UpperCamelCase	:   str			   =					feature_layer_norm_eps
								__UpperCamelCase	:   Union[str, Any]			   =					initializer_range
								__UpperCamelCase	:   Any			   =					vocab_size

								if (
								    (len(self.conv_stride						) != self.num_feat_extract_layers)
								    or (len(self.conv_kernel						) != self.num_feat_extract_layers)
								    or (len(self.conv_dim						) != self.num_feat_extract_layers)
								):
															raise ValueError(
															    "Configuration for convolutional layers is incorrect."
															    "It is required that `len(config.conv_dim)` == `len(config.conv_stride)` == `len(config.conv_kernel)`,"
															    f'but is `len(config.conv_dim) = {len(self.conv_dim						)}`, `len(config.conv_stride)'
															    f'= {len(self.conv_stride						)}`, `len(config.conv_kernel) = {len(self.conv_kernel						)}`.'						)

								# fine-tuning config parameters for SpecAugment: https://arxiv.org/abs/1904.08779
								__UpperCamelCase	:   Optional[int]			   =					apply_spec_augment
								__UpperCamelCase	:   Optional[Any]			   =					mask_time_prob
								__UpperCamelCase	:   int			   =					mask_time_length
								__UpperCamelCase	:   List[Any]			   =					mask_time_min_masks
								__UpperCamelCase	:   Optional[Any]			   =					mask_feature_prob
								__UpperCamelCase	:   List[Any]			   =					mask_feature_length
								__UpperCamelCase	:   str			   =					mask_feature_min_masks

								# ctc loss
								__UpperCamelCase	:   Any			   =					ctc_loss_reduction
								__UpperCamelCase	:   Dict			   =					ctc_zero_infinity

								# sequence classification
								__UpperCamelCase	:   Dict			   =					use_weighted_layer_sum
								__UpperCamelCase	:   Union[str, Any]			   =					classifier_proj_size







	@property
	def    _lowerCamelCase				(					self		:Tuple						)		-> Optional[Any]:
								return functools.reduce(operator.mul     ,   self.conv_stride     ,   1						)







"""simple docstring"""
import os
import unittest

from transformers import BertTokenizerFast
from transformers.models.bert.tokenization_bert import (
    VOCAB_FILES_NAMES,
    BasicTokenizer,
    BertTokenizer,
    WordpieceTokenizer,
    _is_control,
    _is_punctuation,
    _is_whitespace,
)
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin, filter_non_english







@require_tokenizers
class 			UpperCamelCase_       (__A				,						unittest.TestCase     ):
	__magic_name__	      =				BertTokenizer
	__magic_name__	      =				BertTokenizerFast
	__magic_name__	      =				True
	__magic_name__	      =				True
	__magic_name__	      =				filter_non_english





	def      _SCREAMING_SNAKE_CASE      (      self					: Tuple						)						->			List[Any]:
				super().setUp()

				UpperCAmelCase_     : Tuple			     =					[
				    "[UNK]",
				    "[CLS]",
				    "[SEP]",
				    "[PAD]",
				    "[MASK]",
				    "want",
				    "##want",
				    "##ed",
				    "wa",
				    "un",
				    "runn",
				    "##ing",
				    ",",
				    "low",
				    "lowest",
				]
				UpperCAmelCase_     : Any			     =					os.path.join(self.tmpdirname	,	VOCAB_FILES_NAMES["vocab_file"]						)
				with open(self.vocab_file	,	"w"	,	encoding="utf-8"						) as vocab_writer:
							vocab_writer.write("".join([x + "\n" for x in vocab_tokens]						)						)





	def      _SCREAMING_SNAKE_CASE      (      self					: Any	,	lowerCAmelCase_					: str						)						->			Union[str, Any]:
				UpperCAmelCase_     : Tuple			     =					"UNwant\u00E9d,running"
				UpperCAmelCase_     : Any			     =					"unwanted, running"
				return input_text, output_text





	def      _SCREAMING_SNAKE_CASE      (      self					: Optional[Any]						)						->			List[str]:
				UpperCAmelCase_     : Any			     =					self.tokenizer_class(self.vocab_file						)

				UpperCAmelCase_     : Tuple			     =					tokenizer.tokenize("UNwant\u00E9d,running"						)
				self.assertListEqual(lowerCAmelCase_	,	["un", "##want", "##ed", ",", "runn", "##ing"]						)
				self.assertListEqual(tokenizer.convert_tokens_to_ids(lowerCAmelCase_						)	,	[9, 6, 7, 12, 10, 11]						)





	def      _SCREAMING_SNAKE_CASE      (      self					: List[str]						)						->			Any:
				if not self.test_rust_tokenizer:
							return

				UpperCAmelCase_     : Union[str, Any]			     =					self.get_tokenizer()
				UpperCAmelCase_     : List[Any]			     =					self.get_rust_tokenizer()

				UpperCAmelCase_     : List[Any]			     =					"UNwant\u00E9d,running"

				UpperCAmelCase_     : Union[str, Any]			     =					tokenizer.tokenize(lowerCAmelCase_						)
				UpperCAmelCase_     : int			     =					rust_tokenizer.tokenize(lowerCAmelCase_						)
				self.assertListEqual(lowerCAmelCase_	,	lowerCAmelCase_						)

				UpperCAmelCase_     : Any			     =					tokenizer.encode(lowerCAmelCase_	,	add_special_tokens=lowerCAmelCase_						)
				UpperCAmelCase_     : Optional[int]			     =					rust_tokenizer.encode(lowerCAmelCase_	,	add_special_tokens=lowerCAmelCase_						)
				self.assertListEqual(lowerCAmelCase_	,	lowerCAmelCase_						)

				UpperCAmelCase_     : Optional[Any]			     =					self.get_rust_tokenizer()
				UpperCAmelCase_     : Dict			     =					tokenizer.encode(lowerCAmelCase_						)
				UpperCAmelCase_     : Optional[Any]			     =					rust_tokenizer.encode(lowerCAmelCase_						)
				self.assertListEqual(lowerCAmelCase_	,	lowerCAmelCase_						)

				# With lower casing
				UpperCAmelCase_     : Tuple			     =					self.get_tokenizer(do_lower_case=lowerCAmelCase_						)
				UpperCAmelCase_     : Optional[int]			     =					self.get_rust_tokenizer(do_lower_case=lowerCAmelCase_						)

				UpperCAmelCase_     : List[Any]			     =					"UNwant\u00E9d,running"

				UpperCAmelCase_     : List[Any]			     =					tokenizer.tokenize(lowerCAmelCase_						)
				UpperCAmelCase_     : Tuple			     =					rust_tokenizer.tokenize(lowerCAmelCase_						)
				self.assertListEqual(lowerCAmelCase_	,	lowerCAmelCase_						)

				UpperCAmelCase_     : Union[str, Any]			     =					tokenizer.encode(lowerCAmelCase_	,	add_special_tokens=lowerCAmelCase_						)
				UpperCAmelCase_     : Union[str, Any]			     =					rust_tokenizer.encode(lowerCAmelCase_	,	add_special_tokens=lowerCAmelCase_						)
				self.assertListEqual(lowerCAmelCase_	,	lowerCAmelCase_						)

				UpperCAmelCase_     : Optional[Any]			     =					self.get_rust_tokenizer()
				UpperCAmelCase_     : Union[str, Any]			     =					tokenizer.encode(lowerCAmelCase_						)
				UpperCAmelCase_     : Union[str, Any]			     =					rust_tokenizer.encode(lowerCAmelCase_						)
				self.assertListEqual(lowerCAmelCase_	,	lowerCAmelCase_						)





	def      _SCREAMING_SNAKE_CASE      (      self					: Any						)						->			str:
				UpperCAmelCase_     : Optional[Any]			     =					BasicTokenizer()

				self.assertListEqual(tokenizer.tokenize("ah\u535A\u63A8zz"						)	,	["ah", "\u535A", "\u63A8", "zz"]						)





	def      _SCREAMING_SNAKE_CASE      (      self					: List[str]						)						->			Any:
				UpperCAmelCase_     : Optional[Any]			     =					BasicTokenizer(do_lower_case=lowerCAmelCase_						)

				self.assertListEqual(
				    tokenizer.tokenize(" \tHeLLo!how  \n Are yoU?  "						)	,	["hello", "!", "how", "are", "you", "?"]						)
				self.assertListEqual(tokenizer.tokenize("H\u00E9llo"						)	,	["hello"]						)





	def      _SCREAMING_SNAKE_CASE      (      self					: Tuple						)						->			Any:
				UpperCAmelCase_     : List[str]			     =					BasicTokenizer(do_lower_case=lowerCAmelCase_	,	strip_accents=lowerCAmelCase_						)

				self.assertListEqual(
				    tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "						)	,	["hällo", "!", "how", "are", "you", "?"]						)
				self.assertListEqual(tokenizer.tokenize("H\u00E9llo"						)	,	["h\u00E9llo"]						)





	def      _SCREAMING_SNAKE_CASE      (      self					: int						)						->			int:
				UpperCAmelCase_     : Any			     =					BasicTokenizer(do_lower_case=lowerCAmelCase_	,	strip_accents=lowerCAmelCase_						)

				self.assertListEqual(
				    tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "						)	,	["hallo", "!", "how", "are", "you", "?"]						)
				self.assertListEqual(tokenizer.tokenize("H\u00E9llo"						)	,	["hello"]						)





	def      _SCREAMING_SNAKE_CASE      (      self					: List[str]						)						->			Optional[int]:
				UpperCAmelCase_     : Tuple			     =					BasicTokenizer(do_lower_case=lowerCAmelCase_						)

				self.assertListEqual(
				    tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "						)	,	["hallo", "!", "how", "are", "you", "?"]						)
				self.assertListEqual(tokenizer.tokenize("H\u00E9llo"						)	,	["hello"]						)





	def      _SCREAMING_SNAKE_CASE      (      self					: Union[str, Any]						)						->			str:
				UpperCAmelCase_     : List[str]			     =					BasicTokenizer(do_lower_case=lowerCAmelCase_						)

				self.assertListEqual(
				    tokenizer.tokenize(" \tHeLLo!how  \n Are yoU?  "						)	,	["HeLLo", "!", "how", "Are", "yoU", "?"]						)





	def      _SCREAMING_SNAKE_CASE      (      self					: Optional[Any]						)						->			List[Any]:
				UpperCAmelCase_     : Optional[int]			     =					BasicTokenizer(do_lower_case=lowerCAmelCase_	,	strip_accents=lowerCAmelCase_						)

				self.assertListEqual(
				    tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "						)	,	["HäLLo", "!", "how", "Are", "yoU", "?"]						)





	def      _SCREAMING_SNAKE_CASE      (      self					: str						)						->			Optional[Any]:
				UpperCAmelCase_     : Union[str, Any]			     =					BasicTokenizer(do_lower_case=lowerCAmelCase_	,	strip_accents=lowerCAmelCase_						)

				self.assertListEqual(
				    tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "						)	,	["HaLLo", "!", "how", "Are", "yoU", "?"]						)





	def      _SCREAMING_SNAKE_CASE      (      self					: Union[str, Any]						)						->			int:
				UpperCAmelCase_     : Tuple			     =					BasicTokenizer(do_lower_case=lowerCAmelCase_	,	never_split=["[UNK]"]						)

				self.assertListEqual(
				    tokenizer.tokenize(" \tHeLLo!how  \n Are yoU? [UNK]"						)	,	["HeLLo", "!", "how", "Are", "yoU", "?", "[UNK]"]						)





	def      _SCREAMING_SNAKE_CASE      (      self					: Optional[Any]						)						->			Any:
				UpperCAmelCase_     : Tuple			     =					BasicTokenizer()
				UpperCAmelCase_     : Dict			     =					"a\n'll !!to?'d of, can't."
				UpperCAmelCase_     : List[str]			     =					["a", "'", "ll", "!", "!", "to", "?", "'", "d", "of", ",", "can", "'", "t", "."]
				self.assertListEqual(tokenizer.tokenize(lowerCAmelCase_						)	,	lowerCAmelCase_						)





	def      _SCREAMING_SNAKE_CASE      (      self					: Dict						)						->			int:
				UpperCAmelCase_     : int			     =					["[UNK]", "[CLS]", "[SEP]", "want", "##want", "##ed", "wa", "un", "runn", "##ing"]

				UpperCAmelCase_     : Tuple			     =					{}
				for i, token in enumerate(lowerCAmelCase_						):
							UpperCAmelCase_     : Optional[int]			     =					i
				UpperCAmelCase_     : Optional[Any]			     =					WordpieceTokenizer(vocab=lowerCAmelCase_	,	unk_token="[UNK]"						)

				self.assertListEqual(tokenizer.tokenize(""						)	,	[]						)

				self.assertListEqual(tokenizer.tokenize("unwanted running"						)	,	["un", "##want", "##ed", "runn", "##ing"]						)

				self.assertListEqual(tokenizer.tokenize("unwantedX running"						)	,	["[UNK]", "runn", "##ing"]						)





	def      _SCREAMING_SNAKE_CASE      (      self					: Dict						)						->			Optional[int]:
				self.assertTrue(_is_whitespace(" "						)						)
				self.assertTrue(_is_whitespace("\t"						)						)
				self.assertTrue(_is_whitespace("\r"						)						)
				self.assertTrue(_is_whitespace("\n"						)						)
				self.assertTrue(_is_whitespace("\u00A0"						)						)

				self.assertFalse(_is_whitespace("A"						)						)
				self.assertFalse(_is_whitespace("-"						)						)





	def      _SCREAMING_SNAKE_CASE      (      self					: Any						)						->			Optional[int]:
				self.assertTrue(_is_control("\u0005"						)						)

				self.assertFalse(_is_control("A"						)						)
				self.assertFalse(_is_control(" "						)						)
				self.assertFalse(_is_control("\t"						)						)
				self.assertFalse(_is_control("\r"						)						)





	def      _SCREAMING_SNAKE_CASE      (      self					: int						)						->			List[str]:
				self.assertTrue(_is_punctuation("-"						)						)
				self.assertTrue(_is_punctuation("$"						)						)
				self.assertTrue(_is_punctuation("`"						)						)
				self.assertTrue(_is_punctuation("."						)						)

				self.assertFalse(_is_punctuation("A"						)						)
				self.assertFalse(_is_punctuation(" "						)						)





	def      _SCREAMING_SNAKE_CASE      (      self					: str						)						->			Optional[Any]:
				UpperCAmelCase_     : Dict			     =					self.get_tokenizer()
				UpperCAmelCase_     : List[str]			     =					self.get_rust_tokenizer()

				# Example taken from the issue https://github.com/huggingface/tokenizers/issues/340
				self.assertListEqual([tokenizer.tokenize(lowerCAmelCase_						) for t in ["Test", "\xad", "test"]]	,	[["[UNK]"], [], ["[UNK]"]]						)

				self.assertListEqual(
				    [rust_tokenizer.tokenize(lowerCAmelCase_						) for t in ["Test", "\xad", "test"]]	,	[["[UNK]"], [], ["[UNK]"]]						)





	@slow
	def      _SCREAMING_SNAKE_CASE      (      self					: Optional[Any]						)						->			Optional[Any]:
				UpperCAmelCase_     : List[str]			     =					self.tokenizer_class.from_pretrained("bert-base-uncased"						)

				UpperCAmelCase_     : Any			     =					tokenizer.encode("sequence builders"	,	add_special_tokens=lowerCAmelCase_						)
				UpperCAmelCase_     : Union[str, Any]			     =					tokenizer.encode("multi-sequence build"	,	add_special_tokens=lowerCAmelCase_						)

				UpperCAmelCase_     : Tuple			     =					tokenizer.build_inputs_with_special_tokens(lowerCAmelCase_						)
				UpperCAmelCase_     : Dict			     =					tokenizer.build_inputs_with_special_tokens(lowerCAmelCase_	,	lowerCAmelCase_						)

				assert encoded_sentence == [101] + text + [102]
				assert encoded_pair == [101] + text + [102] + text_a + [102]





	def      _SCREAMING_SNAKE_CASE      (      self					: Any						)						->			str:
				for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
							with self.subTest(f"""{tokenizer.__class__.__name__} ({pretrained_name})"""						):
										UpperCAmelCase_     : str			     =					self.rust_tokenizer_class.from_pretrained(lowerCAmelCase_	,	**lowerCAmelCase_						)

										UpperCAmelCase_     : Union[str, Any]			     =					f"""A, naïve {tokenizer_r.mask_token} AllenNLP sentence."""
										UpperCAmelCase_     : Tuple			     =					tokenizer_r.encode_plus(
										    lowerCAmelCase_	,	return_attention_mask=lowerCAmelCase_	,	return_token_type_ids=lowerCAmelCase_	,	return_offsets_mapping=lowerCAmelCase_	,	add_special_tokens=lowerCAmelCase_	,	)

										UpperCAmelCase_     : Optional[int]			     =					tokenizer_r.do_lower_case if hasattr(lowerCAmelCase_	,	"do_lower_case"						) else False
										UpperCAmelCase_     : List[Any]			     =					(
										    [
										        ((0, 0), tokenizer_r.cls_token),
										        ((0, 1), "A"),
										        ((1, 2), ","),
										        ((3, 5), "na"),
										        ((5, 6), "##ï"),
										        ((6, 8), "##ve"),
										        ((9, 15), tokenizer_r.mask_token),
										        ((16, 21), "Allen"),
										        ((21, 23), "##NL"),
										        ((23, 24), "##P"),
										        ((25, 33), "sentence"),
										        ((33, 34), "."),
										        ((0, 0), tokenizer_r.sep_token),
										    ]
										    if not do_lower_case
										    else [
										        ((0, 0), tokenizer_r.cls_token),
										        ((0, 1), "a"),
										        ((1, 2), ","),
										        ((3, 8), "naive"),
										        ((9, 15), tokenizer_r.mask_token),
										        ((16, 21), "allen"),
										        ((21, 23), "##nl"),
										        ((23, 24), "##p"),
										        ((25, 33), "sentence"),
										        ((33, 34), "."),
										        ((0, 0), tokenizer_r.sep_token),
										    ]
										)

										self.assertEqual(
										    [e[1] for e in expected_results]	,	tokenizer_r.convert_ids_to_tokens(tokens["input_ids"]						)						)
										self.assertEqual([e[0] for e in expected_results]	,	tokens["offset_mapping"]						)





	def      _SCREAMING_SNAKE_CASE      (      self					: Union[str, Any]						)						->			Optional[Any]:
				UpperCAmelCase_     : List[Any]			     =					["的", "人", "有"]
				UpperCAmelCase_     : Tuple			     =					"".join(lowerCAmelCase_						)
				for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
							with self.subTest(f"""{tokenizer.__class__.__name__} ({pretrained_name})"""						):
										UpperCAmelCase_     : Optional[Any]			     =					True
										UpperCAmelCase_     : Any			     =					self.tokenizer_class.from_pretrained(lowerCAmelCase_	,	**lowerCAmelCase_						)
										UpperCAmelCase_     : Optional[Any]			     =					self.rust_tokenizer_class.from_pretrained(lowerCAmelCase_	,	**lowerCAmelCase_						)

										UpperCAmelCase_     : Dict			     =					tokenizer_p.encode(lowerCAmelCase_	,	add_special_tokens=lowerCAmelCase_						)
										UpperCAmelCase_     : Dict			     =					tokenizer_r.encode(lowerCAmelCase_	,	add_special_tokens=lowerCAmelCase_						)

										UpperCAmelCase_     : Union[str, Any]			     =					tokenizer_r.convert_ids_to_tokens(lowerCAmelCase_						)
										UpperCAmelCase_     : Any			     =					tokenizer_p.convert_ids_to_tokens(lowerCAmelCase_						)

										# it is expected that each Chinese character is not preceded by "##"
										self.assertListEqual(lowerCAmelCase_	,	lowerCAmelCase_						)
										self.assertListEqual(lowerCAmelCase_	,	lowerCAmelCase_						)

										UpperCAmelCase_     : List[Any]			     =					False
										UpperCAmelCase_     : Optional[Any]			     =					self.rust_tokenizer_class.from_pretrained(lowerCAmelCase_	,	**lowerCAmelCase_						)
										UpperCAmelCase_     : List[Any]			     =					self.tokenizer_class.from_pretrained(lowerCAmelCase_	,	**lowerCAmelCase_						)

										UpperCAmelCase_     : int			     =					tokenizer_r.encode(lowerCAmelCase_	,	add_special_tokens=lowerCAmelCase_						)
										UpperCAmelCase_     : List[str]			     =					tokenizer_p.encode(lowerCAmelCase_	,	add_special_tokens=lowerCAmelCase_						)

										UpperCAmelCase_     : Union[str, Any]			     =					tokenizer_r.convert_ids_to_tokens(lowerCAmelCase_						)
										UpperCAmelCase_     : List[str]			     =					tokenizer_p.convert_ids_to_tokens(lowerCAmelCase_						)

										# it is expected that only the first Chinese character is not preceded by "##".
										UpperCAmelCase_     : Tuple			     =					[
										    f"""##{token}""" if idx != 0 else token for idx, token in enumerate(lowerCAmelCase_						)
										]
										self.assertListEqual(lowerCAmelCase_	,	lowerCAmelCase_						)
										self.assertListEqual(lowerCAmelCase_	,	lowerCAmelCase_						)



