
from __future__ import annotations

import random

# Maximum size of the population.  Bigger could be faster but is more memory expensive.
lowercase  :					Dict													=       200
# Number of elements selected in every generation of evolution. The selection takes
# place from best to worst of that generation and must be smaller than N_POPULATION.
lowercase  :					List[Any]													=       50
# Probability that an element of a generation can mutate, changing one of its genes.
# This will guarantee that all genes will be used during evolution.
lowercase  :					List[Any]													=       0.4
# Just a seed to improve randomness required by the algorithm.
random.seed(random.randint(0, 1000))




def 	_SCREAMING_SNAKE_CASE					(		_lowerCamelCase :			Dict						,  _lowerCamelCase :			Any)					->    Union[str, Any]:






							'''simple docstring'''





							__UpperCamelCase	:   Optional[Any]			   =					len([g for position, g in enumerate(A__) if g == main_target[position]])
							return (item, float(A__))




def 	_SCREAMING_SNAKE_CASE					(		_lowerCamelCase :			Optional[Any]						,  _lowerCamelCase :			Optional[Any])					->    int:






							'''simple docstring'''





							__UpperCamelCase	:   Tuple			   =					random.randint(0						,  len(A__) - 1)
							__UpperCamelCase	:   Optional[int]			   =					parent_a[:random_slice] + parent_a[random_slice:]
							__UpperCamelCase	:   Optional[Any]			   =					parent_a[:random_slice] + parent_a[random_slice:]
							return (child_a, child_a)




def 	_SCREAMING_SNAKE_CASE					(		_lowerCamelCase :			int						,  _lowerCamelCase :			Dict)					->    Tuple:






							'''simple docstring'''





							__UpperCamelCase	:   List[str]			   =					list(A__)
							if random.uniform(0						,  1) < MUTATION_PROBABILITY:
														__UpperCamelCase	:   Tuple			   =					random.choice(A__)
							return "".join(A__)




def 	_SCREAMING_SNAKE_CASE					(		_lowerCamelCase :			List[str]						,  _lowerCamelCase :			str						,  _lowerCamelCase :			Tuple						,  )					->    Optional[int]:






							'''simple docstring'''





							__UpperCamelCase	:   Optional[int]			   =					[]
							# Generate more children proportionally to the fitness score.
							__UpperCamelCase	:   Union[str, Any]			   =					int(parent_a[1] * 100) + 1
							__UpperCamelCase	:   Tuple			   =					10 if child_n >= 10 else child_n
							for _ in range(A__):
														__UpperCamelCase	:   Tuple			   =					population_score[random.randint(0						,  A__)][0]

														__UpperCamelCase	:   List[str]			   =					crossover(parent_a[0]						,  A__)
														# Append new string to the population list.
														pop.append(mutate(A__						,  A__))
														pop.append(mutate(A__						,  A__))
							return pop




def 	_SCREAMING_SNAKE_CASE					(		_lowerCamelCase :			Optional[Any]						,  _lowerCamelCase :			List[str]						,  _lowerCamelCase :			str = True)					->    Union[str, Any]:






							'''simple docstring'''





							if N_POPULATION < N_SELECTED:
														__UpperCamelCase	:   List[str]			   =					F'{N_POPULATION} must be bigger than {N_SELECTED}'
														raise ValueError(A__)
							# Verify that the target contains no genes besides the ones inside genes variable.
							__UpperCamelCase	:   List[Any]			   =					sorted({c for c in target if c not in genes})
							if not_in_genes_list:
														__UpperCamelCase	:   str			   =					F'{not_in_genes_list} is not in genes list, evolution cannot converge'
														raise ValueError(A__)

							# Generate random starting population.
							__UpperCamelCase	:   Optional[int]			   =					[]
							for _ in range(A__):
														population.append("".join([random.choice(A__) for i in range(len(A__))]))

							# Just some logs to know what the algorithms is doing.
							__UpperCamelCase	:   List[str]			   =					0, 0

							# This loop will end when we find a perfect match for our target.
							while True:
														generation += 1
														total_population += len(A__)

														# Random population created. Now it's time to evaluate.

														# Adding a bit of concurrency can make everything faster,
														#
														# import concurrent.futures
														# population_score: list[tuple[str, float]] = []
														# with concurrent.futures.ThreadPoolExecutor(
														#                                   max_workers=NUM_WORKERS) as executor:
														#     futures = {executor.submit(evaluate, item) for item in population}
														#     concurrent.futures.wait(futures)
														#     population_score = [item.result() for item in futures]
														#
														# but with a simple algorithm like this, it will probably be slower.
														# We just need to call evaluate for every item inside the population.
														__UpperCamelCase	:   Union[str, Any]			   =					[evaluate(A__						,  A__) for item in population]

														# Check if there is a matching evolution.
														__UpperCamelCase	:   Dict			   =					sorted(A__						,  key=lambda _lowerCamelCase: x[1]						,  reverse=A__)
														if population_score[0][0] == target:
																					return (generation, total_population, population_score[0][0])

														# Print the best result every 10 generation.
														# Just to know that the algorithm is working.
														if debug and generation % 10 == 0:
																					print(
																					    F'\nGeneration: {generation}'
																					    F'\nTotal Population:{total_population}'
																					    F'\nBest score: {population_score[0][1]}'
																					    F'\nBest string: {population_score[0][0]}')

														# Flush the old population, keeping some of the best evolutions.
														# Keeping this avoid regression of evolution.
														__UpperCamelCase	:   Dict			   =					population[: int(N_POPULATION / 3)]
														population.clear()
														population.extend(A__)
														# Normalize population score to be between 0 and 1.
														__UpperCamelCase	:   Optional[Any]			   =					[
														    (item, score / len(A__)) for item, score in population_score
														]

														# This is selection
														for i in range(A__):
																					population.extend(select(population_score[int(A__)]						,  A__						,  A__))
																					# Check if the population has already reached the maximum value and if so,
																					# break the cycle.  If this check is disabled, the algorithm will take
																					# forever to compute large strings, but will also calculate small strings in
																					# a far fewer generations.
																					if len(A__) > N_POPULATION:
																												break


if __name__ == "__main__":
		lowercase  :					Dict													=       (
		    'This is a genetic algorithm to evaluate, combine, evolve, and mutate a string!'
		)
		lowercase  :					List[Any]													=       list(
		    ' ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklm'
		    'nopqrstuvwxyz.,;!?+-*#@^\'èéòà€ù=)(&%$£/\\'
		)
		lowercase							,			lowercase							,			lowercase  :					Any													=       basic(target_str, genes_list)
		print(
		    f"\nGeneration: {generation}\nTotal Population: {population}\nTarget: {target}"
		)







"""simple docstring"""
import os
import unittest

from transformers import BertTokenizerFast
from transformers.models.bert.tokenization_bert import (
    VOCAB_FILES_NAMES,
    BasicTokenizer,
    BertTokenizer,
    WordpieceTokenizer,
    _is_control,
    _is_punctuation,
    _is_whitespace,
)
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin, filter_non_english







@require_tokenizers
class 			UpperCamelCase_       (__A				,						unittest.TestCase     ):
	__magic_name__	      =				BertTokenizer
	__magic_name__	      =				BertTokenizerFast
	__magic_name__	      =				True
	__magic_name__	      =				True
	__magic_name__	      =				filter_non_english





	def      _SCREAMING_SNAKE_CASE      (      self					: Tuple						)						->			List[Any]:
				super().setUp()

				UpperCAmelCase_     : Tuple			     =					[
				    "[UNK]",
				    "[CLS]",
				    "[SEP]",
				    "[PAD]",
				    "[MASK]",
				    "want",
				    "##want",
				    "##ed",
				    "wa",
				    "un",
				    "runn",
				    "##ing",
				    ",",
				    "low",
				    "lowest",
				]
				UpperCAmelCase_     : Any			     =					os.path.join(self.tmpdirname	,	VOCAB_FILES_NAMES["vocab_file"]						)
				with open(self.vocab_file	,	"w"	,	encoding="utf-8"						) as vocab_writer:
							vocab_writer.write("".join([x + "\n" for x in vocab_tokens]						)						)





	def      _SCREAMING_SNAKE_CASE      (      self					: Any	,	lowerCAmelCase_					: str						)						->			Union[str, Any]:
				UpperCAmelCase_     : Tuple			     =					"UNwant\u00E9d,running"
				UpperCAmelCase_     : Any			     =					"unwanted, running"
				return input_text, output_text





	def      _SCREAMING_SNAKE_CASE      (      self					: Optional[Any]						)						->			List[str]:
				UpperCAmelCase_     : Any			     =					self.tokenizer_class(self.vocab_file						)

				UpperCAmelCase_     : Tuple			     =					tokenizer.tokenize("UNwant\u00E9d,running"						)
				self.assertListEqual(lowerCAmelCase_	,	["un", "##want", "##ed", ",", "runn", "##ing"]						)
				self.assertListEqual(tokenizer.convert_tokens_to_ids(lowerCAmelCase_						)	,	[9, 6, 7, 12, 10, 11]						)





	def      _SCREAMING_SNAKE_CASE      (      self					: List[str]						)						->			Any:
				if not self.test_rust_tokenizer:
							return

				UpperCAmelCase_     : Union[str, Any]			     =					self.get_tokenizer()
				UpperCAmelCase_     : List[Any]			     =					self.get_rust_tokenizer()

				UpperCAmelCase_     : List[Any]			     =					"UNwant\u00E9d,running"

				UpperCAmelCase_     : Union[str, Any]			     =					tokenizer.tokenize(lowerCAmelCase_						)
				UpperCAmelCase_     : int			     =					rust_tokenizer.tokenize(lowerCAmelCase_						)
				self.assertListEqual(lowerCAmelCase_	,	lowerCAmelCase_						)

				UpperCAmelCase_     : Any			     =					tokenizer.encode(lowerCAmelCase_	,	add_special_tokens=lowerCAmelCase_						)
				UpperCAmelCase_     : Optional[int]			     =					rust_tokenizer.encode(lowerCAmelCase_	,	add_special_tokens=lowerCAmelCase_						)
				self.assertListEqual(lowerCAmelCase_	,	lowerCAmelCase_						)

				UpperCAmelCase_     : Optional[Any]			     =					self.get_rust_tokenizer()
				UpperCAmelCase_     : Dict			     =					tokenizer.encode(lowerCAmelCase_						)
				UpperCAmelCase_     : Optional[Any]			     =					rust_tokenizer.encode(lowerCAmelCase_						)
				self.assertListEqual(lowerCAmelCase_	,	lowerCAmelCase_						)

				# With lower casing
				UpperCAmelCase_     : Tuple			     =					self.get_tokenizer(do_lower_case=lowerCAmelCase_						)
				UpperCAmelCase_     : Optional[int]			     =					self.get_rust_tokenizer(do_lower_case=lowerCAmelCase_						)

				UpperCAmelCase_     : List[Any]			     =					"UNwant\u00E9d,running"

				UpperCAmelCase_     : List[Any]			     =					tokenizer.tokenize(lowerCAmelCase_						)
				UpperCAmelCase_     : Tuple			     =					rust_tokenizer.tokenize(lowerCAmelCase_						)
				self.assertListEqual(lowerCAmelCase_	,	lowerCAmelCase_						)

				UpperCAmelCase_     : Union[str, Any]			     =					tokenizer.encode(lowerCAmelCase_	,	add_special_tokens=lowerCAmelCase_						)
				UpperCAmelCase_     : Union[str, Any]			     =					rust_tokenizer.encode(lowerCAmelCase_	,	add_special_tokens=lowerCAmelCase_						)
				self.assertListEqual(lowerCAmelCase_	,	lowerCAmelCase_						)

				UpperCAmelCase_     : Optional[Any]			     =					self.get_rust_tokenizer()
				UpperCAmelCase_     : Union[str, Any]			     =					tokenizer.encode(lowerCAmelCase_						)
				UpperCAmelCase_     : Union[str, Any]			     =					rust_tokenizer.encode(lowerCAmelCase_						)
				self.assertListEqual(lowerCAmelCase_	,	lowerCAmelCase_						)





	def      _SCREAMING_SNAKE_CASE      (      self					: Any						)						->			str:
				UpperCAmelCase_     : Optional[Any]			     =					BasicTokenizer()

				self.assertListEqual(tokenizer.tokenize("ah\u535A\u63A8zz"						)	,	["ah", "\u535A", "\u63A8", "zz"]						)





	def      _SCREAMING_SNAKE_CASE      (      self					: List[str]						)						->			Any:
				UpperCAmelCase_     : Optional[Any]			     =					BasicTokenizer(do_lower_case=lowerCAmelCase_						)

				self.assertListEqual(
				    tokenizer.tokenize(" \tHeLLo!how  \n Are yoU?  "						)	,	["hello", "!", "how", "are", "you", "?"]						)
				self.assertListEqual(tokenizer.tokenize("H\u00E9llo"						)	,	["hello"]						)





	def      _SCREAMING_SNAKE_CASE      (      self					: Tuple						)						->			Any:
				UpperCAmelCase_     : List[str]			     =					BasicTokenizer(do_lower_case=lowerCAmelCase_	,	strip_accents=lowerCAmelCase_						)

				self.assertListEqual(
				    tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "						)	,	["hällo", "!", "how", "are", "you", "?"]						)
				self.assertListEqual(tokenizer.tokenize("H\u00E9llo"						)	,	["h\u00E9llo"]						)





	def      _SCREAMING_SNAKE_CASE      (      self					: int						)						->			int:
				UpperCAmelCase_     : Any			     =					BasicTokenizer(do_lower_case=lowerCAmelCase_	,	strip_accents=lowerCAmelCase_						)

				self.assertListEqual(
				    tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "						)	,	["hallo", "!", "how", "are", "you", "?"]						)
				self.assertListEqual(tokenizer.tokenize("H\u00E9llo"						)	,	["hello"]						)





	def      _SCREAMING_SNAKE_CASE      (      self					: List[str]						)						->			Optional[int]:
				UpperCAmelCase_     : Tuple			     =					BasicTokenizer(do_lower_case=lowerCAmelCase_						)

				self.assertListEqual(
				    tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "						)	,	["hallo", "!", "how", "are", "you", "?"]						)
				self.assertListEqual(tokenizer.tokenize("H\u00E9llo"						)	,	["hello"]						)





	def      _SCREAMING_SNAKE_CASE      (      self					: Union[str, Any]						)						->			str:
				UpperCAmelCase_     : List[str]			     =					BasicTokenizer(do_lower_case=lowerCAmelCase_						)

				self.assertListEqual(
				    tokenizer.tokenize(" \tHeLLo!how  \n Are yoU?  "						)	,	["HeLLo", "!", "how", "Are", "yoU", "?"]						)





	def      _SCREAMING_SNAKE_CASE      (      self					: Optional[Any]						)						->			List[Any]:
				UpperCAmelCase_     : Optional[int]			     =					BasicTokenizer(do_lower_case=lowerCAmelCase_	,	strip_accents=lowerCAmelCase_						)

				self.assertListEqual(
				    tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "						)	,	["HäLLo", "!", "how", "Are", "yoU", "?"]						)





	def      _SCREAMING_SNAKE_CASE      (      self					: str						)						->			Optional[Any]:
				UpperCAmelCase_     : Union[str, Any]			     =					BasicTokenizer(do_lower_case=lowerCAmelCase_	,	strip_accents=lowerCAmelCase_						)

				self.assertListEqual(
				    tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "						)	,	["HaLLo", "!", "how", "Are", "yoU", "?"]						)





	def      _SCREAMING_SNAKE_CASE      (      self					: Union[str, Any]						)						->			int:
				UpperCAmelCase_     : Tuple			     =					BasicTokenizer(do_lower_case=lowerCAmelCase_	,	never_split=["[UNK]"]						)

				self.assertListEqual(
				    tokenizer.tokenize(" \tHeLLo!how  \n Are yoU? [UNK]"						)	,	["HeLLo", "!", "how", "Are", "yoU", "?", "[UNK]"]						)





	def      _SCREAMING_SNAKE_CASE      (      self					: Optional[Any]						)						->			Any:
				UpperCAmelCase_     : Tuple			     =					BasicTokenizer()
				UpperCAmelCase_     : Dict			     =					"a\n'll !!to?'d of, can't."
				UpperCAmelCase_     : List[str]			     =					["a", "'", "ll", "!", "!", "to", "?", "'", "d", "of", ",", "can", "'", "t", "."]
				self.assertListEqual(tokenizer.tokenize(lowerCAmelCase_						)	,	lowerCAmelCase_						)





	def      _SCREAMING_SNAKE_CASE      (      self					: Dict						)						->			int:
				UpperCAmelCase_     : int			     =					["[UNK]", "[CLS]", "[SEP]", "want", "##want", "##ed", "wa", "un", "runn", "##ing"]

				UpperCAmelCase_     : Tuple			     =					{}
				for i, token in enumerate(lowerCAmelCase_						):
							UpperCAmelCase_     : Optional[int]			     =					i
				UpperCAmelCase_     : Optional[Any]			     =					WordpieceTokenizer(vocab=lowerCAmelCase_	,	unk_token="[UNK]"						)

				self.assertListEqual(tokenizer.tokenize(""						)	,	[]						)

				self.assertListEqual(tokenizer.tokenize("unwanted running"						)	,	["un", "##want", "##ed", "runn", "##ing"]						)

				self.assertListEqual(tokenizer.tokenize("unwantedX running"						)	,	["[UNK]", "runn", "##ing"]						)





	def      _SCREAMING_SNAKE_CASE      (      self					: Dict						)						->			Optional[int]:
				self.assertTrue(_is_whitespace(" "						)						)
				self.assertTrue(_is_whitespace("\t"						)						)
				self.assertTrue(_is_whitespace("\r"						)						)
				self.assertTrue(_is_whitespace("\n"						)						)
				self.assertTrue(_is_whitespace("\u00A0"						)						)

				self.assertFalse(_is_whitespace("A"						)						)
				self.assertFalse(_is_whitespace("-"						)						)





	def      _SCREAMING_SNAKE_CASE      (      self					: Any						)						->			Optional[int]:
				self.assertTrue(_is_control("\u0005"						)						)

				self.assertFalse(_is_control("A"						)						)
				self.assertFalse(_is_control(" "						)						)
				self.assertFalse(_is_control("\t"						)						)
				self.assertFalse(_is_control("\r"						)						)





	def      _SCREAMING_SNAKE_CASE      (      self					: int						)						->			List[str]:
				self.assertTrue(_is_punctuation("-"						)						)
				self.assertTrue(_is_punctuation("$"						)						)
				self.assertTrue(_is_punctuation("`"						)						)
				self.assertTrue(_is_punctuation("."						)						)

				self.assertFalse(_is_punctuation("A"						)						)
				self.assertFalse(_is_punctuation(" "						)						)





	def      _SCREAMING_SNAKE_CASE      (      self					: str						)						->			Optional[Any]:
				UpperCAmelCase_     : Dict			     =					self.get_tokenizer()
				UpperCAmelCase_     : List[str]			     =					self.get_rust_tokenizer()

				# Example taken from the issue https://github.com/huggingface/tokenizers/issues/340
				self.assertListEqual([tokenizer.tokenize(lowerCAmelCase_						) for t in ["Test", "\xad", "test"]]	,	[["[UNK]"], [], ["[UNK]"]]						)

				self.assertListEqual(
				    [rust_tokenizer.tokenize(lowerCAmelCase_						) for t in ["Test", "\xad", "test"]]	,	[["[UNK]"], [], ["[UNK]"]]						)





	@slow
	def      _SCREAMING_SNAKE_CASE      (      self					: Optional[Any]						)						->			Optional[Any]:
				UpperCAmelCase_     : List[str]			     =					self.tokenizer_class.from_pretrained("bert-base-uncased"						)

				UpperCAmelCase_     : Any			     =					tokenizer.encode("sequence builders"	,	add_special_tokens=lowerCAmelCase_						)
				UpperCAmelCase_     : Union[str, Any]			     =					tokenizer.encode("multi-sequence build"	,	add_special_tokens=lowerCAmelCase_						)

				UpperCAmelCase_     : Tuple			     =					tokenizer.build_inputs_with_special_tokens(lowerCAmelCase_						)
				UpperCAmelCase_     : Dict			     =					tokenizer.build_inputs_with_special_tokens(lowerCAmelCase_	,	lowerCAmelCase_						)

				assert encoded_sentence == [101] + text + [102]
				assert encoded_pair == [101] + text + [102] + text_a + [102]





	def      _SCREAMING_SNAKE_CASE      (      self					: Any						)						->			str:
				for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
							with self.subTest(f"""{tokenizer.__class__.__name__} ({pretrained_name})"""						):
										UpperCAmelCase_     : str			     =					self.rust_tokenizer_class.from_pretrained(lowerCAmelCase_	,	**lowerCAmelCase_						)

										UpperCAmelCase_     : Union[str, Any]			     =					f"""A, naïve {tokenizer_r.mask_token} AllenNLP sentence."""
										UpperCAmelCase_     : Tuple			     =					tokenizer_r.encode_plus(
										    lowerCAmelCase_	,	return_attention_mask=lowerCAmelCase_	,	return_token_type_ids=lowerCAmelCase_	,	return_offsets_mapping=lowerCAmelCase_	,	add_special_tokens=lowerCAmelCase_	,	)

										UpperCAmelCase_     : Optional[int]			     =					tokenizer_r.do_lower_case if hasattr(lowerCAmelCase_	,	"do_lower_case"						) else False
										UpperCAmelCase_     : List[Any]			     =					(
										    [
										        ((0, 0), tokenizer_r.cls_token),
										        ((0, 1), "A"),
										        ((1, 2), ","),
										        ((3, 5), "na"),
										        ((5, 6), "##ï"),
										        ((6, 8), "##ve"),
										        ((9, 15), tokenizer_r.mask_token),
										        ((16, 21), "Allen"),
										        ((21, 23), "##NL"),
										        ((23, 24), "##P"),
										        ((25, 33), "sentence"),
										        ((33, 34), "."),
										        ((0, 0), tokenizer_r.sep_token),
										    ]
										    if not do_lower_case
										    else [
										        ((0, 0), tokenizer_r.cls_token),
										        ((0, 1), "a"),
										        ((1, 2), ","),
										        ((3, 8), "naive"),
										        ((9, 15), tokenizer_r.mask_token),
										        ((16, 21), "allen"),
										        ((21, 23), "##nl"),
										        ((23, 24), "##p"),
										        ((25, 33), "sentence"),
										        ((33, 34), "."),
										        ((0, 0), tokenizer_r.sep_token),
										    ]
										)

										self.assertEqual(
										    [e[1] for e in expected_results]	,	tokenizer_r.convert_ids_to_tokens(tokens["input_ids"]						)						)
										self.assertEqual([e[0] for e in expected_results]	,	tokens["offset_mapping"]						)





	def      _SCREAMING_SNAKE_CASE      (      self					: Union[str, Any]						)						->			Optional[Any]:
				UpperCAmelCase_     : List[Any]			     =					["的", "人", "有"]
				UpperCAmelCase_     : Tuple			     =					"".join(lowerCAmelCase_						)
				for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
							with self.subTest(f"""{tokenizer.__class__.__name__} ({pretrained_name})"""						):
										UpperCAmelCase_     : Optional[Any]			     =					True
										UpperCAmelCase_     : Any			     =					self.tokenizer_class.from_pretrained(lowerCAmelCase_	,	**lowerCAmelCase_						)
										UpperCAmelCase_     : Optional[Any]			     =					self.rust_tokenizer_class.from_pretrained(lowerCAmelCase_	,	**lowerCAmelCase_						)

										UpperCAmelCase_     : Dict			     =					tokenizer_p.encode(lowerCAmelCase_	,	add_special_tokens=lowerCAmelCase_						)
										UpperCAmelCase_     : Dict			     =					tokenizer_r.encode(lowerCAmelCase_	,	add_special_tokens=lowerCAmelCase_						)

										UpperCAmelCase_     : Union[str, Any]			     =					tokenizer_r.convert_ids_to_tokens(lowerCAmelCase_						)
										UpperCAmelCase_     : Any			     =					tokenizer_p.convert_ids_to_tokens(lowerCAmelCase_						)

										# it is expected that each Chinese character is not preceded by "##"
										self.assertListEqual(lowerCAmelCase_	,	lowerCAmelCase_						)
										self.assertListEqual(lowerCAmelCase_	,	lowerCAmelCase_						)

										UpperCAmelCase_     : List[Any]			     =					False
										UpperCAmelCase_     : Optional[Any]			     =					self.rust_tokenizer_class.from_pretrained(lowerCAmelCase_	,	**lowerCAmelCase_						)
										UpperCAmelCase_     : List[Any]			     =					self.tokenizer_class.from_pretrained(lowerCAmelCase_	,	**lowerCAmelCase_						)

										UpperCAmelCase_     : int			     =					tokenizer_r.encode(lowerCAmelCase_	,	add_special_tokens=lowerCAmelCase_						)
										UpperCAmelCase_     : List[str]			     =					tokenizer_p.encode(lowerCAmelCase_	,	add_special_tokens=lowerCAmelCase_						)

										UpperCAmelCase_     : Union[str, Any]			     =					tokenizer_r.convert_ids_to_tokens(lowerCAmelCase_						)
										UpperCAmelCase_     : List[str]			     =					tokenizer_p.convert_ids_to_tokens(lowerCAmelCase_						)

										# it is expected that only the first Chinese character is not preceded by "##".
										UpperCAmelCase_     : Tuple			     =					[
										    f"""##{token}""" if idx != 0 else token for idx, token in enumerate(lowerCAmelCase_						)
										]
										self.assertListEqual(lowerCAmelCase_	,	lowerCAmelCase_						)
										self.assertListEqual(lowerCAmelCase_	,	lowerCAmelCase_						)



