
from __future__ import annotations

import unittest

from transformers import BlenderbotSmallConfig, BlenderbotSmallTokenizer, is_tf_available
from transformers.testing_utils import require_tf, require_tokenizers, slow
from transformers.utils import cached_property

from ...test_configuration_common import ConfigTester
from ...test_modeling_tf_common import TFModelTesterMixin, ids_tensor
from ...test_pipeline_mixin import PipelineTesterMixin


if is_tf_available():
		import tensorflow as tf

		from transformers import TFAutoModelForSeqaSeqLM, TFBlenderbotSmallForConditionalGeneration, TFBlenderbotSmallModel




@require_tf
class  lowerCamelCase__							:




	'''simple docstring'''




	_A								=     BlenderbotSmallConfig
	_A								=     {}
	_A								=     'gelu'







	def __init__(					self		:Dict     ,   a		:Dict     ,   a		:Union[str, Any]=1_3     ,   a		:List[str]=7     ,   a		:List[Any]=True     ,   a		:List[str]=False     ,   a		:List[str]=9_9     ,   a		:Optional[Any]=3_2     ,   a		:Dict=2     ,   a		:Optional[Any]=4     ,   a		:Optional[Any]=3_7     ,   a		:Dict=0.1     ,   a		:str=0.1     ,   a		:Dict=2_0     ,   a		:Optional[Any]=2     ,   a		:str=1     ,   a		:Dict=0     ,   )		-> Optional[int]:
								__UpperCamelCase	:   Tuple			   =					parent
								__UpperCamelCase	:   Dict			   =					batch_size
								__UpperCamelCase	:   Union[str, Any]			   =					seq_length
								__UpperCamelCase	:   Union[str, Any]			   =					is_training
								__UpperCamelCase	:   Dict			   =					use_labels
								__UpperCamelCase	:   List[str]			   =					vocab_size
								__UpperCamelCase	:   List[Any]			   =					hidden_size
								__UpperCamelCase	:   Tuple			   =					num_hidden_layers
								__UpperCamelCase	:   str			   =					num_attention_heads
								__UpperCamelCase	:   Dict			   =					intermediate_size

								__UpperCamelCase	:   List[str]			   =					hidden_dropout_prob
								__UpperCamelCase	:   Optional[Any]			   =					attention_probs_dropout_prob
								__UpperCamelCase	:   Dict			   =					max_position_embeddings
								__UpperCamelCase	:   Tuple			   =					eos_token_id
								__UpperCamelCase	:   Any			   =					pad_token_id
								__UpperCamelCase	:   Dict			   =					bos_token_id







	def    _lowerCamelCase				(					self		:List[str]						)		-> str:
								__UpperCamelCase	:   str			   =					ids_tensor([self.batch_size, self.seq_length - 1]     ,   self.vocab_size						)
								__UpperCamelCase	:   Optional[int]			   =					tf.expand_dims(tf.constant([self.eos_token_id] * self.batch_size						)     ,   1						)
								__UpperCamelCase	:   Dict			   =					tf.concat([input_ids, eos_tensor]     ,   axis=1						)

								__UpperCamelCase	:   List[str]			   =					ids_tensor([self.batch_size, self.seq_length]     ,   self.vocab_size						)

								__UpperCamelCase	:   Union[str, Any]			   =					self.config_cls(
								    vocab_size=self.vocab_size     ,   d_model=self.hidden_size     ,   encoder_layers=self.num_hidden_layers     ,   decoder_layers=self.num_hidden_layers     ,   encoder_attention_heads=self.num_attention_heads     ,   decoder_attention_heads=self.num_attention_heads     ,   encoder_ffn_dim=self.intermediate_size     ,   decoder_ffn_dim=self.intermediate_size     ,   dropout=self.hidden_dropout_prob     ,   attention_dropout=self.attention_probs_dropout_prob     ,   max_position_embeddings=self.max_position_embeddings     ,   eos_token_ids=[2]     ,   bos_token_id=self.bos_token_id     ,   pad_token_id=self.pad_token_id     ,   decoder_start_token_id=self.pad_token_id     ,   **self.config_updates     ,   )
								__UpperCamelCase	:   Dict			   =					prepare_blenderbot_small_inputs_dict(lowerCAmelCase_     ,   lowerCAmelCase_     ,   lowerCAmelCase_						)
								return config, inputs_dict







	def    _lowerCamelCase				(					self		:Optional[Any]     ,   a		:Optional[Any]     ,   a		:Dict						)		-> List[Any]:
								__UpperCamelCase	:   List[Any]			   =					TFBlenderbotSmallModel(config=lowerCAmelCase_						).get_decoder()
								__UpperCamelCase	:   Union[str, Any]			   =					inputs_dict["input_ids"]

								__UpperCamelCase	:   Any			   =					input_ids[:1, :]
								__UpperCamelCase	:   Dict			   =					inputs_dict["attention_mask"][:1, :]
								__UpperCamelCase	:   Optional[int]			   =					inputs_dict["head_mask"]
								__UpperCamelCase	:   Union[str, Any]			   =					1

								# first forward pass
								__UpperCamelCase	:   str			   =					model(lowerCAmelCase_     ,   attention_mask=lowerCAmelCase_     ,   head_mask=lowerCAmelCase_     ,   use_cache=lowerCAmelCase_						)

								__UpperCamelCase	:   Tuple			   =					outputs.to_tuple()

								# create hypothetical next token and extent to next_input_ids
								__UpperCamelCase	:   Dict			   =					ids_tensor((self.batch_size, 3)     ,   config.vocab_size						)
								__UpperCamelCase	:   Union[str, Any]			   =					tf.cast(ids_tensor((self.batch_size, 3)     ,   2						)     ,   tf.inta						)

								# append to next input_ids and
								__UpperCamelCase	:   List[Any]			   =					tf.concat([input_ids, next_tokens]     ,   axis=-1						)
								__UpperCamelCase	:   Union[str, Any]			   =					tf.concat([attention_mask, next_attn_mask]     ,   axis=-1						)

								__UpperCamelCase	:   Any			   =					model(lowerCAmelCase_     ,   attention_mask=lowerCAmelCase_						)[0]
								__UpperCamelCase	:   int			   =					model(lowerCAmelCase_     ,   attention_mask=lowerCAmelCase_     ,   past_key_values=lowerCAmelCase_						)[0]

								self.parent.assertEqual(next_tokens.shape[1]     ,   output_from_past.shape[1]						)

								# select random slice
								__UpperCamelCase	:   Tuple			   =					int(ids_tensor((1,)     ,   output_from_past.shape[-1]						)						)
								__UpperCamelCase	:   List[Any]			   =					output_from_no_past[:, -3:, random_slice_idx]
								__UpperCamelCase	:   Union[str, Any]			   =					output_from_past[:, :, random_slice_idx]

								# test that outputs are equal for slice
								tf.debugging.assert_near(lowerCAmelCase_     ,   lowerCAmelCase_     ,   rtol=1E-3						)




def 	_SCREAMING_SNAKE_CASE					(		_lowerCamelCase :			Any						,  _lowerCamelCase :			str						,  _lowerCamelCase :			Tuple						,  _lowerCamelCase :			List[str]=None						,  _lowerCamelCase :			Union[str, Any]=None						,  _lowerCamelCase :			Optional[Any]=None						,  _lowerCamelCase :			List[Any]=None						,  _lowerCamelCase :			List[Any]=None						,  )					->    List[Any]:






							'''simple docstring'''





							if attention_mask is None:
														__UpperCamelCase	:   str			   =					tf.cast(tf.math.not_equal(A__						,  config.pad_token_id)						,  tf.inta)
							if decoder_attention_mask is None:
														__UpperCamelCase	:   Optional[int]			   =					tf.concat(
														    [
														        tf.ones(decoder_input_ids[:, :1].shape						,  dtype=tf.inta),
														        tf.cast(tf.math.not_equal(decoder_input_ids[:, 1:]						,  config.pad_token_id)						,  tf.inta),
														    ]						,  axis=-1						,  )
							if head_mask is None:
														__UpperCamelCase	:   str			   =					tf.ones((config.encoder_layers, config.encoder_attention_heads))
							if decoder_head_mask is None:
														__UpperCamelCase	:   str			   =					tf.ones((config.decoder_layers, config.decoder_attention_heads))
							if cross_attn_head_mask is None:
														__UpperCamelCase	:   Optional[Any]			   =					tf.ones((config.decoder_layers, config.decoder_attention_heads))
							return {
							    "input_ids": input_ids,
							    "decoder_input_ids": decoder_input_ids,
							    "attention_mask": attention_mask,
							    "decoder_attention_mask": decoder_attention_mask,
							    "head_mask": head_mask,
							    "decoder_head_mask": decoder_head_mask,
							    "cross_attn_head_mask": cross_attn_head_mask,
							}




@require_tf
class  lowerCamelCase__							(				__A     ,						__A     ,						unittest.TestCase):




	'''simple docstring'''




	_A								=     (
	    (TFBlenderbotSmallForConditionalGeneration, TFBlenderbotSmallModel) if is_tf_available() else ()
	)
	_A								=     (TFBlenderbotSmallForConditionalGeneration,) if is_tf_available() else ()
	_A								=     (
	    {
	        'conversational': TFBlenderbotSmallForConditionalGeneration,
	        'feature-extraction': TFBlenderbotSmallModel,
	        'summarization': TFBlenderbotSmallForConditionalGeneration,
	        'text2text-generation': TFBlenderbotSmallForConditionalGeneration,
	        'translation': TFBlenderbotSmallForConditionalGeneration,
	    }
	    if is_tf_available()
	    else {}
	)
	_A								=     True
	_A								=     False
	_A								=     False







	def    _lowerCamelCase				(					self		:List[str]						)		-> List[Any]:
								__UpperCamelCase	:   Optional[int]			   =					TFBlenderbotSmallModelTester(self						)
								__UpperCamelCase	:   str			   =					ConfigTester(self     ,   config_class=lowerCAmelCase_						)







	def    _lowerCamelCase				(					self		:int						)		-> str:
								self.config_tester.run_common_tests()







	def    _lowerCamelCase				(					self		:Any						)		-> Optional[int]:
								__UpperCamelCase	:   List[str]			   =					self.model_tester.prepare_config_and_inputs_for_common()
								self.model_tester.check_decoder_model_past_large_inputs(*lowerCAmelCase_						)




@require_tokenizers
@require_tf
class  lowerCamelCase__							(				unittest.TestCase):




	'''simple docstring'''




	_A								=     [
	    'Social anxiety\nWow, I am never shy. Do you have anxiety?\nYes. I end up sweating and blushing and feel like '
	    '  i\'m going to throw up.\nand why is that?'
	]
	_A								=     'facebook/blenderbot_small-90M'







	@cached_property
	def    _lowerCamelCase				(					self		:str						)		-> Optional[Any]:
								# use "old" tokenizer here because of bug when downloading new tokenizer
								return BlenderbotSmallTokenizer.from_pretrained("facebook/blenderbot-90M"						)







	@cached_property
	def    _lowerCamelCase				(					self		:List[str]						)		-> Optional[Any]:
								__UpperCamelCase	:   str			   =					TFAutoModelForSeqaSeqLM.from_pretrained(self.model_name						)
								return model







	@slow
	def    _lowerCamelCase				(					self		:str						)		-> str:
								__UpperCamelCase	:   Dict			   =					self.tokenizer(self.src_text     ,   return_tensors="tf"						)
								__UpperCamelCase	:   List[str]			   =					self.model.generate(
								    model_inputs.input_ids     ,   attention_mask=model_inputs.attention_mask     ,   num_beams=2     ,   use_cache=lowerCAmelCase_     ,   )
								__UpperCamelCase	:   Dict			   =					self.tokenizer.batch_decode(generated_ids.numpy()     ,   skip_special_tokens=lowerCAmelCase_						)[0]
								assert generated_words in (
								    "i don't know. i just feel like i'm going to throw up. it's not fun.",
								    "i'm not sure. i just feel like i've been feeling like i have to be in a certain place",
								    "i'm not sure. i just feel like i've been in a bad situation.",
								)







"""simple docstring"""
import os
import unittest

from transformers import BertTokenizerFast
from transformers.models.bert.tokenization_bert import (
    VOCAB_FILES_NAMES,
    BasicTokenizer,
    BertTokenizer,
    WordpieceTokenizer,
    _is_control,
    _is_punctuation,
    _is_whitespace,
)
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin, filter_non_english







@require_tokenizers
class 			UpperCamelCase_       (__A				,						unittest.TestCase     ):
	__magic_name__	      =				BertTokenizer
	__magic_name__	      =				BertTokenizerFast
	__magic_name__	      =				True
	__magic_name__	      =				True
	__magic_name__	      =				filter_non_english





	def      _SCREAMING_SNAKE_CASE      (      self					: Tuple						)						->			List[Any]:
				super().setUp()

				UpperCAmelCase_     : Tuple			     =					[
				    "[UNK]",
				    "[CLS]",
				    "[SEP]",
				    "[PAD]",
				    "[MASK]",
				    "want",
				    "##want",
				    "##ed",
				    "wa",
				    "un",
				    "runn",
				    "##ing",
				    ",",
				    "low",
				    "lowest",
				]
				UpperCAmelCase_     : Any			     =					os.path.join(self.tmpdirname	,	VOCAB_FILES_NAMES["vocab_file"]						)
				with open(self.vocab_file	,	"w"	,	encoding="utf-8"						) as vocab_writer:
							vocab_writer.write("".join([x + "\n" for x in vocab_tokens]						)						)





	def      _SCREAMING_SNAKE_CASE      (      self					: Any	,	lowerCAmelCase_					: str						)						->			Union[str, Any]:
				UpperCAmelCase_     : Tuple			     =					"UNwant\u00E9d,running"
				UpperCAmelCase_     : Any			     =					"unwanted, running"
				return input_text, output_text





	def      _SCREAMING_SNAKE_CASE      (      self					: Optional[Any]						)						->			List[str]:
				UpperCAmelCase_     : Any			     =					self.tokenizer_class(self.vocab_file						)

				UpperCAmelCase_     : Tuple			     =					tokenizer.tokenize("UNwant\u00E9d,running"						)
				self.assertListEqual(lowerCAmelCase_	,	["un", "##want", "##ed", ",", "runn", "##ing"]						)
				self.assertListEqual(tokenizer.convert_tokens_to_ids(lowerCAmelCase_						)	,	[9, 6, 7, 12, 10, 11]						)





	def      _SCREAMING_SNAKE_CASE      (      self					: List[str]						)						->			Any:
				if not self.test_rust_tokenizer:
							return

				UpperCAmelCase_     : Union[str, Any]			     =					self.get_tokenizer()
				UpperCAmelCase_     : List[Any]			     =					self.get_rust_tokenizer()

				UpperCAmelCase_     : List[Any]			     =					"UNwant\u00E9d,running"

				UpperCAmelCase_     : Union[str, Any]			     =					tokenizer.tokenize(lowerCAmelCase_						)
				UpperCAmelCase_     : int			     =					rust_tokenizer.tokenize(lowerCAmelCase_						)
				self.assertListEqual(lowerCAmelCase_	,	lowerCAmelCase_						)

				UpperCAmelCase_     : Any			     =					tokenizer.encode(lowerCAmelCase_	,	add_special_tokens=lowerCAmelCase_						)
				UpperCAmelCase_     : Optional[int]			     =					rust_tokenizer.encode(lowerCAmelCase_	,	add_special_tokens=lowerCAmelCase_						)
				self.assertListEqual(lowerCAmelCase_	,	lowerCAmelCase_						)

				UpperCAmelCase_     : Optional[Any]			     =					self.get_rust_tokenizer()
				UpperCAmelCase_     : Dict			     =					tokenizer.encode(lowerCAmelCase_						)
				UpperCAmelCase_     : Optional[Any]			     =					rust_tokenizer.encode(lowerCAmelCase_						)
				self.assertListEqual(lowerCAmelCase_	,	lowerCAmelCase_						)

				# With lower casing
				UpperCAmelCase_     : Tuple			     =					self.get_tokenizer(do_lower_case=lowerCAmelCase_						)
				UpperCAmelCase_     : Optional[int]			     =					self.get_rust_tokenizer(do_lower_case=lowerCAmelCase_						)

				UpperCAmelCase_     : List[Any]			     =					"UNwant\u00E9d,running"

				UpperCAmelCase_     : List[Any]			     =					tokenizer.tokenize(lowerCAmelCase_						)
				UpperCAmelCase_     : Tuple			     =					rust_tokenizer.tokenize(lowerCAmelCase_						)
				self.assertListEqual(lowerCAmelCase_	,	lowerCAmelCase_						)

				UpperCAmelCase_     : Union[str, Any]			     =					tokenizer.encode(lowerCAmelCase_	,	add_special_tokens=lowerCAmelCase_						)
				UpperCAmelCase_     : Union[str, Any]			     =					rust_tokenizer.encode(lowerCAmelCase_	,	add_special_tokens=lowerCAmelCase_						)
				self.assertListEqual(lowerCAmelCase_	,	lowerCAmelCase_						)

				UpperCAmelCase_     : Optional[Any]			     =					self.get_rust_tokenizer()
				UpperCAmelCase_     : Union[str, Any]			     =					tokenizer.encode(lowerCAmelCase_						)
				UpperCAmelCase_     : Union[str, Any]			     =					rust_tokenizer.encode(lowerCAmelCase_						)
				self.assertListEqual(lowerCAmelCase_	,	lowerCAmelCase_						)





	def      _SCREAMING_SNAKE_CASE      (      self					: Any						)						->			str:
				UpperCAmelCase_     : Optional[Any]			     =					BasicTokenizer()

				self.assertListEqual(tokenizer.tokenize("ah\u535A\u63A8zz"						)	,	["ah", "\u535A", "\u63A8", "zz"]						)





	def      _SCREAMING_SNAKE_CASE      (      self					: List[str]						)						->			Any:
				UpperCAmelCase_     : Optional[Any]			     =					BasicTokenizer(do_lower_case=lowerCAmelCase_						)

				self.assertListEqual(
				    tokenizer.tokenize(" \tHeLLo!how  \n Are yoU?  "						)	,	["hello", "!", "how", "are", "you", "?"]						)
				self.assertListEqual(tokenizer.tokenize("H\u00E9llo"						)	,	["hello"]						)





	def      _SCREAMING_SNAKE_CASE      (      self					: Tuple						)						->			Any:
				UpperCAmelCase_     : List[str]			     =					BasicTokenizer(do_lower_case=lowerCAmelCase_	,	strip_accents=lowerCAmelCase_						)

				self.assertListEqual(
				    tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "						)	,	["hällo", "!", "how", "are", "you", "?"]						)
				self.assertListEqual(tokenizer.tokenize("H\u00E9llo"						)	,	["h\u00E9llo"]						)





	def      _SCREAMING_SNAKE_CASE      (      self					: int						)						->			int:
				UpperCAmelCase_     : Any			     =					BasicTokenizer(do_lower_case=lowerCAmelCase_	,	strip_accents=lowerCAmelCase_						)

				self.assertListEqual(
				    tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "						)	,	["hallo", "!", "how", "are", "you", "?"]						)
				self.assertListEqual(tokenizer.tokenize("H\u00E9llo"						)	,	["hello"]						)





	def      _SCREAMING_SNAKE_CASE      (      self					: List[str]						)						->			Optional[int]:
				UpperCAmelCase_     : Tuple			     =					BasicTokenizer(do_lower_case=lowerCAmelCase_						)

				self.assertListEqual(
				    tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "						)	,	["hallo", "!", "how", "are", "you", "?"]						)
				self.assertListEqual(tokenizer.tokenize("H\u00E9llo"						)	,	["hello"]						)





	def      _SCREAMING_SNAKE_CASE      (      self					: Union[str, Any]						)						->			str:
				UpperCAmelCase_     : List[str]			     =					BasicTokenizer(do_lower_case=lowerCAmelCase_						)

				self.assertListEqual(
				    tokenizer.tokenize(" \tHeLLo!how  \n Are yoU?  "						)	,	["HeLLo", "!", "how", "Are", "yoU", "?"]						)





	def      _SCREAMING_SNAKE_CASE      (      self					: Optional[Any]						)						->			List[Any]:
				UpperCAmelCase_     : Optional[int]			     =					BasicTokenizer(do_lower_case=lowerCAmelCase_	,	strip_accents=lowerCAmelCase_						)

				self.assertListEqual(
				    tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "						)	,	["HäLLo", "!", "how", "Are", "yoU", "?"]						)





	def      _SCREAMING_SNAKE_CASE      (      self					: str						)						->			Optional[Any]:
				UpperCAmelCase_     : Union[str, Any]			     =					BasicTokenizer(do_lower_case=lowerCAmelCase_	,	strip_accents=lowerCAmelCase_						)

				self.assertListEqual(
				    tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "						)	,	["HaLLo", "!", "how", "Are", "yoU", "?"]						)





	def      _SCREAMING_SNAKE_CASE      (      self					: Union[str, Any]						)						->			int:
				UpperCAmelCase_     : Tuple			     =					BasicTokenizer(do_lower_case=lowerCAmelCase_	,	never_split=["[UNK]"]						)

				self.assertListEqual(
				    tokenizer.tokenize(" \tHeLLo!how  \n Are yoU? [UNK]"						)	,	["HeLLo", "!", "how", "Are", "yoU", "?", "[UNK]"]						)





	def      _SCREAMING_SNAKE_CASE      (      self					: Optional[Any]						)						->			Any:
				UpperCAmelCase_     : Tuple			     =					BasicTokenizer()
				UpperCAmelCase_     : Dict			     =					"a\n'll !!to?'d of, can't."
				UpperCAmelCase_     : List[str]			     =					["a", "'", "ll", "!", "!", "to", "?", "'", "d", "of", ",", "can", "'", "t", "."]
				self.assertListEqual(tokenizer.tokenize(lowerCAmelCase_						)	,	lowerCAmelCase_						)





	def      _SCREAMING_SNAKE_CASE      (      self					: Dict						)						->			int:
				UpperCAmelCase_     : int			     =					["[UNK]", "[CLS]", "[SEP]", "want", "##want", "##ed", "wa", "un", "runn", "##ing"]

				UpperCAmelCase_     : Tuple			     =					{}
				for i, token in enumerate(lowerCAmelCase_						):
							UpperCAmelCase_     : Optional[int]			     =					i
				UpperCAmelCase_     : Optional[Any]			     =					WordpieceTokenizer(vocab=lowerCAmelCase_	,	unk_token="[UNK]"						)

				self.assertListEqual(tokenizer.tokenize(""						)	,	[]						)

				self.assertListEqual(tokenizer.tokenize("unwanted running"						)	,	["un", "##want", "##ed", "runn", "##ing"]						)

				self.assertListEqual(tokenizer.tokenize("unwantedX running"						)	,	["[UNK]", "runn", "##ing"]						)





	def      _SCREAMING_SNAKE_CASE      (      self					: Dict						)						->			Optional[int]:
				self.assertTrue(_is_whitespace(" "						)						)
				self.assertTrue(_is_whitespace("\t"						)						)
				self.assertTrue(_is_whitespace("\r"						)						)
				self.assertTrue(_is_whitespace("\n"						)						)
				self.assertTrue(_is_whitespace("\u00A0"						)						)

				self.assertFalse(_is_whitespace("A"						)						)
				self.assertFalse(_is_whitespace("-"						)						)





	def      _SCREAMING_SNAKE_CASE      (      self					: Any						)						->			Optional[int]:
				self.assertTrue(_is_control("\u0005"						)						)

				self.assertFalse(_is_control("A"						)						)
				self.assertFalse(_is_control(" "						)						)
				self.assertFalse(_is_control("\t"						)						)
				self.assertFalse(_is_control("\r"						)						)





	def      _SCREAMING_SNAKE_CASE      (      self					: int						)						->			List[str]:
				self.assertTrue(_is_punctuation("-"						)						)
				self.assertTrue(_is_punctuation("$"						)						)
				self.assertTrue(_is_punctuation("`"						)						)
				self.assertTrue(_is_punctuation("."						)						)

				self.assertFalse(_is_punctuation("A"						)						)
				self.assertFalse(_is_punctuation(" "						)						)





	def      _SCREAMING_SNAKE_CASE      (      self					: str						)						->			Optional[Any]:
				UpperCAmelCase_     : Dict			     =					self.get_tokenizer()
				UpperCAmelCase_     : List[str]			     =					self.get_rust_tokenizer()

				# Example taken from the issue https://github.com/huggingface/tokenizers/issues/340
				self.assertListEqual([tokenizer.tokenize(lowerCAmelCase_						) for t in ["Test", "\xad", "test"]]	,	[["[UNK]"], [], ["[UNK]"]]						)

				self.assertListEqual(
				    [rust_tokenizer.tokenize(lowerCAmelCase_						) for t in ["Test", "\xad", "test"]]	,	[["[UNK]"], [], ["[UNK]"]]						)





	@slow
	def      _SCREAMING_SNAKE_CASE      (      self					: Optional[Any]						)						->			Optional[Any]:
				UpperCAmelCase_     : List[str]			     =					self.tokenizer_class.from_pretrained("bert-base-uncased"						)

				UpperCAmelCase_     : Any			     =					tokenizer.encode("sequence builders"	,	add_special_tokens=lowerCAmelCase_						)
				UpperCAmelCase_     : Union[str, Any]			     =					tokenizer.encode("multi-sequence build"	,	add_special_tokens=lowerCAmelCase_						)

				UpperCAmelCase_     : Tuple			     =					tokenizer.build_inputs_with_special_tokens(lowerCAmelCase_						)
				UpperCAmelCase_     : Dict			     =					tokenizer.build_inputs_with_special_tokens(lowerCAmelCase_	,	lowerCAmelCase_						)

				assert encoded_sentence == [101] + text + [102]
				assert encoded_pair == [101] + text + [102] + text_a + [102]





	def      _SCREAMING_SNAKE_CASE      (      self					: Any						)						->			str:
				for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
							with self.subTest(f"""{tokenizer.__class__.__name__} ({pretrained_name})"""						):
										UpperCAmelCase_     : str			     =					self.rust_tokenizer_class.from_pretrained(lowerCAmelCase_	,	**lowerCAmelCase_						)

										UpperCAmelCase_     : Union[str, Any]			     =					f"""A, naïve {tokenizer_r.mask_token} AllenNLP sentence."""
										UpperCAmelCase_     : Tuple			     =					tokenizer_r.encode_plus(
										    lowerCAmelCase_	,	return_attention_mask=lowerCAmelCase_	,	return_token_type_ids=lowerCAmelCase_	,	return_offsets_mapping=lowerCAmelCase_	,	add_special_tokens=lowerCAmelCase_	,	)

										UpperCAmelCase_     : Optional[int]			     =					tokenizer_r.do_lower_case if hasattr(lowerCAmelCase_	,	"do_lower_case"						) else False
										UpperCAmelCase_     : List[Any]			     =					(
										    [
										        ((0, 0), tokenizer_r.cls_token),
										        ((0, 1), "A"),
										        ((1, 2), ","),
										        ((3, 5), "na"),
										        ((5, 6), "##ï"),
										        ((6, 8), "##ve"),
										        ((9, 15), tokenizer_r.mask_token),
										        ((16, 21), "Allen"),
										        ((21, 23), "##NL"),
										        ((23, 24), "##P"),
										        ((25, 33), "sentence"),
										        ((33, 34), "."),
										        ((0, 0), tokenizer_r.sep_token),
										    ]
										    if not do_lower_case
										    else [
										        ((0, 0), tokenizer_r.cls_token),
										        ((0, 1), "a"),
										        ((1, 2), ","),
										        ((3, 8), "naive"),
										        ((9, 15), tokenizer_r.mask_token),
										        ((16, 21), "allen"),
										        ((21, 23), "##nl"),
										        ((23, 24), "##p"),
										        ((25, 33), "sentence"),
										        ((33, 34), "."),
										        ((0, 0), tokenizer_r.sep_token),
										    ]
										)

										self.assertEqual(
										    [e[1] for e in expected_results]	,	tokenizer_r.convert_ids_to_tokens(tokens["input_ids"]						)						)
										self.assertEqual([e[0] for e in expected_results]	,	tokens["offset_mapping"]						)





	def      _SCREAMING_SNAKE_CASE      (      self					: Union[str, Any]						)						->			Optional[Any]:
				UpperCAmelCase_     : List[Any]			     =					["的", "人", "有"]
				UpperCAmelCase_     : Tuple			     =					"".join(lowerCAmelCase_						)
				for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
							with self.subTest(f"""{tokenizer.__class__.__name__} ({pretrained_name})"""						):
										UpperCAmelCase_     : Optional[Any]			     =					True
										UpperCAmelCase_     : Any			     =					self.tokenizer_class.from_pretrained(lowerCAmelCase_	,	**lowerCAmelCase_						)
										UpperCAmelCase_     : Optional[Any]			     =					self.rust_tokenizer_class.from_pretrained(lowerCAmelCase_	,	**lowerCAmelCase_						)

										UpperCAmelCase_     : Dict			     =					tokenizer_p.encode(lowerCAmelCase_	,	add_special_tokens=lowerCAmelCase_						)
										UpperCAmelCase_     : Dict			     =					tokenizer_r.encode(lowerCAmelCase_	,	add_special_tokens=lowerCAmelCase_						)

										UpperCAmelCase_     : Union[str, Any]			     =					tokenizer_r.convert_ids_to_tokens(lowerCAmelCase_						)
										UpperCAmelCase_     : Any			     =					tokenizer_p.convert_ids_to_tokens(lowerCAmelCase_						)

										# it is expected that each Chinese character is not preceded by "##"
										self.assertListEqual(lowerCAmelCase_	,	lowerCAmelCase_						)
										self.assertListEqual(lowerCAmelCase_	,	lowerCAmelCase_						)

										UpperCAmelCase_     : List[Any]			     =					False
										UpperCAmelCase_     : Optional[Any]			     =					self.rust_tokenizer_class.from_pretrained(lowerCAmelCase_	,	**lowerCAmelCase_						)
										UpperCAmelCase_     : List[Any]			     =					self.tokenizer_class.from_pretrained(lowerCAmelCase_	,	**lowerCAmelCase_						)

										UpperCAmelCase_     : int			     =					tokenizer_r.encode(lowerCAmelCase_	,	add_special_tokens=lowerCAmelCase_						)
										UpperCAmelCase_     : List[str]			     =					tokenizer_p.encode(lowerCAmelCase_	,	add_special_tokens=lowerCAmelCase_						)

										UpperCAmelCase_     : Union[str, Any]			     =					tokenizer_r.convert_ids_to_tokens(lowerCAmelCase_						)
										UpperCAmelCase_     : List[str]			     =					tokenizer_p.convert_ids_to_tokens(lowerCAmelCase_						)

										# it is expected that only the first Chinese character is not preceded by "##".
										UpperCAmelCase_     : Tuple			     =					[
										    f"""##{token}""" if idx != 0 else token for idx, token in enumerate(lowerCAmelCase_						)
										]
										self.assertListEqual(lowerCAmelCase_	,	lowerCAmelCase_						)
										self.assertListEqual(lowerCAmelCase_	,	lowerCAmelCase_						)



