

"""simple docstring"""





from __future__ import annotations

import unittest

from transformers import RoFormerConfig, is_tf_available
from transformers.testing_utils import require_tf, slow

from ...test_configuration_common import ConfigTester
from ...test_modeling_tf_common import TFModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_tf_available():
							import tensorflow as tf

							from transformers import (
							    TFRoFormerForCausalLM,
							    TFRoFormerForMaskedLM,
							    TFRoFormerForMultipleChoice,
							    TFRoFormerForQuestionAnswering,
							    TFRoFormerForSequenceClassification,
							    TFRoFormerForTokenClassification,
							    TFRoFormerModel,
							)
							from transformers.models.roformer.modeling_tf_roformer import (
							    TFRoFormerSelfAttention,
							    TFRoFormerSinusoidalPositionalEmbedding,
							)







class 				SCREAMING_SNAKE_CASE					:






					"""simple docstring"""



					def __init__(       self			:				Dict   ,lowercase_			:				Tuple   ,lowercase_			:				List[str]=1_3   ,lowercase_			:				int=7   ,lowercase_			:				Optional[Any]=True   ,lowercase_			:				Any=True   ,lowercase_			:				int=True   ,lowercase_			:				Any=True   ,lowercase_			:				str=9_9   ,lowercase_			:				Union[str, Any]=3_2   ,lowercase_			:				str=2   ,lowercase_			:				Union[str, Any]=4   ,lowercase_			:				Optional[int]=3_7   ,lowercase_			:				Any="gelu"   ,lowercase_			:				str=0.1   ,lowercase_			:				Dict=0.1   ,lowercase_			:				Optional[int]=5_1_2   ,lowercase_			:				Optional[Any]=1_6   ,lowercase_			:				Optional[Any]=2   ,lowercase_			:				int=0.02   ,lowercase_			:				Union[str, Any]=3   ,lowercase_			:				Optional[Any]=4   ,lowercase_			:				Dict=None   ,):
						lowerCAmelCase__	:						Tuple  				=	parent
						lowerCAmelCase__	:						Dict  				=	1_3
						lowerCAmelCase__	:						List[str]  				=	7
						lowerCAmelCase__	:						Union[str, Any]  				=	True
						lowerCAmelCase__	:						Any  				=	True
						lowerCAmelCase__	:						Optional[Any]  				=	True
						lowerCAmelCase__	:						Optional[int]  				=	True
						lowerCAmelCase__	:						List[Any]  				=	9_9
						lowerCAmelCase__	:						Any  				=	3_2
						lowerCAmelCase__	:						Optional[int]  				=	2
						lowerCAmelCase__	:						Optional[Any]  				=	4
						lowerCAmelCase__	:						Any  				=	3_7
						lowerCAmelCase__	:						Tuple  				=	'''gelu'''
						lowerCAmelCase__	:						Any  				=	0.1
						lowerCAmelCase__	:						Union[str, Any]  				=	0.1
						lowerCAmelCase__	:						List[Any]  				=	5_1_2
						lowerCAmelCase__	:						str  				=	1_6
						lowerCAmelCase__	:						Union[str, Any]  				=	2
						lowerCAmelCase__	:						List[Any]  				=	0.02
						lowerCAmelCase__	:						Optional[Any]  				=	3
						lowerCAmelCase__	:						Union[str, Any]  				=	4
						lowerCAmelCase__	:						Optional[int]  				=	None







					def        __lowerCAmelCase (       self			:				Optional[int]		):
						lowerCAmelCase__	:						Tuple  				=	ids_tensor([self.batch_size, self.seq_length]   ,self.vocab_size		)

						lowerCAmelCase__	:						List[Any]  				=	None
						if self.use_input_mask:
							lowerCAmelCase__	:						Any  				=	random_attention_mask([self.batch_size, self.seq_length]		)

						lowerCAmelCase__	:						List[Any]  				=	None
						if self.use_token_type_ids:
							lowerCAmelCase__	:						List[str]  				=	ids_tensor([self.batch_size, self.seq_length]   ,self.type_vocab_size		)

						lowerCAmelCase__	:						Tuple  				=	None
						lowerCAmelCase__	:						Optional[int]  				=	None
						lowerCAmelCase__	:						int  				=	None
						if self.use_labels:
							lowerCAmelCase__	:						Union[str, Any]  				=	ids_tensor([self.batch_size]   ,self.type_sequence_label_size		)
							lowerCAmelCase__	:						Union[str, Any]  				=	ids_tensor([self.batch_size, self.seq_length]   ,self.num_labels		)
							lowerCAmelCase__	:						Tuple  				=	ids_tensor([self.batch_size]   ,self.num_choices		)

						lowerCAmelCase__	:						Tuple  				=	RoFormerConfig(
						    vocab_size=self.vocab_size   ,hidden_size=self.hidden_size   ,num_hidden_layers=self.num_hidden_layers   ,num_attention_heads=self.num_attention_heads   ,intermediate_size=self.intermediate_size   ,hidden_act=self.hidden_act   ,hidden_dropout_prob=self.hidden_dropout_prob   ,attention_probs_dropout_prob=self.attention_probs_dropout_prob   ,max_position_embeddings=self.max_position_embeddings   ,type_vocab_size=self.type_vocab_size   ,initializer_range=self.initializer_range   ,return_dict=lowercase_   ,)

						return config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels







					def        __lowerCAmelCase (       self			:				List[str]   ,lowercase_			:				List[str]   ,lowercase_			:				Tuple   ,lowercase_			:				Tuple   ,lowercase_			:				Optional[int]   ,lowercase_			:				str   ,lowercase_			:				str   ,lowercase_			:				Dict		):
						lowerCAmelCase__	:						Optional[int]  				=	TFRoFormerModel(config=lowercase_		)
						lowerCAmelCase__	:						Optional[Any]  				=	{'''input_ids''': input_ids, '''attention_mask''': input_mask, '''token_type_ids''': token_type_ids}

						lowerCAmelCase__	:						str  				=	[input_ids, input_mask]
						lowerCAmelCase__	:						str  				=	model(lowercase_		)

						lowerCAmelCase__	:						Optional[int]  				=	model(lowercase_		)

						self.parent.assertEqual(result.last_hidden_state.shape   ,(self.batch_size, self.seq_length, self.hidden_size)		)







					def        __lowerCAmelCase (       self			:				List[str]   ,lowercase_			:				Union[str, Any]   ,lowercase_			:				Any   ,lowercase_			:				Union[str, Any]   ,lowercase_			:				int   ,lowercase_			:				Optional[int]   ,lowercase_			:				Optional[Any]   ,lowercase_			:				Optional[int]		):
						lowerCAmelCase__	:						Dict  				=	True
						lowerCAmelCase__	:						Tuple  				=	TFRoFormerForCausalLM(config=lowercase_		)
						lowerCAmelCase__	:						List[str]  				=	{
						    '''input_ids''': input_ids,
						    '''attention_mask''': input_mask,
						    '''token_type_ids''': token_type_ids,
						}
						lowerCAmelCase__	:						Dict  				=	model(lowercase_		)['''logits''']
						self.parent.assertListEqual(
						    list(prediction_scores.numpy().shape		)   ,[self.batch_size, self.seq_length, self.vocab_size]		)







					def        __lowerCAmelCase (       self			:				int   ,lowercase_			:				Any   ,lowercase_			:				int   ,lowercase_			:				int   ,lowercase_			:				str   ,lowercase_			:				str   ,lowercase_			:				Optional[int]   ,lowercase_			:				Any		):
						lowerCAmelCase__	:						Optional[Any]  				=	TFRoFormerForMaskedLM(config=lowercase_		)
						lowerCAmelCase__	:						Dict  				=	{
						    '''input_ids''': input_ids,
						    '''attention_mask''': input_mask,
						    '''token_type_ids''': token_type_ids,
						}
						lowerCAmelCase__	:						Union[str, Any]  				=	model(lowercase_		)
						self.parent.assertEqual(result.logits.shape   ,(self.batch_size, self.seq_length, self.vocab_size)		)







					def        __lowerCAmelCase (       self			:				int   ,lowercase_			:				List[Any]   ,lowercase_			:				List[Any]   ,lowercase_			:				Any   ,lowercase_			:				str   ,lowercase_			:				int   ,lowercase_			:				List[Any]   ,lowercase_			:				List[str]		):
						lowerCAmelCase__	:						List[str]  				=	self.num_labels
						lowerCAmelCase__	:						List[str]  				=	TFRoFormerForSequenceClassification(config=lowercase_		)
						lowerCAmelCase__	:						int  				=	{
						    '''input_ids''': input_ids,
						    '''attention_mask''': input_mask,
						    '''token_type_ids''': token_type_ids,
						}

						lowerCAmelCase__	:						str  				=	model(lowercase_		)
						self.parent.assertEqual(result.logits.shape   ,(self.batch_size, self.num_labels)		)







					def        __lowerCAmelCase (       self			:				int   ,lowercase_			:				List[str]   ,lowercase_			:				Dict   ,lowercase_			:				Any   ,lowercase_			:				str   ,lowercase_			:				str   ,lowercase_			:				List[Any]   ,lowercase_			:				Optional[int]		):
						lowerCAmelCase__	:						List[Any]  				=	self.num_choices
						lowerCAmelCase__	:						Tuple  				=	TFRoFormerForMultipleChoice(config=lowercase_		)
						lowerCAmelCase__	:						List[str]  				=	tf.tile(tf.expand_dims(lowercase_   ,1		)   ,(1, self.num_choices, 1)		)
						lowerCAmelCase__	:						Tuple  				=	tf.tile(tf.expand_dims(lowercase_   ,1		)   ,(1, self.num_choices, 1)		)
						lowerCAmelCase__	:						Tuple  				=	tf.tile(tf.expand_dims(lowercase_   ,1		)   ,(1, self.num_choices, 1)		)
						lowerCAmelCase__	:						Optional[Any]  				=	{
						    '''input_ids''': multiple_choice_inputs_ids,
						    '''attention_mask''': multiple_choice_input_mask,
						    '''token_type_ids''': multiple_choice_token_type_ids,
						}
						lowerCAmelCase__	:						Union[str, Any]  				=	model(lowercase_		)
						self.parent.assertEqual(result.logits.shape   ,(self.batch_size, self.num_choices)		)







					def        __lowerCAmelCase (       self			:				int   ,lowercase_			:				Optional[Any]   ,lowercase_			:				List[str]   ,lowercase_			:				List[str]   ,lowercase_			:				List[str]   ,lowercase_			:				Optional[int]   ,lowercase_			:				Any   ,lowercase_			:				Dict		):
						lowerCAmelCase__	:						Any  				=	self.num_labels
						lowerCAmelCase__	:						List[Any]  				=	TFRoFormerForTokenClassification(config=lowercase_		)
						lowerCAmelCase__	:						Optional[int]  				=	{
						    '''input_ids''': input_ids,
						    '''attention_mask''': input_mask,
						    '''token_type_ids''': token_type_ids,
						}
						lowerCAmelCase__	:						Optional[int]  				=	model(lowercase_		)
						self.parent.assertEqual(result.logits.shape   ,(self.batch_size, self.seq_length, self.num_labels)		)







					def        __lowerCAmelCase (       self			:				str   ,lowercase_			:				List[str]   ,lowercase_			:				Any   ,lowercase_			:				Any   ,lowercase_			:				Dict   ,lowercase_			:				str   ,lowercase_			:				Any   ,lowercase_			:				Any		):
						lowerCAmelCase__	:						List[Any]  				=	TFRoFormerForQuestionAnswering(config=lowercase_		)
						lowerCAmelCase__	:						Optional[int]  				=	{
						    '''input_ids''': input_ids,
						    '''attention_mask''': input_mask,
						    '''token_type_ids''': token_type_ids,
						}

						lowerCAmelCase__	:						str  				=	model(lowercase_		)
						self.parent.assertEqual(result.start_logits.shape   ,(self.batch_size, self.seq_length)		)
						self.parent.assertEqual(result.end_logits.shape   ,(self.batch_size, self.seq_length)		)







					def        __lowerCAmelCase (       self			:				Dict		):
						lowerCAmelCase__	:						Optional[Any]  				=	self.prepare_config_and_inputs()
						(
						    (
						    lowerCAmelCase__
						)				,(
						    lowerCAmelCase__
						)				,(
						    lowerCAmelCase__
						)				,(
						    lowerCAmelCase__
						)				,(
						    lowerCAmelCase__
						)				,(
						    lowerCAmelCase__
						)				,(
						    lowerCAmelCase__
						)				,
						)	:						Optional[int]  				=	config_and_inputs
						lowerCAmelCase__	:						List[Any]  				=	{'''input_ids''': input_ids, '''token_type_ids''': token_type_ids, '''attention_mask''': input_mask}
						return config, inputs_dict







@require_tf
class 				SCREAMING_SNAKE_CASE					(		a_    ,							a_    ,							unittest.TestCase       ):






					"""simple docstring"""



					lowercase__				 =       (
					    (
					        TFRoFormerModel,
					        TFRoFormerForCausalLM,
					        TFRoFormerForMaskedLM,
					        TFRoFormerForQuestionAnswering,
					        TFRoFormerForSequenceClassification,
					        TFRoFormerForTokenClassification,
					        TFRoFormerForMultipleChoice,
					    )
					    if is_tf_available()
					    else ()
					)
					lowercase__				 =       (
					    {
					        "feature-extraction": TFRoFormerModel,
					        "fill-mask": TFRoFormerForMaskedLM,
					        "question-answering": TFRoFormerForQuestionAnswering,
					        "text-classification": TFRoFormerForSequenceClassification,
					        "text-generation": TFRoFormerForCausalLM,
					        "token-classification": TFRoFormerForTokenClassification,
					        "zero-shot": TFRoFormerForSequenceClassification,
					    }
					    if is_tf_available()
					    else {}
					)

					lowercase__				 =       False
					lowercase__				 =       False







					def        __lowerCAmelCase (       self			:				Union[str, Any]   ,lowercase_			:				Dict   ,lowercase_			:				int   ,lowercase_			:				Optional[int]   ,lowercase_			:				Optional[Any]   ,lowercase_			:				str		):
						if pipeline_test_casse_name == "TextGenerationPipelineTests":
							return True

						return False







					def        __lowerCAmelCase (       self			:				str		):
						lowerCAmelCase__	:						Any  				=	TFRoFormerModelTester(self		)
						lowerCAmelCase__	:						Any  				=	ConfigTester(self   ,config_class=lowercase_   ,hidden_size=3_7		)







					def        __lowerCAmelCase (       self			:				Optional[Any]		):
						self.config_tester.run_common_tests()







					def        __lowerCAmelCase (       self			:				List[Any]		):
						lowerCAmelCase__	:						Tuple  				=	self.model_tester.prepare_config_and_inputs()
						self.model_tester.create_and_check_model(*lowercase_		)







					def        __lowerCAmelCase (       self			:				str		):
						lowerCAmelCase__	:						str  				=	self.model_tester.prepare_config_and_inputs()
						self.model_tester.create_and_check_for_masked_lm(*lowercase_		)







					def        __lowerCAmelCase (       self			:				Any		):
						lowerCAmelCase__	:						Optional[int]  				=	self.model_tester.prepare_config_and_inputs()
						self.model_tester.create_and_check_lm_head(*lowercase_		)







					def        __lowerCAmelCase (       self			:				List[str]		):
						lowerCAmelCase__	:						Dict  				=	self.model_tester.prepare_config_and_inputs()
						self.model_tester.create_and_check_for_multiple_choice(*lowercase_		)







					def        __lowerCAmelCase (       self			:				Tuple		):
						lowerCAmelCase__	:						Union[str, Any]  				=	self.model_tester.prepare_config_and_inputs()
						self.model_tester.create_and_check_for_question_answering(*lowercase_		)







					def        __lowerCAmelCase (       self			:				int		):
						lowerCAmelCase__	:						List[Any]  				=	self.model_tester.prepare_config_and_inputs()
						self.model_tester.create_and_check_for_sequence_classification(*lowercase_		)







					def        __lowerCAmelCase (       self			:				List[str]		):
						lowerCAmelCase__	:						List[Any]  				=	self.model_tester.prepare_config_and_inputs()
						self.model_tester.create_and_check_for_token_classification(*lowercase_		)







					@slow
					def        __lowerCAmelCase (       self			:				Tuple		):
						lowerCAmelCase__	:						Dict  				=	TFRoFormerModel.from_pretrained('''junnyu/roformer_chinese_base'''		)
						self.assertIsNotNone(lowercase_		)







@require_tf
class 				SCREAMING_SNAKE_CASE					(		unittest.TestCase       ):






					"""simple docstring"""



					@slow
					def        __lowerCAmelCase (       self			:				Tuple		):
						lowerCAmelCase__	:						Dict  				=	TFRoFormerForMaskedLM.from_pretrained('''junnyu/roformer_chinese_base'''		)
						lowerCAmelCase__	:						Dict  				=	tf.constant([[0, 1, 2, 3, 4, 5]]		)
						lowerCAmelCase__	:						Optional[int]  				=	model(lowercase_		)[0]

						# TODO Replace vocab size
						lowerCAmelCase__	:						int  				=	5_0_0_0_0

						lowerCAmelCase__	:						int  				=	[1, 6, vocab_size]
						self.assertEqual(output.shape   ,lowercase_		)

						print(output[:, :3, :3]		)

						# TODO Replace values below with what was printed above.
						lowerCAmelCase__	:						Optional[int]  				=	tf.constant(
						    [
						        [
						            [-0.1205_3341, -1.026_4901, 0.2922_1946],
						            [-1.513_3783, 0.19_7433, 0.1519_0607],
						            [-5.013_5403, -3.90_0256, -0.8403_8764],
						        ]
						    ]		)
						tf.debugging.assert_near(output[:, :3, :3]   ,lowercase_   ,atol=1E-4		)







@require_tf
class 				SCREAMING_SNAKE_CASE					(		unittest.TestCase       ):






					"""simple docstring"""



					lowercase__				 =       1e-4







					def        __lowerCAmelCase (       self			:				List[Any]		):
						lowerCAmelCase__	:						List[Any]  				=	tf.constant([[4, 1_0]]		)
						lowerCAmelCase__	:						Optional[int]  				=	TFRoFormerSinusoidalPositionalEmbedding(num_positions=6   ,embedding_dim=6		)

						lowerCAmelCase__	:						List[Any]  				=	emba(input_ids.shape		)
						lowerCAmelCase__	:						int  				=	tf.constant(
						    [[0.0000, 0.0000, 0.0000, 1.0000, 1.0000, 1.0000], [0.8415, 0.0464, 0.0022, 0.5403, 0.9989, 1.0000]]		)

						tf.debugging.assert_near(lowercase_   ,lowercase_   ,atol=self.tolerance		)







					def        __lowerCAmelCase (       self			:				List[str]		):
						lowerCAmelCase__	:						Tuple  				=	tf.constant(
						    [
						        [0.0000, 0.0000, 0.0000, 0.0000, 0.0000],
						        [0.8415, 0.8219, 0.8020, 0.7819, 0.7617],
						        [0.9093, 0.9364, 0.9581, 0.9749, 0.9870],
						    ]		)
						lowerCAmelCase__	:						Tuple  				=	TFRoFormerSinusoidalPositionalEmbedding(num_positions=5_1_2   ,embedding_dim=5_1_2		)
						emba([2, 1_6, 5_1_2]		)
						lowerCAmelCase__	:						int  				=	emba.weight[:3, :5]

						tf.debugging.assert_near(lowercase_   ,lowercase_   ,atol=self.tolerance		)







@require_tf
class 				SCREAMING_SNAKE_CASE					(		unittest.TestCase       ):






					"""simple docstring"""



					lowercase__				 =       1e-4







					def        __lowerCAmelCase (       self			:				str		):
						# 2,12,16,64
						lowerCAmelCase__	:						Optional[Any]  				=	tf.reshape(tf.range(2 * 1_2 * 1_6 * 6_4   ,dtype=tf.floataa		)   ,shape=(2, 1_2, 1_6, 6_4)		) / 1_0_0

						lowerCAmelCase__	:						Optional[Any]  				=	-tf.reshape(tf.range(2 * 1_2 * 1_6 * 6_4   ,dtype=tf.floataa		)   ,shape=(2, 1_2, 1_6, 6_4)		) / 1_0_0

						lowerCAmelCase__	:						Optional[int]  				=	TFRoFormerSinusoidalPositionalEmbedding(num_positions=3_2   ,embedding_dim=6_4		)
						lowerCAmelCase__	:						str  				=	embed_positions([2, 1_6, 7_6_8]		)[None, None, :, :]

						lowerCAmelCase__				,lowerCAmelCase__	:						Any  				=	TFRoFormerSelfAttention.apply_rotary_position_embeddings(
						    lowercase_   ,lowercase_   ,lowercase_		)

						lowerCAmelCase__	:						str  				=	tf.constant(
						    [
						        [0.0000, 0.0100, 0.0200, 0.0300, 0.0400, 0.0500, 0.0600, 0.0700],
						        [-0.2012, 0.8897, 0.0263, 0.9401, 0.2074, 0.9463, 0.3481, 0.9343],
						        [-1.7057, 0.6271, -1.2145, 1.3897, -0.6303, 1.7647, -0.1173, 1.8985],
						        [-2.1731, -1.6397, -2.7358, 0.2854, -2.1840, 1.7183, -1.3018, 2.4871],
						        [0.2717, -3.6173, -2.9206, -2.1988, -3.6638, 0.3858, -2.9155, 2.2980],
						        [3.9859, -2.1580, -0.7984, -4.4904, -4.1181, -2.0252, -4.4782, 1.1253],
						    ]		)
						lowerCAmelCase__	:						Union[str, Any]  				=	tf.constant(
						    [
						        [0.0000, -0.0100, -0.0200, -0.0300, -0.0400, -0.0500, -0.0600, -0.0700],
						        [0.2012, -0.8897, -0.0263, -0.9401, -0.2074, -0.9463, -0.3481, -0.9343],
						        [1.7057, -0.6271, 1.2145, -1.3897, 0.6303, -1.7647, 0.1173, -1.8985],
						        [2.1731, 1.6397, 2.7358, -0.2854, 2.1840, -1.7183, 1.3018, -2.4871],
						        [-0.2717, 3.6173, 2.9206, 2.1988, 3.6638, -0.3858, 2.9155, -2.2980],
						        [-3.9859, 2.1580, 0.7984, 4.4904, 4.1181, 2.0252, 4.4782, -1.1253],
						    ]		)

						tf.debugging.assert_near(query_layer[0, 0, :6, :8]   ,lowercase_   ,atol=self.tolerance		)
						tf.debugging.assert_near(key_layer[0, 0, :6, :8]   ,lowercase_   ,atol=self.tolerance		)










import unittest

from transformers import DebertaVaTokenizer, DebertaVaTokenizerFast
from transformers.testing_utils import get_tests_dir, require_sentencepiece, require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin


UpperCamelCase__    			=			get_tests_dir("""fixtures/spiece.model""")
@require_sentencepiece
@require_tokenizers
class 			a__    (					snake_case__ ,				unittest.TestCase      ):
						_a							:    Optional[Any]					=    DebertaVaTokenizer
						_a							:    Optional[Any]					=    DebertaVaTokenizerFast
						_a							:    List[str]					=    True
						_a							:    Optional[Any]					=    True






						def 				__SCREAMING_SNAKE_CASE(   self			):




											"""simple docstring"""



											super().setUp()

											# We have a SentencePiece fixture for testing
											__lowerCAmelCase				   =       DebertaVaTokenizer(_A		,			unk_token="<unk>"			)
											tokenizer.save_pretrained(self.tmpdirname			)






						def 				__SCREAMING_SNAKE_CASE(   self		,			_A			):




											"""simple docstring"""



											__lowerCAmelCase				   =       "this is a test"
											__lowerCAmelCase				   =       "this is a test"
											return input_text, output_text






						def 				__SCREAMING_SNAKE_CASE(   self			):




											"""simple docstring"""



											__lowerCAmelCase				   =       "<pad>"
											__lowerCAmelCase				   =       0

											self.assertEqual(self.get_tokenizer()._convert_token_to_id(_A			)		,			_A			)
											self.assertEqual(self.get_tokenizer()._convert_id_to_token(_A			)		,			_A			)






						def 				__SCREAMING_SNAKE_CASE(   self			):




											"""simple docstring"""



											__lowerCAmelCase				   =       list(self.get_tokenizer().get_vocab().keys()			)
											self.assertEqual(vocab_keys[0]		,			"<pad>"			)
											self.assertEqual(vocab_keys[1]		,			"<unk>"			)
											self.assertEqual(vocab_keys[-1]		,			"[PAD]"			)
											self.assertEqual(len(_A			)		,			3_0_0_0_1			)






						def 				__SCREAMING_SNAKE_CASE(   self			):




											"""simple docstring"""



											self.assertEqual(self.get_tokenizer().vocab_size		,			3_0_0_0_0			)






						def 				__SCREAMING_SNAKE_CASE(   self			):




											"""simple docstring"""



											__lowerCAmelCase				   =       " \tHeLLo!how  \n Are yoU?  "
											__lowerCAmelCase				   =       ["▁hello", "!", "how", "▁are", "▁you", "?"]
											# fmt: on

											__lowerCAmelCase				   =       DebertaVaTokenizer(_A		,			do_lower_case=_A			)
											__lowerCAmelCase				   =       tokenizer.convert_ids_to_tokens(tokenizer.encode(_A		,			add_special_tokens=_A			)			)

											self.assertListEqual(_A		,			_A			)

											__lowerCAmelCase				   =       DebertaVaTokenizerFast(_A		,			do_lower_case=_A			)
											__lowerCAmelCase				   =       rust_tokenizer.convert_ids_to_tokens(rust_tokenizer.encode(_A		,			add_special_tokens=_A			)			)

											self.assertListEqual(_A		,			_A			)






						@unittest.skip("There is an inconsistency between slow and fast tokenizer due to a bug in the fast one."			)
						def 				__SCREAMING_SNAKE_CASE(   self			):




											"""simple docstring"""



											pass






						@unittest.skip("There is an inconsistency between slow and fast tokenizer due to a bug in the fast one."			)
						def 				__SCREAMING_SNAKE_CASE(   self			):




											"""simple docstring"""



											pass






						def 				__SCREAMING_SNAKE_CASE(   self			):




											"""simple docstring"""



											__lowerCAmelCase				   =       "I was born in 92000, and this is falsé."
											__lowerCAmelCase				   =       ["▁", "<unk>", "▁was", "▁born", "▁in", "▁9", "2000", "▁", ",", "▁and", "▁this", "▁is", "▁fal", "s", "<unk>", "▁", ".", ]
											# fmt: on

											__lowerCAmelCase				   =       DebertaVaTokenizer(_A		,			split_by_punct=_A			)
											__lowerCAmelCase				   =       tokenizer.convert_ids_to_tokens(tokenizer.encode(_A		,			add_special_tokens=_A			)			)

											self.assertListEqual(_A		,			_A			)

											__lowerCAmelCase				   =       DebertaVaTokenizerFast(_A		,			split_by_punct=_A			)
											__lowerCAmelCase				   =       rust_tokenizer.convert_ids_to_tokens(rust_tokenizer.encode(_A		,			add_special_tokens=_A			)			)

											self.assertListEqual(_A		,			_A			)






						def 				__SCREAMING_SNAKE_CASE(   self			):




											"""simple docstring"""



											__lowerCAmelCase				   =       "I was born in 92000, and this is falsé."
											__lowerCAmelCase				   =       ["▁i", "▁was", "▁born", "▁in", "▁9", "2000", "▁", ",", "▁and", "▁this", "▁is", "▁fal", "s", "<unk>", "▁", ".", ]
											# fmt: on

											__lowerCAmelCase				   =       DebertaVaTokenizer(_A		,			do_lower_case=_A		,			split_by_punct=_A			)
											__lowerCAmelCase				   =       tokenizer.convert_ids_to_tokens(tokenizer.encode(_A		,			add_special_tokens=_A			)			)
											self.assertListEqual(_A		,			_A			)

											__lowerCAmelCase				   =       DebertaVaTokenizerFast(_A		,			do_lower_case=_A		,			split_by_punct=_A			)
											__lowerCAmelCase				   =       rust_tokenizer.convert_ids_to_tokens(rust_tokenizer.encode(_A		,			add_special_tokens=_A			)			)
											self.assertListEqual(_A		,			_A			)






						def 				__SCREAMING_SNAKE_CASE(   self			):




											"""simple docstring"""



											__lowerCAmelCase				   =       "I was born in 92000, and this is falsé."
											__lowerCAmelCase				   =       ["▁i", "▁was", "▁born", "▁in", "▁9", "2000", ",", "▁and", "▁this", "▁is", "▁fal", "s", "<unk>", ".", ]
											# fmt: on

											__lowerCAmelCase				   =       DebertaVaTokenizer(_A		,			do_lower_case=_A		,			split_by_punct=_A			)
											__lowerCAmelCase				   =       tokenizer.convert_ids_to_tokens(tokenizer.encode(_A		,			add_special_tokens=_A			)			)

											self.assertListEqual(_A		,			_A			)

											__lowerCAmelCase				   =       DebertaVaTokenizerFast(_A		,			do_lower_case=_A		,			split_by_punct=_A			)
											__lowerCAmelCase				   =       rust_tokenizer.convert_ids_to_tokens(rust_tokenizer.encode(_A		,			add_special_tokens=_A			)			)

											self.assertListEqual(_A		,			_A			)






						def 				__SCREAMING_SNAKE_CASE(   self			):




											"""simple docstring"""



											__lowerCAmelCase				   =       "I was born in 92000, and this is falsé."
											__lowerCAmelCase				   =       ["▁", "<unk>", "▁was", "▁born", "▁in", "▁9", "2000", "▁", ",", "▁and", "▁this", "▁is", "▁fal", "s", "<unk>", "▁", ".", ]
											# fmt: on

											__lowerCAmelCase				   =       DebertaVaTokenizer(_A		,			do_lower_case=_A		,			split_by_punct=_A			)
											__lowerCAmelCase				   =       tokenizer.convert_ids_to_tokens(tokenizer.encode(_A		,			add_special_tokens=_A			)			)

											self.assertListEqual(_A		,			_A			)

											__lowerCAmelCase				   =       DebertaVaTokenizerFast(_A		,			do_lower_case=_A		,			split_by_punct=_A			)
											__lowerCAmelCase				   =       rust_tokenizer.convert_ids_to_tokens(rust_tokenizer.encode(_A		,			add_special_tokens=_A			)			)

											self.assertListEqual(_A		,			_A			)






						def 				__SCREAMING_SNAKE_CASE(   self			):




											"""simple docstring"""



											__lowerCAmelCase				   =       " \tHeLLo!how  \n Are yoU?  "
											__lowerCAmelCase				   =       ["▁", "<unk>", "e", "<unk>", "o", "!", "how", "▁", "<unk>", "re", "▁yo", "<unk>", "?"]
											# fmt: on

											__lowerCAmelCase				   =       DebertaVaTokenizer(_A		,			do_lower_case=_A		,			split_by_punct=_A			)
											__lowerCAmelCase				   =       tokenizer.convert_ids_to_tokens(tokenizer.encode(_A		,			add_special_tokens=_A			)			)

											self.assertListEqual(_A		,			_A			)

											__lowerCAmelCase				   =       DebertaVaTokenizerFast(_A		,			do_lower_case=_A		,			split_by_punct=_A			)
											__lowerCAmelCase				   =       rust_tokenizer.convert_ids_to_tokens(rust_tokenizer.encode(_A		,			add_special_tokens=_A			)			)

											self.assertListEqual(_A		,			_A			)






						def 				__SCREAMING_SNAKE_CASE(   self			):




											"""simple docstring"""



											__lowerCAmelCase				   =       self.get_tokenizer()
											__lowerCAmelCase				   =       self.get_rust_tokenizer()

											__lowerCAmelCase				   =       "I was born in 92000, and this is falsé."

											__lowerCAmelCase				   =       tokenizer.convert_ids_to_tokens(tokenizer.encode(_A		,			add_special_tokens=_A			)			)
											__lowerCAmelCase				   =       rust_tokenizer.convert_ids_to_tokens(rust_tokenizer.encode(_A		,			add_special_tokens=_A			)			)
											self.assertListEqual(_A		,			_A			)

											__lowerCAmelCase				   =       tokenizer.encode(_A		,			add_special_tokens=_A			)
											__lowerCAmelCase				   =       rust_tokenizer.encode(_A		,			add_special_tokens=_A			)
											self.assertListEqual(_A		,			_A			)

											__lowerCAmelCase				   =       self.get_rust_tokenizer()
											__lowerCAmelCase				   =       tokenizer.encode(_A			)
											__lowerCAmelCase				   =       rust_tokenizer.encode(_A			)
											self.assertListEqual(_A		,			_A			)






						def 				__SCREAMING_SNAKE_CASE(   self			):




											"""simple docstring"""



											__lowerCAmelCase				   =       "This is a test"
											__lowerCAmelCase				   =       [1_3, 1, 4_3_9_8, 2_5, 2_1, 1_2_8_9]
											__lowerCAmelCase				   =       ["▁", "T", "his", "▁is", "▁a", "▁test"]
											__lowerCAmelCase				   =       ["▁", "<unk>", "his", "▁is", "▁a", "▁test"]

											__lowerCAmelCase				   =       DebertaVaTokenizer(_A		,			keep_accents=_A			)
											__lowerCAmelCase				   =       DebertaVaTokenizerFast(_A		,			keep_accents=_A			)

											__lowerCAmelCase				   =       tokenizer.encode(_A		,			add_special_tokens=_A			)
											self.assertListEqual(_A		,			_A			)
											__lowerCAmelCase				   =       tokenizer.tokenize(_A			)
											self.assertListEqual(_A		,			_A			)
											__lowerCAmelCase				   =       tokenizer.convert_ids_to_tokens(_A			)
											self.assertListEqual(_A		,			_A			)

											__lowerCAmelCase				   =       rust_tokenizer.encode(_A		,			add_special_tokens=_A			)
											self.assertListEqual(_A		,			_A			)
											__lowerCAmelCase				   =       rust_tokenizer.tokenize(_A			)
											self.assertListEqual(_A		,			_A			)
											__lowerCAmelCase				   =       rust_tokenizer.convert_ids_to_tokens(_A			)
											self.assertListEqual(_A		,			_A			)

											# fmt: off
											__lowerCAmelCase				   =       "I was born in 92000, and this is falsé."
											__lowerCAmelCase				   =       [1_3, 1, 2_3, 3_8_6, 1_9, 5_6_1, 3_0_5_0, 1_5, 1_7, 4_8, 2_5, 8_2_5_6, 1_8, 1, 9]
											__lowerCAmelCase				   =       ["▁", "I", "▁was", "▁born", "▁in", "▁9", "2000", ",", "▁and", "▁this", "▁is", "▁fal", "s", "é", ".", ]
											__lowerCAmelCase				   =       ["▁", "<unk>", "▁was", "▁born", "▁in", "▁9", "2000", ",", "▁and", "▁this", "▁is", "▁fal", "s", "<unk>", ".", ]
											# fmt: on

											__lowerCAmelCase				   =       tokenizer.encode(_A		,			add_special_tokens=_A			)
											self.assertListEqual(_A		,			_A			)
											__lowerCAmelCase				   =       tokenizer.tokenize(_A			)
											self.assertListEqual(_A		,			_A			)
											__lowerCAmelCase				   =       tokenizer.convert_ids_to_tokens(_A			)
											self.assertListEqual(_A		,			_A			)

											__lowerCAmelCase				   =       rust_tokenizer.encode(_A		,			add_special_tokens=_A			)
											self.assertListEqual(_A		,			_A			)
											__lowerCAmelCase				   =       rust_tokenizer.tokenize(_A			)
											self.assertListEqual(_A		,			_A			)
											__lowerCAmelCase				   =       rust_tokenizer.convert_ids_to_tokens(_A			)
											self.assertListEqual(_A		,			_A			)






						def 				__SCREAMING_SNAKE_CASE(   self			):




											"""simple docstring"""



											__lowerCAmelCase				   =       DebertaVaTokenizer(_A			)

											__lowerCAmelCase				   =       tokenizer.encode("sequence builders"			)
											__lowerCAmelCase				   =       tokenizer.encode("multi-sequence build"			)

											__lowerCAmelCase				   =       tokenizer.build_inputs_with_special_tokens(_A			)
											__lowerCAmelCase				   =       tokenizer.build_inputs_with_special_tokens(_A		,			_A			)

											self.assertEqual([tokenizer.cls_token_id] + text + [tokenizer.sep_token_id]		,			_A			)
											self.assertEqual(
											    [tokenizer.cls_token_id] + text + [tokenizer.sep_token_id] + text_a + [tokenizer.sep_token_id]		,			_A		,			)






						@slow
						def 				__SCREAMING_SNAKE_CASE(   self			):




											"""simple docstring"""



											__lowerCAmelCase				   =       {"input_ids": [[1, 3_9_8_6_7, 3_6, 1_9_3_9_0, 4_8_6, 2_7, 3_5_0_5_2, 8_1_4_3_6, 1_8, 6_0_6_8_5, 1_2_2_5, 7, 3_5_0_5_2, 8_1_4_3_6, 1_8, 9_3_6_7, 1_6_8_9_9, 1_8, 1_5_9_3_7, 5_3, 5_9_4, 7_7_3, 1_8, 1_6_2_8_7, 3_0_4_6_5, 3_6, 1_5_9_3_7, 6, 4_1_1_3_9, 3_8, 3_6_9_7_9, 6_0_7_6_3, 1_9_1, 6, 3_4_1_3_2, 9_9, 6, 5_0_5_3_8, 3_9_0, 4_3_2_3_0, 6, 3_4_1_3_2, 2_7_7_9, 2_0_8_5_0, 1_4, 6_9_9, 1_0_7_2, 1_1_9_4, 3_6, 3_8_2, 1_0_9_0_1, 5_3, 7, 6_9_9, 1_0_7_2, 2_0_8_4, 3_6, 2_0_4_2_2, 6_3_0, 5_3, 1_9, 1_0_5, 3_0_4_9, 1_8_9_6, 1_0_5_3, 1_6_8_9_9, 1_5_0_6, 1_1, 3_7_9_7_8, 4_2_4_3, 7, 1_2_3_7, 3_1_8_6_9, 2_0_0, 1_6_5_6_6, 6_5_4, 6, 3_5_0_5_2, 8_1_4_3_6, 7, 5_5_6_3_0, 1_3_5_9_3, 4, 2], [1, 2_6, 1_5_0_1_1, 1_3, 6_6_7, 8, 1_0_5_3, 1_8, 2_3_6_1_1, 1_2_3_7, 7_2_3_5_6, 1_2_8_2_0, 3_4, 1_0_4_1_3_4, 1_2_0_9, 3_5, 1_3_3_1_3, 6_6_2_7, 2_1, 2_0_2, 3_4_7, 7, 1_6_4, 2_3_9_9, 1_1, 4_6, 4_4_8_5, 4, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 5, 1_2_3_2, 2_8_6_4, 1_5_7_8_5, 1_4_9_5_1, 1_0_5, 5, 8_5_8_1, 1_2_5_0, 4, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]], "token_type_ids": [[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]], "attention_mask": [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]]}  # noqa: E501
											# fmt: on

											self.tokenizer_integration_test_util(
											    expected_encoding=_A		,			model_name="microsoft/deberta-v2-xlarge"		,			revision="ad6e42c1532ddf3a15c39246b63f5559d558b670"		,			)




