
'''simple docstring'''





from __future__ import annotations

import unittest

import numpy as np

from transformers import LayoutLMConfig, is_tf_available
from transformers.testing_utils import require_tf, slow

from ...test_configuration_common import ConfigTester
from ...test_modeling_tf_common import TFModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_tf_available():
 import tensorflow as tf

 from transformers.models.layoutlm.modeling_tf_layoutlm import (
     TF_LAYOUTLM_PRETRAINED_MODEL_ARCHIVE_LIST,
     TFLayoutLMForMaskedLM,
     TFLayoutLMForQuestionAnswering,
     TFLayoutLMForSequenceClassification,
     TFLayoutLMForTokenClassification,
     TFLayoutLMModel,
 )


class        UpperCamelCase__     :






 def __init__(	self     :str		, _A     :int		, _A     :Optional[int]=13		, _A     :Optional[Any]=7		, _A     :List[str]=True		, _A     :Dict=True		, _A     :Dict=True		, _A     :int=True		, _A     :List[str]=99		, _A     :int=32		, _A     :Union[str, Any]=2		, _A     :Optional[Any]=4		, _A     :List[Any]=37		, _A     :Union[str, Any]="gelu"		, _A     :Dict=0.1		, _A     :int=0.1		, _A     :Union[str, Any]=512		, _A     :Dict=16		, _A     :Optional[Any]=2		, _A     :str=0.02		, _A     :Dict=3		, _A     :Tuple=4		, _A     :Dict=None		, _A     :Union[str, Any]=1_000		, ) ->	List[str]:







       '''simple docstring'''



       __A						=	parent
       __A						=	batch_size
       __A						=	seq_length
       __A						=	is_training
       __A						=	use_input_mask
       __A						=	use_token_type_ids
       __A						=	use_labels
       __A						=	vocab_size
       __A						=	hidden_size
       __A						=	num_hidden_layers
       __A						=	num_attention_heads
       __A						=	intermediate_size
       __A						=	hidden_act
       __A						=	hidden_dropout_prob
       __A						=	attention_probs_dropout_prob
       __A						=	max_position_embeddings
       __A						=	type_vocab_size
       __A						=	type_sequence_label_size
       __A						=	initializer_range
       __A						=	num_labels
       __A						=	num_choices
       __A						=	scope
       __A						=	range_bbox






 def 		lowercase_  (	self     :Tuple	) ->	List[str]:







       '''simple docstring'''



       __A						=	ids_tensor([self.batch_size, self.seq_length]		, self.vocab_size	)

       # convert bbox to numpy since TF does not support item assignment
       __A						=	ids_tensor([self.batch_size, self.seq_length, 4]		, self.range_bbox	).numpy()
       # Ensure that bbox is legal
       for i in range(bbox.shape[0]	):
             for j in range(bbox.shape[1]	):
                   if bbox[i, j, 3] < bbox[i, j, 1]:
                         __A						=	bbox[i, j, 3]
                         __A						=	bbox[i, j, 1]
                         __A						=	t
                   if bbox[i, j, 2] < bbox[i, j, 0]:
                         __A						=	bbox[i, j, 2]
                         __A						=	bbox[i, j, 0]
                         __A						=	t
       __A						=	tf.convert_to_tensor(_A	)

       __A						=	None
       if self.use_input_mask:
             __A						=	random_attention_mask([self.batch_size, self.seq_length]	)

       __A						=	None
       if self.use_token_type_ids:
             __A						=	ids_tensor([self.batch_size, self.seq_length]		, self.type_vocab_size	)

       __A						=	None
       __A						=	None
       __A						=	None
       if self.use_labels:
             __A						=	ids_tensor([self.batch_size]		, self.type_sequence_label_size	)
             __A						=	ids_tensor([self.batch_size, self.seq_length]		, self.num_labels	)
             __A						=	ids_tensor([self.batch_size]		, self.num_choices	)

       __A						=	LayoutLMConfig(
           vocab_size=self.vocab_size		, hidden_size=self.hidden_size		, num_hidden_layers=self.num_hidden_layers		, num_attention_heads=self.num_attention_heads		, intermediate_size=self.intermediate_size		, hidden_act=self.hidden_act		, hidden_dropout_prob=self.hidden_dropout_prob		, attention_probs_dropout_prob=self.attention_probs_dropout_prob		, max_position_embeddings=self.max_position_embeddings		, type_vocab_size=self.type_vocab_size		, initializer_range=self.initializer_range		, )

       return config, input_ids, bbox, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels






 def 		lowercase_  (	self     :Union[str, Any]		, _A     :List[str]		, _A     :Any		, _A     :List[Any]		, _A     :Any		, _A     :Any		, _A     :Union[str, Any]		, _A     :Union[str, Any]		, _A     :Optional[Any]	) ->	List[str]:







       '''simple docstring'''



       __A						=	TFLayoutLMModel(config=_A	)

       __A						=	model(_A		, _A		, attention_mask=_A		, token_type_ids=_A	)
       __A						=	model(_A		, _A		, token_type_ids=_A	)
       __A						=	model(_A		, _A	)
       self.parent.assertEqual(result.last_hidden_state.shape		, (self.batch_size, self.seq_length, self.hidden_size)	)
       self.parent.assertEqual(result.pooler_output.shape		, (self.batch_size, self.hidden_size)	)






 def 		lowercase_  (	self     :List[str]		, _A     :Optional[int]		, _A     :List[Any]		, _A     :int		, _A     :Dict		, _A     :Tuple		, _A     :Optional[int]		, _A     :Union[str, Any]		, _A     :List[Any]	) ->	Optional[Any]:







       '''simple docstring'''



       __A						=	TFLayoutLMForMaskedLM(config=_A	)

       __A						=	model(_A		, _A		, attention_mask=_A		, token_type_ids=_A		, labels=_A	)
       self.parent.assertEqual(result.logits.shape		, (self.batch_size, self.seq_length, self.vocab_size)	)






 def 		lowercase_  (	self     :List[str]		, _A     :List[Any]		, _A     :Tuple		, _A     :Dict		, _A     :Union[str, Any]		, _A     :Tuple		, _A     :Optional[Any]		, _A     :Optional[int]		, _A     :Any	) ->	str:







       '''simple docstring'''



       __A						=	self.num_labels
       __A						=	TFLayoutLMForSequenceClassification(config=_A	)

       __A						=	model(_A		, _A		, attention_mask=_A		, token_type_ids=_A	)
       self.parent.assertEqual(result.logits.shape		, (self.batch_size, self.num_labels)	)






 def 		lowercase_  (	self     :Dict		, _A     :List[Any]		, _A     :Any		, _A     :List[str]		, _A     :List[Any]		, _A     :int		, _A     :List[Any]		, _A     :Optional[Any]		, _A     :Dict	) ->	Any:







       '''simple docstring'''



       __A						=	self.num_labels
       __A						=	TFLayoutLMForTokenClassification(config=_A	)

       __A						=	model(_A		, _A		, attention_mask=_A		, token_type_ids=_A		, labels=_A	)
       self.parent.assertEqual(result.logits.shape		, (self.batch_size, self.seq_length, self.num_labels)	)






 def 		lowercase_  (	self     :Tuple		, _A     :Optional[int]		, _A     :Optional[int]		, _A     :Optional[Any]		, _A     :str		, _A     :Dict		, _A     :int		, _A     :str		, _A     :int	) ->	Dict:







       '''simple docstring'''



       __A						=	TFLayoutLMForQuestionAnswering(config=_A	)

       __A						=	model(_A		, _A		, attention_mask=_A		, token_type_ids=_A	)
       self.parent.assertEqual(result.start_logits.shape		, (self.batch_size, self.seq_length)	)
       self.parent.assertEqual(result.end_logits.shape		, (self.batch_size, self.seq_length)	)






 def 		lowercase_  (	self     :str	) ->	Dict:







       '''simple docstring'''



       __A						=	self.prepare_config_and_inputs()
       (
           (
           __A							
       )		,					(
           __A							
       )		,					(
           __A							
       )		,					(
           __A							
       )		,					(
           __A							
       )		,					(
           __A							
       )		,					(
           __A							
       )		,					(
           __A							
       )		,												
       )						=	config_and_inputs
       __A						=	{
           'input_ids': input_ids,
           'bbox': bbox,
           'token_type_ids': token_type_ids,
           'attention_mask': input_mask,
       }
       return config, inputs_dict


@require_tf
class        UpperCamelCase__     (			snake_case__					,   snake_case__					,   unittest.TestCase):
 UpperCAmelCase__			:			List[Any]   =    (
     (
         TFLayoutLMModel,
         TFLayoutLMForMaskedLM,
         TFLayoutLMForTokenClassification,
         TFLayoutLMForSequenceClassification,
         TFLayoutLMForQuestionAnswering,
     )
     if is_tf_available()
     else ()
 )
 UpperCAmelCase__			:			Optional[int]   =    (
     {
         """feature-extraction""": TFLayoutLMModel,
         """fill-mask""": TFLayoutLMForMaskedLM,
         """text-classification""": TFLayoutLMForSequenceClassification,
         """token-classification""": TFLayoutLMForTokenClassification,
         """zero-shot""": TFLayoutLMForSequenceClassification,
     }
     if is_tf_available()
     else {}
 )
 UpperCAmelCase__			:			Optional[Any]   =    False
 UpperCAmelCase__			:			Dict   =    True
 UpperCAmelCase__			:			List[str]   =    1_0






 def 		lowercase_  (	self     :Any	) ->	Tuple:







       '''simple docstring'''



       __A						=	TFLayoutLMModelTester(self	)
       __A						=	ConfigTester(self		, config_class=_A		, hidden_size=37	)






 def 		lowercase_  (	self     :Optional[int]	) ->	Union[str, Any]:







       '''simple docstring'''



       self.config_tester.run_common_tests()






 def 		lowercase_  (	self     :Optional[int]	) ->	Tuple:







       '''simple docstring'''



       __A						=	self.model_tester.prepare_config_and_inputs()
       self.model_tester.create_and_check_model(*_A	)






 def 		lowercase_  (	self     :Dict	) ->	int:







       '''simple docstring'''



       __A						=	self.model_tester.prepare_config_and_inputs()
       self.model_tester.create_and_check_for_masked_lm(*_A	)






 def 		lowercase_  (	self     :List[Any]	) ->	List[str]:







       '''simple docstring'''



       __A						=	self.model_tester.prepare_config_and_inputs()
       self.model_tester.create_and_check_for_sequence_classification(*_A	)






 def 		lowercase_  (	self     :Any	) ->	Tuple:







       '''simple docstring'''



       __A						=	self.model_tester.prepare_config_and_inputs()
       self.model_tester.create_and_check_for_token_classification(*_A	)






 def 		lowercase_  (	self     :Dict	) ->	List[Any]:







       '''simple docstring'''



       __A						=	self.model_tester.prepare_config_and_inputs()
       self.model_tester.create_and_check_for_question_answering(*_A	)






 @slow
 def 		lowercase_  (	self     :List[str]	) ->	str:







       '''simple docstring'''



       for model_name in TF_LAYOUTLM_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
             __A						=	TFLayoutLMModel.from_pretrained(_A	)
             self.assertIsNotNone(_A	)






 @unittest.skip('Onnx compliancy broke with TF 2.10'	)
 def 		lowercase_  (	self     :Tuple	) ->	List[str]:







       '''simple docstring'''



       pass





def 				snake_case							(						)->	Dict:







      """simple docstring"""
      # Here we prepare a batch of 2 sequences to test a LayoutLM forward pass on:
      # fmt: off
      __A						=	tf.convert_to_tensor([[1_0_1,1_0_1_9,1_0_1_4,1_0_1_6,1_0_3_7,1_2_8_4_9,4_7_4_7,1_0_0_4,1_4_2_4_6,2_2_7_8,5_4_3_9,4_5_2_4,5_0_0_2,2_9_3_0,2_1_9_3,2_9_3_0,4_3_4_1,3_2_0_8,1_0_0_5,1_0_5_5,2_1_7_1,2_8_4_8,1_1_3_0_0,3_5_3_1,1_0_2],[1_0_1,4_0_7_0,4_0_3_4,7_0_2_0,1_0_2_4,3_0_5_8,1_0_1_5,1_0_1_3,2_8_6_1,1_0_1_3,6_0_7_0,1_9_2_7_4,2_7_7_2,6_2_0_5,2_7_8_1_4,1_6_1_4_7,1_6_1_4_7,4_3_4_3,2_0_4_7,1_0_2_8_3,1_0_9_6_9,1_4_3_8_9,1_0_1_2,2_3_3_8,1_0_2]]		)  # noqa: E231
      __A						=	tf.convert_to_tensor([[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1],[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1],]		)  # noqa: E231
      __A						=	tf.convert_to_tensor([[[0,0,0,0],[4_2_3,2_3_7,4_4_0,2_5_1],[4_2_7,2_7_2,4_4_1,2_8_7],[4_1_9,1_1_5,4_3_7,1_2_9],[9_6_1,8_8_5,9_9_2,9_1_2],[2_5_6,3_8,3_3_0,5_8],[2_5_6,3_8,3_3_0,5_8],[3_3_6,4_2,3_5_3,5_7],[3_6_0,3_9,4_0_1,5_6],[3_6_0,3_9,4_0_1,5_6],[4_1_1,3_9,4_7_1,5_9],[4_7_9,4_1,5_2_8,5_9],[5_3_3,3_9,6_3_0,6_0],[6_7,1_1_3,1_3_4,1_3_1],[1_4_1,1_1_5,2_0_9,1_3_2],[6_8,1_4_9,1_3_3,1_6_6],[1_4_1,1_4_9,1_8_7,1_6_4],[1_9_5,1_4_8,2_8_7,1_6_5],[1_9_5,1_4_8,2_8_7,1_6_5],[1_9_5,1_4_8,2_8_7,1_6_5],[2_9_5,1_4_8,3_4_9,1_6_5],[4_4_1,1_4_9,4_9_2,1_6_6],[4_9_7,1_4_9,5_4_6,1_6_4],[6_4,2_0_1,1_2_5,2_1_8],[1_0_0_0,1_0_0_0,1_0_0_0,1_0_0_0]],[[0,0,0,0],[6_6_2,1_5_0,7_5_4,1_6_6],[6_6_5,1_9_9,7_4_2,2_1_1],[5_1_9,2_1_3,5_5_4,2_2_8],[5_1_9,2_1_3,5_5_4,2_2_8],[1_3_4,4_3_3,1_8_7,4_5_4],[1_3_0,4_6_7,2_0_4,4_8_0],[1_3_0,4_6_7,2_0_4,4_8_0],[1_3_0,4_6_7,2_0_4,4_8_0],[1_3_0,4_6_7,2_0_4,4_8_0],[1_3_0,4_6_7,2_0_4,4_8_0],[3_1_4,4_6_9,3_7_6,4_8_2],[5_0_4,6_8_4,5_8_2,7_0_6],[9_4_1,8_2_5,9_7_3,9_0_0],[9_4_1,8_2_5,9_7_3,9_0_0],[9_4_1,8_2_5,9_7_3,9_0_0],[9_4_1,8_2_5,9_7_3,9_0_0],[6_1_0,7_4_9,6_5_2,7_6_5],[1_3_0,6_5_9,1_6_8,6_7_2],[1_7_6,6_5_7,2_3_7,6_7_2],[2_3_8,6_5_7,3_1_2,6_7_2],[4_4_3,6_5_3,6_2_8,6_7_2],[4_4_3,6_5_3,6_2_8,6_7_2],[7_1_6,3_0_1,8_2_5,3_1_7],[1_0_0_0,1_0_0_0,1_0_0_0,1_0_0_0]]]		)  # noqa: E231
      __A						=	tf.convert_to_tensor([[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]]		)  # noqa: E231
      # these are sequence labels (i.e. at the token level)
      __A						=	tf.convert_to_tensor([[-1_0_0,1_0,1_0,1_0,9,1,-1_0_0,7,7,-1_0_0,7,7,4,2,5,2,8,8,-1_0_0,-1_0_0,5,0,3,2,-1_0_0],[-1_0_0,1_2,1_2,1_2,-1_0_0,1_2,1_0,-1_0_0,-1_0_0,-1_0_0,-1_0_0,1_0,1_2,9,-1_0_0,-1_0_0,-1_0_0,1_0,1_0,1_0,9,1_2,-1_0_0,1_0,-1_0_0]]		)  # noqa: E231
      # fmt: on

      return input_ids, attention_mask, bbox, token_type_ids, labels


@require_tf
class        UpperCamelCase__     (			unittest.TestCase):






 @slow
 def 		lowercase_  (	self     :Tuple	) ->	Optional[Any]:







       '''simple docstring'''



       __A						=	TFLayoutLMModel.from_pretrained('microsoft/layoutlm-base-uncased'	)

       __A		,					__A		,					__A		,					__A		,					__A						=	prepare_layoutlm_batch_inputs()

       # forward pass
       __A						=	model(input_ids=_A		, bbox=_A		, attention_mask=_A		, token_type_ids=_A	)

       # test the sequence output on [0, :3, :3]
       __A						=	tf.convert_to_tensor(
           [[0.1_785, -0.1_947, -0.0_425], [-0.3_254, -0.2_807, 0.2_553], [-0.5_391, -0.3_322, 0.3_364]]		, )

       self.assertTrue(np.allclose(outputs.last_hidden_state[0, :3, :3]		, _A		, atol=1E-3	)	)

       # test the pooled output on [1, :3]
       __A						=	tf.convert_to_tensor([-0.6_580, -0.0_214, 0.8_552]	)

       self.assertTrue(np.allclose(outputs.pooler_output[1, :3]		, _A		, atol=1E-3	)	)






 @slow
 def 		lowercase_  (	self     :Tuple	) ->	Any:







       '''simple docstring'''



       __A						=	TFLayoutLMForSequenceClassification.from_pretrained('microsoft/layoutlm-base-uncased'		, num_labels=2	)

       __A		,					__A		,					__A		,					__A		,					__A						=	prepare_layoutlm_batch_inputs()

       # forward pass
       __A						=	model(
           input_ids=_A		, bbox=_A		, attention_mask=_A		, token_type_ids=_A		, labels=tf.convert_to_tensor([1, 1]	)		, )

       # test whether we get a loss as a scalar
       __A						=	outputs.loss
       __A						=	(2,)
       self.assertEqual(loss.shape		, _A	)

       # test the shape of the logits
       __A						=	outputs.logits
       __A						=	(2, 2)
       self.assertEqual(logits.shape		, _A	)






 @slow
 def 		lowercase_  (	self     :Any	) ->	int:







       '''simple docstring'''



       __A						=	TFLayoutLMForTokenClassification.from_pretrained('microsoft/layoutlm-base-uncased'		, num_labels=13	)

       __A		,					__A		,					__A		,					__A		,					__A						=	prepare_layoutlm_batch_inputs()

       # forward pass
       __A						=	model(
           input_ids=_A		, bbox=_A		, attention_mask=_A		, token_type_ids=_A		, labels=_A	)

       # test the shape of the logits
       __A						=	outputs.logits
       __A						=	tf.convert_to_tensor((2, 25, 13)	)
       self.assertEqual(logits.shape		, _A	)






 @slow
 def 		lowercase_  (	self     :List[Any]	) ->	List[str]:







       '''simple docstring'''



       __A						=	TFLayoutLMForQuestionAnswering.from_pretrained('microsoft/layoutlm-base-uncased'	)

       __A		,					__A		,					__A		,					__A		,					__A						=	prepare_layoutlm_batch_inputs()

       # forward pass
       __A						=	model(input_ids=_A		, bbox=_A		, attention_mask=_A		, token_type_ids=_A	)

       # test the shape of the logits
       __A						=	tf.convert_to_tensor((2, 25)	)
       self.assertEqual(outputs.start_logits.shape		, _A	)
       self.assertEqual(outputs.end_logits.shape		, _A	)












import unittest

from transformers import DebertaVaTokenizer, DebertaVaTokenizerFast
from transformers.testing_utils import get_tests_dir, require_sentencepiece, require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin


UpperCamelCase__    			=			get_tests_dir("""fixtures/spiece.model""")
@require_sentencepiece
@require_tokenizers
class 			a__    (					snake_case__ ,				unittest.TestCase      ):
						_a							:    Optional[Any]					=    DebertaVaTokenizer
						_a							:    Optional[Any]					=    DebertaVaTokenizerFast
						_a							:    List[str]					=    True
						_a							:    Optional[Any]					=    True






						def 				__SCREAMING_SNAKE_CASE(   self			):




											"""simple docstring"""



											super().setUp()

											# We have a SentencePiece fixture for testing
											__lowerCAmelCase				   =       DebertaVaTokenizer(_A		,			unk_token="<unk>"			)
											tokenizer.save_pretrained(self.tmpdirname			)






						def 				__SCREAMING_SNAKE_CASE(   self		,			_A			):




											"""simple docstring"""



											__lowerCAmelCase				   =       "this is a test"
											__lowerCAmelCase				   =       "this is a test"
											return input_text, output_text






						def 				__SCREAMING_SNAKE_CASE(   self			):




											"""simple docstring"""



											__lowerCAmelCase				   =       "<pad>"
											__lowerCAmelCase				   =       0

											self.assertEqual(self.get_tokenizer()._convert_token_to_id(_A			)		,			_A			)
											self.assertEqual(self.get_tokenizer()._convert_id_to_token(_A			)		,			_A			)






						def 				__SCREAMING_SNAKE_CASE(   self			):




											"""simple docstring"""



											__lowerCAmelCase				   =       list(self.get_tokenizer().get_vocab().keys()			)
											self.assertEqual(vocab_keys[0]		,			"<pad>"			)
											self.assertEqual(vocab_keys[1]		,			"<unk>"			)
											self.assertEqual(vocab_keys[-1]		,			"[PAD]"			)
											self.assertEqual(len(_A			)		,			3_0_0_0_1			)






						def 				__SCREAMING_SNAKE_CASE(   self			):




											"""simple docstring"""



											self.assertEqual(self.get_tokenizer().vocab_size		,			3_0_0_0_0			)






						def 				__SCREAMING_SNAKE_CASE(   self			):




											"""simple docstring"""



											__lowerCAmelCase				   =       " \tHeLLo!how  \n Are yoU?  "
											__lowerCAmelCase				   =       ["▁hello", "!", "how", "▁are", "▁you", "?"]
											# fmt: on

											__lowerCAmelCase				   =       DebertaVaTokenizer(_A		,			do_lower_case=_A			)
											__lowerCAmelCase				   =       tokenizer.convert_ids_to_tokens(tokenizer.encode(_A		,			add_special_tokens=_A			)			)

											self.assertListEqual(_A		,			_A			)

											__lowerCAmelCase				   =       DebertaVaTokenizerFast(_A		,			do_lower_case=_A			)
											__lowerCAmelCase				   =       rust_tokenizer.convert_ids_to_tokens(rust_tokenizer.encode(_A		,			add_special_tokens=_A			)			)

											self.assertListEqual(_A		,			_A			)






						@unittest.skip("There is an inconsistency between slow and fast tokenizer due to a bug in the fast one."			)
						def 				__SCREAMING_SNAKE_CASE(   self			):




											"""simple docstring"""



											pass






						@unittest.skip("There is an inconsistency between slow and fast tokenizer due to a bug in the fast one."			)
						def 				__SCREAMING_SNAKE_CASE(   self			):




											"""simple docstring"""



											pass






						def 				__SCREAMING_SNAKE_CASE(   self			):




											"""simple docstring"""



											__lowerCAmelCase				   =       "I was born in 92000, and this is falsé."
											__lowerCAmelCase				   =       ["▁", "<unk>", "▁was", "▁born", "▁in", "▁9", "2000", "▁", ",", "▁and", "▁this", "▁is", "▁fal", "s", "<unk>", "▁", ".", ]
											# fmt: on

											__lowerCAmelCase				   =       DebertaVaTokenizer(_A		,			split_by_punct=_A			)
											__lowerCAmelCase				   =       tokenizer.convert_ids_to_tokens(tokenizer.encode(_A		,			add_special_tokens=_A			)			)

											self.assertListEqual(_A		,			_A			)

											__lowerCAmelCase				   =       DebertaVaTokenizerFast(_A		,			split_by_punct=_A			)
											__lowerCAmelCase				   =       rust_tokenizer.convert_ids_to_tokens(rust_tokenizer.encode(_A		,			add_special_tokens=_A			)			)

											self.assertListEqual(_A		,			_A			)






						def 				__SCREAMING_SNAKE_CASE(   self			):




											"""simple docstring"""



											__lowerCAmelCase				   =       "I was born in 92000, and this is falsé."
											__lowerCAmelCase				   =       ["▁i", "▁was", "▁born", "▁in", "▁9", "2000", "▁", ",", "▁and", "▁this", "▁is", "▁fal", "s", "<unk>", "▁", ".", ]
											# fmt: on

											__lowerCAmelCase				   =       DebertaVaTokenizer(_A		,			do_lower_case=_A		,			split_by_punct=_A			)
											__lowerCAmelCase				   =       tokenizer.convert_ids_to_tokens(tokenizer.encode(_A		,			add_special_tokens=_A			)			)
											self.assertListEqual(_A		,			_A			)

											__lowerCAmelCase				   =       DebertaVaTokenizerFast(_A		,			do_lower_case=_A		,			split_by_punct=_A			)
											__lowerCAmelCase				   =       rust_tokenizer.convert_ids_to_tokens(rust_tokenizer.encode(_A		,			add_special_tokens=_A			)			)
											self.assertListEqual(_A		,			_A			)






						def 				__SCREAMING_SNAKE_CASE(   self			):




											"""simple docstring"""



											__lowerCAmelCase				   =       "I was born in 92000, and this is falsé."
											__lowerCAmelCase				   =       ["▁i", "▁was", "▁born", "▁in", "▁9", "2000", ",", "▁and", "▁this", "▁is", "▁fal", "s", "<unk>", ".", ]
											# fmt: on

											__lowerCAmelCase				   =       DebertaVaTokenizer(_A		,			do_lower_case=_A		,			split_by_punct=_A			)
											__lowerCAmelCase				   =       tokenizer.convert_ids_to_tokens(tokenizer.encode(_A		,			add_special_tokens=_A			)			)

											self.assertListEqual(_A		,			_A			)

											__lowerCAmelCase				   =       DebertaVaTokenizerFast(_A		,			do_lower_case=_A		,			split_by_punct=_A			)
											__lowerCAmelCase				   =       rust_tokenizer.convert_ids_to_tokens(rust_tokenizer.encode(_A		,			add_special_tokens=_A			)			)

											self.assertListEqual(_A		,			_A			)






						def 				__SCREAMING_SNAKE_CASE(   self			):




											"""simple docstring"""



											__lowerCAmelCase				   =       "I was born in 92000, and this is falsé."
											__lowerCAmelCase				   =       ["▁", "<unk>", "▁was", "▁born", "▁in", "▁9", "2000", "▁", ",", "▁and", "▁this", "▁is", "▁fal", "s", "<unk>", "▁", ".", ]
											# fmt: on

											__lowerCAmelCase				   =       DebertaVaTokenizer(_A		,			do_lower_case=_A		,			split_by_punct=_A			)
											__lowerCAmelCase				   =       tokenizer.convert_ids_to_tokens(tokenizer.encode(_A		,			add_special_tokens=_A			)			)

											self.assertListEqual(_A		,			_A			)

											__lowerCAmelCase				   =       DebertaVaTokenizerFast(_A		,			do_lower_case=_A		,			split_by_punct=_A			)
											__lowerCAmelCase				   =       rust_tokenizer.convert_ids_to_tokens(rust_tokenizer.encode(_A		,			add_special_tokens=_A			)			)

											self.assertListEqual(_A		,			_A			)






						def 				__SCREAMING_SNAKE_CASE(   self			):




											"""simple docstring"""



											__lowerCAmelCase				   =       " \tHeLLo!how  \n Are yoU?  "
											__lowerCAmelCase				   =       ["▁", "<unk>", "e", "<unk>", "o", "!", "how", "▁", "<unk>", "re", "▁yo", "<unk>", "?"]
											# fmt: on

											__lowerCAmelCase				   =       DebertaVaTokenizer(_A		,			do_lower_case=_A		,			split_by_punct=_A			)
											__lowerCAmelCase				   =       tokenizer.convert_ids_to_tokens(tokenizer.encode(_A		,			add_special_tokens=_A			)			)

											self.assertListEqual(_A		,			_A			)

											__lowerCAmelCase				   =       DebertaVaTokenizerFast(_A		,			do_lower_case=_A		,			split_by_punct=_A			)
											__lowerCAmelCase				   =       rust_tokenizer.convert_ids_to_tokens(rust_tokenizer.encode(_A		,			add_special_tokens=_A			)			)

											self.assertListEqual(_A		,			_A			)






						def 				__SCREAMING_SNAKE_CASE(   self			):




											"""simple docstring"""



											__lowerCAmelCase				   =       self.get_tokenizer()
											__lowerCAmelCase				   =       self.get_rust_tokenizer()

											__lowerCAmelCase				   =       "I was born in 92000, and this is falsé."

											__lowerCAmelCase				   =       tokenizer.convert_ids_to_tokens(tokenizer.encode(_A		,			add_special_tokens=_A			)			)
											__lowerCAmelCase				   =       rust_tokenizer.convert_ids_to_tokens(rust_tokenizer.encode(_A		,			add_special_tokens=_A			)			)
											self.assertListEqual(_A		,			_A			)

											__lowerCAmelCase				   =       tokenizer.encode(_A		,			add_special_tokens=_A			)
											__lowerCAmelCase				   =       rust_tokenizer.encode(_A		,			add_special_tokens=_A			)
											self.assertListEqual(_A		,			_A			)

											__lowerCAmelCase				   =       self.get_rust_tokenizer()
											__lowerCAmelCase				   =       tokenizer.encode(_A			)
											__lowerCAmelCase				   =       rust_tokenizer.encode(_A			)
											self.assertListEqual(_A		,			_A			)






						def 				__SCREAMING_SNAKE_CASE(   self			):




											"""simple docstring"""



											__lowerCAmelCase				   =       "This is a test"
											__lowerCAmelCase				   =       [1_3, 1, 4_3_9_8, 2_5, 2_1, 1_2_8_9]
											__lowerCAmelCase				   =       ["▁", "T", "his", "▁is", "▁a", "▁test"]
											__lowerCAmelCase				   =       ["▁", "<unk>", "his", "▁is", "▁a", "▁test"]

											__lowerCAmelCase				   =       DebertaVaTokenizer(_A		,			keep_accents=_A			)
											__lowerCAmelCase				   =       DebertaVaTokenizerFast(_A		,			keep_accents=_A			)

											__lowerCAmelCase				   =       tokenizer.encode(_A		,			add_special_tokens=_A			)
											self.assertListEqual(_A		,			_A			)
											__lowerCAmelCase				   =       tokenizer.tokenize(_A			)
											self.assertListEqual(_A		,			_A			)
											__lowerCAmelCase				   =       tokenizer.convert_ids_to_tokens(_A			)
											self.assertListEqual(_A		,			_A			)

											__lowerCAmelCase				   =       rust_tokenizer.encode(_A		,			add_special_tokens=_A			)
											self.assertListEqual(_A		,			_A			)
											__lowerCAmelCase				   =       rust_tokenizer.tokenize(_A			)
											self.assertListEqual(_A		,			_A			)
											__lowerCAmelCase				   =       rust_tokenizer.convert_ids_to_tokens(_A			)
											self.assertListEqual(_A		,			_A			)

											# fmt: off
											__lowerCAmelCase				   =       "I was born in 92000, and this is falsé."
											__lowerCAmelCase				   =       [1_3, 1, 2_3, 3_8_6, 1_9, 5_6_1, 3_0_5_0, 1_5, 1_7, 4_8, 2_5, 8_2_5_6, 1_8, 1, 9]
											__lowerCAmelCase				   =       ["▁", "I", "▁was", "▁born", "▁in", "▁9", "2000", ",", "▁and", "▁this", "▁is", "▁fal", "s", "é", ".", ]
											__lowerCAmelCase				   =       ["▁", "<unk>", "▁was", "▁born", "▁in", "▁9", "2000", ",", "▁and", "▁this", "▁is", "▁fal", "s", "<unk>", ".", ]
											# fmt: on

											__lowerCAmelCase				   =       tokenizer.encode(_A		,			add_special_tokens=_A			)
											self.assertListEqual(_A		,			_A			)
											__lowerCAmelCase				   =       tokenizer.tokenize(_A			)
											self.assertListEqual(_A		,			_A			)
											__lowerCAmelCase				   =       tokenizer.convert_ids_to_tokens(_A			)
											self.assertListEqual(_A		,			_A			)

											__lowerCAmelCase				   =       rust_tokenizer.encode(_A		,			add_special_tokens=_A			)
											self.assertListEqual(_A		,			_A			)
											__lowerCAmelCase				   =       rust_tokenizer.tokenize(_A			)
											self.assertListEqual(_A		,			_A			)
											__lowerCAmelCase				   =       rust_tokenizer.convert_ids_to_tokens(_A			)
											self.assertListEqual(_A		,			_A			)






						def 				__SCREAMING_SNAKE_CASE(   self			):




											"""simple docstring"""



											__lowerCAmelCase				   =       DebertaVaTokenizer(_A			)

											__lowerCAmelCase				   =       tokenizer.encode("sequence builders"			)
											__lowerCAmelCase				   =       tokenizer.encode("multi-sequence build"			)

											__lowerCAmelCase				   =       tokenizer.build_inputs_with_special_tokens(_A			)
											__lowerCAmelCase				   =       tokenizer.build_inputs_with_special_tokens(_A		,			_A			)

											self.assertEqual([tokenizer.cls_token_id] + text + [tokenizer.sep_token_id]		,			_A			)
											self.assertEqual(
											    [tokenizer.cls_token_id] + text + [tokenizer.sep_token_id] + text_a + [tokenizer.sep_token_id]		,			_A		,			)






						@slow
						def 				__SCREAMING_SNAKE_CASE(   self			):




											"""simple docstring"""



											__lowerCAmelCase				   =       {"input_ids": [[1, 3_9_8_6_7, 3_6, 1_9_3_9_0, 4_8_6, 2_7, 3_5_0_5_2, 8_1_4_3_6, 1_8, 6_0_6_8_5, 1_2_2_5, 7, 3_5_0_5_2, 8_1_4_3_6, 1_8, 9_3_6_7, 1_6_8_9_9, 1_8, 1_5_9_3_7, 5_3, 5_9_4, 7_7_3, 1_8, 1_6_2_8_7, 3_0_4_6_5, 3_6, 1_5_9_3_7, 6, 4_1_1_3_9, 3_8, 3_6_9_7_9, 6_0_7_6_3, 1_9_1, 6, 3_4_1_3_2, 9_9, 6, 5_0_5_3_8, 3_9_0, 4_3_2_3_0, 6, 3_4_1_3_2, 2_7_7_9, 2_0_8_5_0, 1_4, 6_9_9, 1_0_7_2, 1_1_9_4, 3_6, 3_8_2, 1_0_9_0_1, 5_3, 7, 6_9_9, 1_0_7_2, 2_0_8_4, 3_6, 2_0_4_2_2, 6_3_0, 5_3, 1_9, 1_0_5, 3_0_4_9, 1_8_9_6, 1_0_5_3, 1_6_8_9_9, 1_5_0_6, 1_1, 3_7_9_7_8, 4_2_4_3, 7, 1_2_3_7, 3_1_8_6_9, 2_0_0, 1_6_5_6_6, 6_5_4, 6, 3_5_0_5_2, 8_1_4_3_6, 7, 5_5_6_3_0, 1_3_5_9_3, 4, 2], [1, 2_6, 1_5_0_1_1, 1_3, 6_6_7, 8, 1_0_5_3, 1_8, 2_3_6_1_1, 1_2_3_7, 7_2_3_5_6, 1_2_8_2_0, 3_4, 1_0_4_1_3_4, 1_2_0_9, 3_5, 1_3_3_1_3, 6_6_2_7, 2_1, 2_0_2, 3_4_7, 7, 1_6_4, 2_3_9_9, 1_1, 4_6, 4_4_8_5, 4, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 5, 1_2_3_2, 2_8_6_4, 1_5_7_8_5, 1_4_9_5_1, 1_0_5, 5, 8_5_8_1, 1_2_5_0, 4, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]], "token_type_ids": [[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]], "attention_mask": [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]]}  # noqa: E501
											# fmt: on

											self.tokenizer_integration_test_util(
											    expected_encoding=_A		,			model_name="microsoft/deberta-v2-xlarge"		,			revision="ad6e42c1532ddf3a15c39246b63f5559d558b670"		,			)




