





import unittest

from transformers import DebertaVaConfig, is_torch_available
from transformers.testing_utils import require_sentencepiece, require_tokenizers, require_torch, slow, torch_device

from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, ids_tensor
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
  import torch

  from transformers import (
      DebertaVaForMaskedLM,
      DebertaVaForMultipleChoice,
      DebertaVaForQuestionAnswering,
      DebertaVaForSequenceClassification,
      DebertaVaForTokenClassification,
      DebertaVaModel,
  )
  from transformers.models.deberta_va.modeling_deberta_va import DEBERTA_V2_PRETRAINED_MODEL_ARCHIVE_LIST



class 					__snake_case	(  __SCREAMING_SNAKE_CASE    ):


    '''simple docstring'''







    def __init__(       self   ,				A_   ,				A_=13   ,				A_=7   ,				A_=True   ,				A_=True   ,				A_=True   ,				A_=True   ,				A_=99   ,				A_=32   ,				A_=5   ,				A_=4   ,				A_=37   ,				A_="gelu"   ,				A_=0.1   ,				A_=0.1   ,				A_=5_12   ,				A_=16   ,				A_=2   ,				A_=0.02   ,				A_=False   ,				A_=True   ,				A_="None"   ,				A_=3   ,				A_=4   ,				A_=None   ,				):





        '''simple docstring'''

        SCREAMING_SNAKE_CASE__											=			parent
        SCREAMING_SNAKE_CASE__											=			batch_size
        SCREAMING_SNAKE_CASE__											=			seq_length
        SCREAMING_SNAKE_CASE__											=			is_training
        SCREAMING_SNAKE_CASE__											=			use_input_mask
        SCREAMING_SNAKE_CASE__											=			use_token_type_ids
        SCREAMING_SNAKE_CASE__											=			use_labels
        SCREAMING_SNAKE_CASE__											=			vocab_size
        SCREAMING_SNAKE_CASE__											=			hidden_size
        SCREAMING_SNAKE_CASE__											=			num_hidden_layers
        SCREAMING_SNAKE_CASE__											=			num_attention_heads
        SCREAMING_SNAKE_CASE__											=			intermediate_size
        SCREAMING_SNAKE_CASE__											=			hidden_act
        SCREAMING_SNAKE_CASE__											=			hidden_dropout_prob
        SCREAMING_SNAKE_CASE__											=			attention_probs_dropout_prob
        SCREAMING_SNAKE_CASE__											=			max_position_embeddings
        SCREAMING_SNAKE_CASE__											=			type_vocab_size
        SCREAMING_SNAKE_CASE__											=			type_sequence_label_size
        SCREAMING_SNAKE_CASE__											=			initializer_range
        SCREAMING_SNAKE_CASE__											=			num_labels
        SCREAMING_SNAKE_CASE__											=			num_choices
        SCREAMING_SNAKE_CASE__											=			relative_attention
        SCREAMING_SNAKE_CASE__											=			position_biased_input
        SCREAMING_SNAKE_CASE__											=			pos_att_type
        SCREAMING_SNAKE_CASE__											=			scope



    def  lowercase_					(       self     ):





        '''simple docstring'''

        SCREAMING_SNAKE_CASE__											=			ids_tensor([self.batch_size, self.seq_length]   ,				self.vocab_size     )

        SCREAMING_SNAKE_CASE__											=			None
        if self.use_input_mask:
            SCREAMING_SNAKE_CASE__											=			ids_tensor([self.batch_size, self.seq_length]   ,				vocab_size=2     )

        SCREAMING_SNAKE_CASE__											=			None
        if self.use_token_type_ids:
            SCREAMING_SNAKE_CASE__											=			ids_tensor([self.batch_size, self.seq_length]   ,				self.type_vocab_size     )

        SCREAMING_SNAKE_CASE__											=			None
        SCREAMING_SNAKE_CASE__											=			None
        SCREAMING_SNAKE_CASE__											=			None
        if self.use_labels:
            SCREAMING_SNAKE_CASE__											=			ids_tensor([self.batch_size]   ,				self.type_sequence_label_size     )
            SCREAMING_SNAKE_CASE__											=			ids_tensor([self.batch_size, self.seq_length]   ,				self.num_labels     )
            SCREAMING_SNAKE_CASE__											=			ids_tensor([self.batch_size]   ,				self.num_choices     )

        SCREAMING_SNAKE_CASE__											=			self.get_config()

        return config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels



    def  lowercase_					(       self     ):





        '''simple docstring'''

        return DebertaVaConfig(
            vocab_size=self.vocab_size   ,				hidden_size=self.hidden_size   ,				num_hidden_layers=self.num_hidden_layers   ,				num_attention_heads=self.num_attention_heads   ,				intermediate_size=self.intermediate_size   ,				hidden_act=self.hidden_act   ,				hidden_dropout_prob=self.hidden_dropout_prob   ,				attention_probs_dropout_prob=self.attention_probs_dropout_prob   ,				max_position_embeddings=self.max_position_embeddings   ,				type_vocab_size=self.type_vocab_size   ,				initializer_range=self.initializer_range   ,				relative_attention=self.relative_attention   ,				position_biased_input=self.position_biased_input   ,				pos_att_type=self.pos_att_type   ,				)



    def  lowercase_					(       self   ,				A_     ):





        '''simple docstring'''

        self.parent.assertListEqual(list(result.loss.size()     )   ,				[]     )



    def  lowercase_					(       self   ,				A_   ,				A_   ,				A_   ,				A_   ,				A_   ,				A_   ,				A_     ):





        '''simple docstring'''

        SCREAMING_SNAKE_CASE__											=			DebertaVaModel(config=A_     )
        model.to(A_     )
        model.eval()
        SCREAMING_SNAKE_CASE__											=			model(A_   ,				attention_mask=A_   ,				token_type_ids=A_     )[0]
        SCREAMING_SNAKE_CASE__											=			model(A_   ,				token_type_ids=A_     )[0]
        SCREAMING_SNAKE_CASE__											=			model(A_     )[0]

        self.parent.assertListEqual(list(sequence_output.size()     )   ,				[self.batch_size, self.seq_length, self.hidden_size]     )



    def  lowercase_					(       self   ,				A_   ,				A_   ,				A_   ,				A_   ,				A_   ,				A_   ,				A_     ):





        '''simple docstring'''

        SCREAMING_SNAKE_CASE__											=			DebertaVaForMaskedLM(config=A_     )
        model.to(A_     )
        model.eval()
        SCREAMING_SNAKE_CASE__											=			model(A_   ,				attention_mask=A_   ,				token_type_ids=A_   ,				labels=A_     )

        self.parent.assertEqual(result.logits.shape   ,				(self.batch_size, self.seq_length, self.vocab_size)     )



    def  lowercase_					(       self   ,				A_   ,				A_   ,				A_   ,				A_   ,				A_   ,				A_   ,				A_     ):





        '''simple docstring'''

        SCREAMING_SNAKE_CASE__											=			self.num_labels
        SCREAMING_SNAKE_CASE__											=			DebertaVaForSequenceClassification(A_     )
        model.to(A_     )
        model.eval()
        SCREAMING_SNAKE_CASE__											=			model(A_   ,				attention_mask=A_   ,				token_type_ids=A_   ,				labels=A_     )
        self.parent.assertListEqual(list(result.logits.size()     )   ,				[self.batch_size, self.num_labels]     )
        self.check_loss_output(A_     )



    def  lowercase_					(       self   ,				A_   ,				A_   ,				A_   ,				A_   ,				A_   ,				A_   ,				A_     ):





        '''simple docstring'''

        SCREAMING_SNAKE_CASE__											=			self.num_labels
        SCREAMING_SNAKE_CASE__											=			DebertaVaForTokenClassification(config=A_     )
        model.to(A_     )
        model.eval()
        SCREAMING_SNAKE_CASE__											=			model(A_   ,				attention_mask=A_   ,				token_type_ids=A_   ,				labels=A_     )
        self.parent.assertEqual(result.logits.shape   ,				(self.batch_size, self.seq_length, self.num_labels)     )



    def  lowercase_					(       self   ,				A_   ,				A_   ,				A_   ,				A_   ,				A_   ,				A_   ,				A_     ):





        '''simple docstring'''

        SCREAMING_SNAKE_CASE__											=			DebertaVaForQuestionAnswering(config=A_     )
        model.to(A_     )
        model.eval()
        SCREAMING_SNAKE_CASE__											=			model(
            A_   ,				attention_mask=A_   ,				token_type_ids=A_   ,				start_positions=A_   ,				end_positions=A_   ,				)
        self.parent.assertEqual(result.start_logits.shape   ,				(self.batch_size, self.seq_length)     )
        self.parent.assertEqual(result.end_logits.shape   ,				(self.batch_size, self.seq_length)     )



    def  lowercase_					(       self   ,				A_   ,				A_   ,				A_   ,				A_   ,				A_   ,				A_   ,				A_     ):





        '''simple docstring'''

        SCREAMING_SNAKE_CASE__											=			DebertaVaForMultipleChoice(config=A_     )
        model.to(A_     )
        model.eval()
        SCREAMING_SNAKE_CASE__											=			input_ids.unsqueeze(1     ).expand(-1   ,				self.num_choices   ,				-1     ).contiguous()
        SCREAMING_SNAKE_CASE__											=			token_type_ids.unsqueeze(1     ).expand(-1   ,				self.num_choices   ,				-1     ).contiguous()
        SCREAMING_SNAKE_CASE__											=			input_mask.unsqueeze(1     ).expand(-1   ,				self.num_choices   ,				-1     ).contiguous()
        SCREAMING_SNAKE_CASE__											=			model(
            A_   ,				attention_mask=A_   ,				token_type_ids=A_   ,				labels=A_   ,				)
        self.parent.assertEqual(result.logits.shape   ,				(self.batch_size, self.num_choices)     )




    def  lowercase_					(       self     ):





        '''simple docstring'''

        SCREAMING_SNAKE_CASE__											=			self.prepare_config_and_inputs()
        (
            (
            SCREAMING_SNAKE_CASE__
        )				,		(
            SCREAMING_SNAKE_CASE__
        )				,		(
            SCREAMING_SNAKE_CASE__
        )				,		(
            SCREAMING_SNAKE_CASE__
        )				,		(
            SCREAMING_SNAKE_CASE__
        )				,		(
            SCREAMING_SNAKE_CASE__
        )				,		(
            SCREAMING_SNAKE_CASE__
        )				,		
        )											=			config_and_inputs
        SCREAMING_SNAKE_CASE__											=			{'''input_ids''': input_ids, '''token_type_ids''': token_type_ids, '''attention_mask''': input_mask}
        return config, inputs_dict





@require_torch
class 					__snake_case	(  __SCREAMING_SNAKE_CASE			,     __SCREAMING_SNAKE_CASE			,     unittest.TestCase    ):


    '''simple docstring'''







    lowerCamelCase__    :		List[str]				=	(
        (
            DebertaVaModel,
            DebertaVaForMaskedLM,
            DebertaVaForSequenceClassification,
            DebertaVaForTokenClassification,
            DebertaVaForQuestionAnswering,
            DebertaVaForMultipleChoice,
        )
        if is_torch_available()
        else ()
    )
    lowerCamelCase__    :		Tuple				=	(
        {
            """feature-extraction""": DebertaVaModel,
            """fill-mask""": DebertaVaForMaskedLM,
            """question-answering""": DebertaVaForQuestionAnswering,
            """text-classification""": DebertaVaForSequenceClassification,
            """token-classification""": DebertaVaForTokenClassification,
            """zero-shot""": DebertaVaForSequenceClassification,
        }
        if is_torch_available()
        else {}
    )

    lowerCamelCase__    :		str				=	True
    lowerCamelCase__    :		Union[str, Any]				=	False
    lowerCamelCase__    :		List[Any]				=	False
    lowerCamelCase__    :		Union[str, Any]				=	False
    lowerCamelCase__    :		Any				=	False



    def  lowercase_					(       self     ):





        '''simple docstring'''

        SCREAMING_SNAKE_CASE__											=			DebertaVaModelTester(self     )
        SCREAMING_SNAKE_CASE__											=			ConfigTester(self   ,				config_class=A_   ,				hidden_size=37     )



    def  lowercase_					(       self     ):





        '''simple docstring'''

        self.config_tester.run_common_tests()



    def  lowercase_					(       self     ):





        '''simple docstring'''

        SCREAMING_SNAKE_CASE__											=			self.model_tester.prepare_config_and_inputs()
        self.model_tester.create_and_check_deberta_model(*A_     )



    def  lowercase_					(       self     ):





        '''simple docstring'''

        SCREAMING_SNAKE_CASE__											=			self.model_tester.prepare_config_and_inputs()
        self.model_tester.create_and_check_deberta_for_sequence_classification(*A_     )



    def  lowercase_					(       self     ):





        '''simple docstring'''

        SCREAMING_SNAKE_CASE__											=			self.model_tester.prepare_config_and_inputs()
        self.model_tester.create_and_check_deberta_for_masked_lm(*A_     )



    def  lowercase_					(       self     ):





        '''simple docstring'''

        SCREAMING_SNAKE_CASE__											=			self.model_tester.prepare_config_and_inputs()
        self.model_tester.create_and_check_deberta_for_question_answering(*A_     )



    def  lowercase_					(       self     ):





        '''simple docstring'''

        SCREAMING_SNAKE_CASE__											=			self.model_tester.prepare_config_and_inputs()
        self.model_tester.create_and_check_deberta_for_token_classification(*A_     )



    def  lowercase_					(       self     ):





        '''simple docstring'''

        SCREAMING_SNAKE_CASE__											=			self.model_tester.prepare_config_and_inputs()
        self.model_tester.create_and_check_deberta_for_multiple_choice(*A_     )




    @slow
    def  lowercase_					(       self     ):





        '''simple docstring'''

        for model_name in DEBERTA_V2_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
            SCREAMING_SNAKE_CASE__											=			DebertaVaModel.from_pretrained(A_     )
            self.assertIsNotNone(A_     )





@require_torch
@require_sentencepiece
@require_tokenizers
class 					__snake_case	(  unittest.TestCase    ):


    '''simple docstring'''







    @unittest.skip(reason='''Model not available yet'''     )
    def  lowercase_					(       self     ):





        '''simple docstring'''

        pass




    @slow
    def  lowercase_					(       self     ):





        '''simple docstring'''

        SCREAMING_SNAKE_CASE__											=			DebertaVaModel.from_pretrained('''microsoft/deberta-v2-xlarge'''     )

        SCREAMING_SNAKE_CASE__											=			torch.tensor([[0, 3_14_14, 2_32, 3_28, 7_40, 11_40, 1_26_95, 69, 4_60_78, 15_88, 2]]     )
        SCREAMING_SNAKE_CASE__											=			torch.tensor([[0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]]     )
        with torch.no_grad():
            SCREAMING_SNAKE_CASE__											=			model(A_   ,				attention_mask=A_     )[0]
        # compare the actual values for a slice.
        SCREAMING_SNAKE_CASE__											=			torch.tensor(
            [[[0.2356, 0.1948, 0.0369], [-0.1063, 0.3586, -0.5152], [-0.6399, -0.0259, -0.2525]]]     )
        self.assertTrue(torch.allclose(output[:, 1:4, 1:4]   ,				A_   ,				atol=1E-4     )   ,				f'''{output[:, 1:4, 1:4]}'''     )

'''simple docstring'''

from __future__ import annotations

import unittest

from transformers import RoFormerConfig, is_tf_available
from transformers.testing_utils import require_tf, slow

from ...test_configuration_common import ConfigTester
from ...test_modeling_tf_common import TFModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_tf_available():
  import tensorflow as tf

  from transformers import (
      TFRoFormerForCausalLM,
      TFRoFormerForMaskedLM,
      TFRoFormerForMultipleChoice,
      TFRoFormerForQuestionAnswering,
      TFRoFormerForSequenceClassification,
      TFRoFormerForTokenClassification,
      TFRoFormerModel,
  )
  from transformers.models.roformer.modeling_tf_roformer import (
      TFRoFormerSelfAttention,
      TFRoFormerSinusoidalPositionalEmbedding,
  )


class 					_lowercase						:
      '''simple docstring'''


      def __init__(					self       ,lowerCamelCase_       ,lowerCamelCase_=13       ,lowerCamelCase_=7       ,lowerCamelCase_=True       ,lowerCamelCase_=True       ,lowerCamelCase_=True       ,lowerCamelCase_=True       ,lowerCamelCase_=99       ,lowerCamelCase_=32       ,lowerCamelCase_=2       ,lowerCamelCase_=4       ,lowerCamelCase_=37       ,lowerCamelCase_="gelu"       ,lowerCamelCase_=0.1       ,lowerCamelCase_=0.1       ,lowerCamelCase_=512       ,lowerCamelCase_=16       ,lowerCamelCase_=2       ,lowerCamelCase_=0.02       ,lowerCamelCase_=3       ,lowerCamelCase_=4       ,lowerCamelCase_=None       ,)      ->      str:



         '''simple docstring'''



         UpperCAmelCase__				:					Any 					=	parent
         UpperCAmelCase__				:					str 					=	13
         UpperCAmelCase__				:					Any 					=	7
         UpperCAmelCase__				:					str 					=	True
         UpperCAmelCase__				:					List[Any] 					=	True
         UpperCAmelCase__				:					Tuple 					=	True
         UpperCAmelCase__				:					List[str] 					=	True
         UpperCAmelCase__				:					List[Any] 					=	99
         UpperCAmelCase__				:					str 					=	32
         UpperCAmelCase__				:					Dict 					=	2
         UpperCAmelCase__				:					Union[str, Any] 					=	4
         UpperCAmelCase__				:					Dict 					=	37
         UpperCAmelCase__				:					Dict 					=	'''gelu'''
         UpperCAmelCase__				:					List[str] 					=	0.1
         UpperCAmelCase__				:					List[str] 					=	0.1
         UpperCAmelCase__				:					Optional[int] 					=	512
         UpperCAmelCase__				:					Any 					=	16
         UpperCAmelCase__				:					List[Any] 					=	2
         UpperCAmelCase__				:					Optional[Any] 					=	0.02
         UpperCAmelCase__				:					Optional[int] 					=	3
         UpperCAmelCase__				:					Optional[int] 					=	4
         UpperCAmelCase__				:					Optional[int] 					=	None




      def 		lowerCAmelCase__       (					self				)      ->      Optional[int]:



         '''simple docstring'''



         UpperCAmelCase__				:					Tuple 					=	ids_tensor([self.batch_size, self.seq_length]       ,self.vocab_size				)

         UpperCAmelCase__				:					List[str] 					=	None
         if self.use_input_mask:
            UpperCAmelCase__				:					int 					=	random_attention_mask([self.batch_size, self.seq_length]				)

         UpperCAmelCase__				:					str 					=	None
         if self.use_token_type_ids:
            UpperCAmelCase__				:					Dict 					=	ids_tensor([self.batch_size, self.seq_length]       ,self.type_vocab_size				)

         UpperCAmelCase__				:					List[str] 					=	None
         UpperCAmelCase__				:					Any 					=	None
         UpperCAmelCase__				:					Dict 					=	None
         if self.use_labels:
            UpperCAmelCase__				:					Optional[Any] 					=	ids_tensor([self.batch_size]       ,self.type_sequence_label_size				)
            UpperCAmelCase__				:					Dict 					=	ids_tensor([self.batch_size, self.seq_length]       ,self.num_labels				)
            UpperCAmelCase__				:					List[str] 					=	ids_tensor([self.batch_size]       ,self.num_choices				)

         UpperCAmelCase__				:					Union[str, Any] 					=	RoFormerConfig(
             vocab_size=self.vocab_size       ,hidden_size=self.hidden_size       ,num_hidden_layers=self.num_hidden_layers       ,num_attention_heads=self.num_attention_heads       ,intermediate_size=self.intermediate_size       ,hidden_act=self.hidden_act       ,hidden_dropout_prob=self.hidden_dropout_prob       ,attention_probs_dropout_prob=self.attention_probs_dropout_prob       ,max_position_embeddings=self.max_position_embeddings       ,type_vocab_size=self.type_vocab_size       ,initializer_range=self.initializer_range       ,return_dict=lowerCamelCase_       ,)

         return config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels




      def 		lowerCAmelCase__       (					self       ,lowerCamelCase_       ,lowerCamelCase_       ,lowerCamelCase_       ,lowerCamelCase_       ,lowerCamelCase_       ,lowerCamelCase_       ,lowerCamelCase_				)      ->      int:



         '''simple docstring'''



         UpperCAmelCase__				:					List[str] 					=	TFRoFormerModel(config=lowerCamelCase_				)
         UpperCAmelCase__				:					str 					=	{'''input_ids''': input_ids, '''attention_mask''': input_mask, '''token_type_ids''': token_type_ids}

         UpperCAmelCase__				:					str 					=	[input_ids, input_mask]
         UpperCAmelCase__				:					Any 					=	model(lowerCamelCase_				)

         UpperCAmelCase__				:					List[Any] 					=	model(lowerCamelCase_				)

         self.parent.assertEqual(result.last_hidden_state.shape       ,(self.batch_size, self.seq_length, self.hidden_size)				)




      def 		lowerCAmelCase__       (					self       ,lowerCamelCase_       ,lowerCamelCase_       ,lowerCamelCase_       ,lowerCamelCase_       ,lowerCamelCase_       ,lowerCamelCase_       ,lowerCamelCase_				)      ->      Tuple:



         '''simple docstring'''



         UpperCAmelCase__				:					Optional[int] 					=	True
         UpperCAmelCase__				:					Any 					=	TFRoFormerForCausalLM(config=lowerCamelCase_				)
         UpperCAmelCase__				:					List[str] 					=	{
             '''input_ids''': input_ids,
             '''attention_mask''': input_mask,
             '''token_type_ids''': token_type_ids,
         }
         UpperCAmelCase__				:					int 					=	model(lowerCamelCase_				)['''logits''']
         self.parent.assertListEqual(
             list(prediction_scores.numpy().shape				)       ,[self.batch_size, self.seq_length, self.vocab_size]				)




      def 		lowerCAmelCase__       (					self       ,lowerCamelCase_       ,lowerCamelCase_       ,lowerCamelCase_       ,lowerCamelCase_       ,lowerCamelCase_       ,lowerCamelCase_       ,lowerCamelCase_				)      ->      List[str]:



         '''simple docstring'''



         UpperCAmelCase__				:					Union[str, Any] 					=	TFRoFormerForMaskedLM(config=lowerCamelCase_				)
         UpperCAmelCase__				:					Dict 					=	{
             '''input_ids''': input_ids,
             '''attention_mask''': input_mask,
             '''token_type_ids''': token_type_ids,
         }
         UpperCAmelCase__				:					Dict 					=	model(lowerCamelCase_				)
         self.parent.assertEqual(result.logits.shape       ,(self.batch_size, self.seq_length, self.vocab_size)				)




      def 		lowerCAmelCase__       (					self       ,lowerCamelCase_       ,lowerCamelCase_       ,lowerCamelCase_       ,lowerCamelCase_       ,lowerCamelCase_       ,lowerCamelCase_       ,lowerCamelCase_				)      ->      Union[str, Any]:



         '''simple docstring'''



         UpperCAmelCase__				:					Optional[Any] 					=	self.num_labels
         UpperCAmelCase__				:					List[Any] 					=	TFRoFormerForSequenceClassification(config=lowerCamelCase_				)
         UpperCAmelCase__				:					List[Any] 					=	{
             '''input_ids''': input_ids,
             '''attention_mask''': input_mask,
             '''token_type_ids''': token_type_ids,
         }

         UpperCAmelCase__				:					Union[str, Any] 					=	model(lowerCamelCase_				)
         self.parent.assertEqual(result.logits.shape       ,(self.batch_size, self.num_labels)				)




      def 		lowerCAmelCase__       (					self       ,lowerCamelCase_       ,lowerCamelCase_       ,lowerCamelCase_       ,lowerCamelCase_       ,lowerCamelCase_       ,lowerCamelCase_       ,lowerCamelCase_				)      ->      Dict:



         '''simple docstring'''



         UpperCAmelCase__				:					int 					=	self.num_choices
         UpperCAmelCase__				:					Optional[int] 					=	TFRoFormerForMultipleChoice(config=lowerCamelCase_				)
         UpperCAmelCase__				:					Tuple 					=	tf.tile(tf.expand_dims(lowerCamelCase_       ,1				)       ,(1, self.num_choices, 1)				)
         UpperCAmelCase__				:					Optional[Any] 					=	tf.tile(tf.expand_dims(lowerCamelCase_       ,1				)       ,(1, self.num_choices, 1)				)
         UpperCAmelCase__				:					Dict 					=	tf.tile(tf.expand_dims(lowerCamelCase_       ,1				)       ,(1, self.num_choices, 1)				)
         UpperCAmelCase__				:					Optional[int] 					=	{
             '''input_ids''': multiple_choice_inputs_ids,
             '''attention_mask''': multiple_choice_input_mask,
             '''token_type_ids''': multiple_choice_token_type_ids,
         }
         UpperCAmelCase__				:					str 					=	model(lowerCamelCase_				)
         self.parent.assertEqual(result.logits.shape       ,(self.batch_size, self.num_choices)				)




      def 		lowerCAmelCase__       (					self       ,lowerCamelCase_       ,lowerCamelCase_       ,lowerCamelCase_       ,lowerCamelCase_       ,lowerCamelCase_       ,lowerCamelCase_       ,lowerCamelCase_				)      ->      Any:



         '''simple docstring'''



         UpperCAmelCase__				:					Dict 					=	self.num_labels
         UpperCAmelCase__				:					Optional[Any] 					=	TFRoFormerForTokenClassification(config=lowerCamelCase_				)
         UpperCAmelCase__				:					Dict 					=	{
             '''input_ids''': input_ids,
             '''attention_mask''': input_mask,
             '''token_type_ids''': token_type_ids,
         }
         UpperCAmelCase__				:					Optional[int] 					=	model(lowerCamelCase_				)
         self.parent.assertEqual(result.logits.shape       ,(self.batch_size, self.seq_length, self.num_labels)				)




      def 		lowerCAmelCase__       (					self       ,lowerCamelCase_       ,lowerCamelCase_       ,lowerCamelCase_       ,lowerCamelCase_       ,lowerCamelCase_       ,lowerCamelCase_       ,lowerCamelCase_				)      ->      Optional[int]:



         '''simple docstring'''



         UpperCAmelCase__				:					Optional[Any] 					=	TFRoFormerForQuestionAnswering(config=lowerCamelCase_				)
         UpperCAmelCase__				:					Any 					=	{
             '''input_ids''': input_ids,
             '''attention_mask''': input_mask,
             '''token_type_ids''': token_type_ids,
         }

         UpperCAmelCase__				:					int 					=	model(lowerCamelCase_				)
         self.parent.assertEqual(result.start_logits.shape       ,(self.batch_size, self.seq_length)				)
         self.parent.assertEqual(result.end_logits.shape       ,(self.batch_size, self.seq_length)				)




      def 		lowerCAmelCase__       (					self				)      ->      Any:



         '''simple docstring'''



         UpperCAmelCase__				:					Optional[Any] 					=	self.prepare_config_and_inputs()
         (
             (
             UpperCAmelCase__
         )					,   (
             UpperCAmelCase__
         )					,   (
             UpperCAmelCase__
         )					,   (
             UpperCAmelCase__
         )					,   (
             UpperCAmelCase__
         )					,   (
             UpperCAmelCase__
         )					,   (
             UpperCAmelCase__
         )					,   
         )				:					List[Any] 					=	config_and_inputs
         UpperCAmelCase__				:					Optional[int] 					=	{'''input_ids''': input_ids, '''token_type_ids''': token_type_ids, '''attention_mask''': input_mask}
         return config, inputs_dict




@require_tf
class 					_lowercase						(   lowerCAmelCase		,lowerCAmelCase		,unittest.TestCase ):
      '''simple docstring'''


      UpperCAmelCase_   :   List[str]   				= (
          (
              TFRoFormerModel,
              TFRoFormerForCausalLM,
              TFRoFormerForMaskedLM,
              TFRoFormerForQuestionAnswering,
              TFRoFormerForSequenceClassification,
              TFRoFormerForTokenClassification,
              TFRoFormerForMultipleChoice,
          )
          if is_tf_available()
          else ()
      )
      UpperCAmelCase_   :   int   				= (
          {
              '''feature-extraction''': TFRoFormerModel,
              '''fill-mask''': TFRoFormerForMaskedLM,
              '''question-answering''': TFRoFormerForQuestionAnswering,
              '''text-classification''': TFRoFormerForSequenceClassification,
              '''text-generation''': TFRoFormerForCausalLM,
              '''token-classification''': TFRoFormerForTokenClassification,
              '''zero-shot''': TFRoFormerForSequenceClassification,
          }
          if is_tf_available()
          else {}
      )

      UpperCAmelCase_   :   Optional[int]   				= False
      UpperCAmelCase_   :   Optional[int]   				= False




      def 		lowerCAmelCase__       (					self       ,lowerCamelCase_       ,lowerCamelCase_       ,lowerCamelCase_       ,lowerCamelCase_       ,lowerCamelCase_				)      ->      Any:



         '''simple docstring'''



         if pipeline_test_casse_name == "TextGenerationPipelineTests":
            return True

         return False




      def 		lowerCAmelCase__       (					self				)      ->      Union[str, Any]:



         '''simple docstring'''



         UpperCAmelCase__				:					List[Any] 					=	TFRoFormerModelTester(self				)
         UpperCAmelCase__				:					int 					=	ConfigTester(self       ,config_class=lowerCamelCase_       ,hidden_size=37				)




      def 		lowerCAmelCase__       (					self				)      ->      Dict:



         '''simple docstring'''



         self.config_tester.run_common_tests()




      def 		lowerCAmelCase__       (					self				)      ->      List[Any]:



         '''simple docstring'''



         UpperCAmelCase__				:					Any 					=	self.model_tester.prepare_config_and_inputs()
         self.model_tester.create_and_check_model(*lowerCamelCase_				)




      def 		lowerCAmelCase__       (					self				)      ->      Optional[Any]:



         '''simple docstring'''



         UpperCAmelCase__				:					Tuple 					=	self.model_tester.prepare_config_and_inputs()
         self.model_tester.create_and_check_for_masked_lm(*lowerCamelCase_				)




      def 		lowerCAmelCase__       (					self				)      ->      List[Any]:



         '''simple docstring'''



         UpperCAmelCase__				:					str 					=	self.model_tester.prepare_config_and_inputs()
         self.model_tester.create_and_check_lm_head(*lowerCamelCase_				)




      def 		lowerCAmelCase__       (					self				)      ->      Optional[Any]:



         '''simple docstring'''



         UpperCAmelCase__				:					Optional[int] 					=	self.model_tester.prepare_config_and_inputs()
         self.model_tester.create_and_check_for_multiple_choice(*lowerCamelCase_				)




      def 		lowerCAmelCase__       (					self				)      ->      Any:



         '''simple docstring'''



         UpperCAmelCase__				:					List[str] 					=	self.model_tester.prepare_config_and_inputs()
         self.model_tester.create_and_check_for_question_answering(*lowerCamelCase_				)




      def 		lowerCAmelCase__       (					self				)      ->      str:



         '''simple docstring'''



         UpperCAmelCase__				:					Optional[int] 					=	self.model_tester.prepare_config_and_inputs()
         self.model_tester.create_and_check_for_sequence_classification(*lowerCamelCase_				)




      def 		lowerCAmelCase__       (					self				)      ->      List[Any]:



         '''simple docstring'''



         UpperCAmelCase__				:					str 					=	self.model_tester.prepare_config_and_inputs()
         self.model_tester.create_and_check_for_token_classification(*lowerCamelCase_				)




      @slow
      def 		lowerCAmelCase__       (					self				)      ->      Union[str, Any]:



         '''simple docstring'''



         UpperCAmelCase__				:					List[str] 					=	TFRoFormerModel.from_pretrained('''junnyu/roformer_chinese_base'''				)
         self.assertIsNotNone(lowerCamelCase_				)




@require_tf
class 					_lowercase						(   unittest.TestCase ):
      '''simple docstring'''


      @slow
      def 		lowerCAmelCase__       (					self				)      ->      Dict:



         '''simple docstring'''



         UpperCAmelCase__				:					List[Any] 					=	TFRoFormerForMaskedLM.from_pretrained('''junnyu/roformer_chinese_base'''				)
         UpperCAmelCase__				:					Dict 					=	tf.constant([[0, 1, 2, 3, 4, 5]]				)
         UpperCAmelCase__				:					Union[str, Any] 					=	model(lowerCamelCase_				)[0]

         # TODO Replace vocab size
         UpperCAmelCase__				:					List[str] 					=	50000

         UpperCAmelCase__				:					int 					=	[1, 6, vocab_size]
         self.assertEqual(output.shape       ,lowerCamelCase_				)

         print(output[:, :3, :3]				)

         # TODO Replace values below with what was printed above.
         UpperCAmelCase__				:					Tuple 					=	tf.constant(
             [
                 [
                     [-0.1205_3341, -1.026_4901, 0.2922_1946],
                     [-1.513_3783, 0.19_7433, 0.1519_0607],
                     [-5.013_5403, -3.90_0256, -0.8403_8764],
                 ]
             ]				)
         tf.debugging.assert_near(output[:, :3, :3]       ,lowerCamelCase_       ,atol=1e-4				)




@require_tf
class 					_lowercase						(   unittest.TestCase ):
      '''simple docstring'''


      UpperCAmelCase_   :   int   				= 1E-4




      def 		lowerCAmelCase__       (					self				)      ->      Optional[Any]:



         '''simple docstring'''



         UpperCAmelCase__				:					Union[str, Any] 					=	tf.constant([[4, 10]]				)
         UpperCAmelCase__				:					List[str] 					=	TFRoFormerSinusoidalPositionalEmbedding(num_positions=6       ,embedding_dim=6				)

         UpperCAmelCase__				:					Optional[Any] 					=	emba(input_ids.shape				)
         UpperCAmelCase__				:					List[str] 					=	tf.constant(
             [[0.0000, 0.0000, 0.0000, 1.0000, 1.0000, 1.0000], [0.8415, 0.0464, 0.0022, 0.5403, 0.9989, 1.0000]]				)

         tf.debugging.assert_near(lowerCamelCase_       ,lowerCamelCase_       ,atol=self.tolerance				)




      def 		lowerCAmelCase__       (					self				)      ->      List[str]:



         '''simple docstring'''



         UpperCAmelCase__				:					Any 					=	tf.constant(
             [
                 [0.0000, 0.0000, 0.0000, 0.0000, 0.0000],
                 [0.8415, 0.8219, 0.8020, 0.7819, 0.7617],
                 [0.9093, 0.9364, 0.9581, 0.9749, 0.9870],
             ]				)
         UpperCAmelCase__				:					Dict 					=	TFRoFormerSinusoidalPositionalEmbedding(num_positions=512       ,embedding_dim=512				)
         emba([2, 16, 512]				)
         UpperCAmelCase__				:					Optional[int] 					=	emba.weight[:3, :5]

         tf.debugging.assert_near(lowerCamelCase_       ,lowerCamelCase_       ,atol=self.tolerance				)




@require_tf
class 					_lowercase						(   unittest.TestCase ):
      '''simple docstring'''


      UpperCAmelCase_   :   Optional[int]   				= 1E-4




      def 		lowerCAmelCase__       (					self				)      ->      List[str]:



         '''simple docstring'''



         UpperCAmelCase__				:					Optional[int] 					=	tf.reshape(tf.range(2 * 12 * 16 * 64       ,dtype=tf.floataa				)       ,shape=(2, 12, 16, 64)				) / 100

         UpperCAmelCase__				:					int 					=	-tf.reshape(tf.range(2 * 12 * 16 * 64       ,dtype=tf.floataa				)       ,shape=(2, 12, 16, 64)				) / 100

         UpperCAmelCase__				:					Optional[Any] 					=	TFRoFormerSinusoidalPositionalEmbedding(num_positions=32       ,embedding_dim=64				)
         UpperCAmelCase__				:					int 					=	embed_positions([2, 16, 768]				)[None, None, :, :]

         UpperCAmelCase__					,   UpperCAmelCase__				:					Dict 					=	TFRoFormerSelfAttention.apply_rotary_position_embeddings(
             lowerCamelCase_       ,lowerCamelCase_       ,lowerCamelCase_				)

         UpperCAmelCase__				:					Any 					=	tf.constant(
             [
                 [0.0000, 0.0100, 0.0200, 0.0300, 0.0400, 0.0500, 0.0600, 0.0700],
                 [-0.2012, 0.8897, 0.0263, 0.9401, 0.2074, 0.9463, 0.3481, 0.9343],
                 [-1.7057, 0.6271, -1.2145, 1.3897, -0.6303, 1.7647, -0.1173, 1.8985],
                 [-2.1731, -1.6397, -2.7358, 0.2854, -2.1840, 1.7183, -1.3018, 2.4871],
                 [0.2717, -3.6173, -2.9206, -2.1988, -3.6638, 0.3858, -2.9155, 2.2980],
                 [3.9859, -2.1580, -0.7984, -4.4904, -4.1181, -2.0252, -4.4782, 1.1253],
             ]				)
         UpperCAmelCase__				:					List[str] 					=	tf.constant(
             [
                 [0.0000, -0.0100, -0.0200, -0.0300, -0.0400, -0.0500, -0.0600, -0.0700],
                 [0.2012, -0.8897, -0.0263, -0.9401, -0.2074, -0.9463, -0.3481, -0.9343],
                 [1.7057, -0.6271, 1.2145, -1.3897, 0.6303, -1.7647, 0.1173, -1.8985],
                 [2.1731, 1.6397, 2.7358, -0.2854, 2.1840, -1.7183, 1.3018, -2.4871],
                 [-0.2717, 3.6173, 2.9206, 2.1988, 3.6638, -0.3858, 2.9155, -2.2980],
                 [-3.9859, 2.1580, 0.7984, 4.4904, 4.1181, 2.0252, 4.4782, -1.1253],
             ]				)

         tf.debugging.assert_near(query_layer[0, 0, :6, :8]       ,lowerCamelCase_       ,atol=self.tolerance				)
         tf.debugging.assert_near(key_layer[0, 0, :6, :8]       ,lowerCamelCase_       ,atol=self.tolerance				)




