



from __future__ import annotations

import unittest

from transformers import DebertaVaConfig, is_tf_available
from transformers.testing_utils import require_tf, slow

from ...test_configuration_common import ConfigTester
from ...test_modeling_tf_common import TFModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_tf_available():
     import tensorflow as tf

     from transformers import (
         TFDebertaVaForMaskedLM,
         TFDebertaVaForQuestionAnswering,
         TFDebertaVaForSequenceClassification,
         TFDebertaVaForTokenClassification,
         TFDebertaVaModel,
     )






class SCREAMING_SNAKE_CASE__     :




    '''simple docstring'''






    def __init__(       self,		lowerCamelCase__,		lowerCamelCase__=13,		lowerCamelCase__=7,		lowerCamelCase__=True,		lowerCamelCase__=True,		lowerCamelCase__=True,		lowerCamelCase__=True,		lowerCamelCase__=99,		lowerCamelCase__=32,		lowerCamelCase__=2,		lowerCamelCase__=4,		lowerCamelCase__=37,		lowerCamelCase__="gelu",		lowerCamelCase__=0.1,		lowerCamelCase__=0.1,		lowerCamelCase__=512,		lowerCamelCase__=16,		lowerCamelCase__=2,		lowerCamelCase__=0.02,		lowerCamelCase__=False,		lowerCamelCase__=True,		lowerCamelCase__="None",		lowerCamelCase__=3,		lowerCamelCase__=4,		lowerCamelCase__=None,		):
        A :       Optional[Any]          =   parent
        A :       Any          =   batch_size
        A :       Optional[int]          =   seq_length
        A :       List[str]          =   is_training
        A :       List[str]          =   use_input_mask
        A :       List[Any]          =   use_token_type_ids
        A :       Any          =   use_labels
        A :       Union[str, Any]          =   vocab_size
        A :       str          =   hidden_size
        A :       Tuple          =   num_hidden_layers
        A :       str          =   num_attention_heads
        A :       Optional[Any]          =   intermediate_size
        A :       Tuple          =   hidden_act
        A :       List[str]          =   hidden_dropout_prob
        A :       Any          =   attention_probs_dropout_prob
        A :       Any          =   max_position_embeddings
        A :       Any          =   type_vocab_size
        A :       str          =   type_sequence_label_size
        A :       List[str]          =   initializer_range
        A :       Optional[Any]          =   num_labels
        A :       int          =   num_choices
        A :       int          =   relative_attention
        A :       Dict          =   position_biased_input
        A :       Any          =   pos_att_type
        A :       List[Any]          =   scope






    def 			_lowerCAmelCase					(       self		):
        A :       str          =   ids_tensor([self.batch_size, self.seq_length],		self.vocab_size		)

        A :       Optional[int]          =   None
        if self.use_input_mask:
            A :       Union[str, Any]          =   random_attention_mask([self.batch_size, self.seq_length]		)

        A :       Tuple          =   None
        if self.use_token_type_ids:
            A :       Dict          =   ids_tensor([self.batch_size, self.seq_length],		self.type_vocab_size		)

        A :       str          =   None
        A :       List[str]          =   None
        A :       List[Any]          =   None
        if self.use_labels:
            A :       Dict          =   ids_tensor([self.batch_size],		self.type_sequence_label_size		)
            A :       Tuple          =   ids_tensor([self.batch_size, self.seq_length],		self.num_labels		)

        A :       Dict          =   DebertaVaConfig(
            vocab_size=self.vocab_size,		hidden_size=self.hidden_size,		num_hidden_layers=self.num_hidden_layers,		num_attention_heads=self.num_attention_heads,		intermediate_size=self.intermediate_size,		hidden_act=self.hidden_act,		hidden_dropout_prob=self.hidden_dropout_prob,		attention_probs_dropout_prob=self.attention_probs_dropout_prob,		max_position_embeddings=self.max_position_embeddings,		type_vocab_size=self.type_vocab_size,		relative_attention=self.relative_attention,		position_biased_input=self.position_biased_input,		initializer_range=self.initializer_range,		return_dict=lowerCamelCase__,		)

        return config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels






    def 			_lowerCAmelCase					(       self,		lowerCamelCase__,		lowerCamelCase__,		lowerCamelCase__,		lowerCamelCase__,		lowerCamelCase__,		lowerCamelCase__,		lowerCamelCase__		):
        A :       str          =   TFDebertaVaModel(config=lowerCamelCase__		)
        A :       Optional[Any]          =   {"""input_ids""": input_ids, """attention_mask""": input_mask, """token_type_ids""": token_type_ids}

        A :       Union[str, Any]          =   [input_ids, input_mask]
        A :       Optional[int]          =   model(lowerCamelCase__		)

        A :       int          =   model(lowerCamelCase__		)

        self.parent.assertEqual(result.last_hidden_state.shape,		(self.batch_size, self.seq_length, self.hidden_size)		)






    def 			_lowerCAmelCase					(       self,		lowerCamelCase__,		lowerCamelCase__,		lowerCamelCase__,		lowerCamelCase__,		lowerCamelCase__,		lowerCamelCase__,		lowerCamelCase__		):
        A :       List[str]          =   TFDebertaVaForMaskedLM(config=lowerCamelCase__		)
        A :       Any          =   {
            """input_ids""": input_ids,
            """attention_mask""": input_mask,
            """token_type_ids""": token_type_ids,
        }
        A :       Dict          =   model(lowerCamelCase__		)
        self.parent.assertEqual(result.logits.shape,		(self.batch_size, self.seq_length, self.vocab_size)		)






    def 			_lowerCAmelCase					(       self,		lowerCamelCase__,		lowerCamelCase__,		lowerCamelCase__,		lowerCamelCase__,		lowerCamelCase__,		lowerCamelCase__,		lowerCamelCase__		):
        A :       Tuple          =   self.num_labels
        A :       List[str]          =   TFDebertaVaForSequenceClassification(config=lowerCamelCase__		)
        A :       Optional[int]          =   {
            """input_ids""": input_ids,
            """attention_mask""": input_mask,
            """token_type_ids""": token_type_ids,
        }

        A :       Optional[Any]          =   model(lowerCamelCase__		)
        self.parent.assertEqual(result.logits.shape,		(self.batch_size, self.num_labels)		)






    def 			_lowerCAmelCase					(       self,		lowerCamelCase__,		lowerCamelCase__,		lowerCamelCase__,		lowerCamelCase__,		lowerCamelCase__,		lowerCamelCase__,		lowerCamelCase__		):
        A :       int          =   self.num_labels
        A :       Tuple          =   TFDebertaVaForTokenClassification(config=lowerCamelCase__		)
        A :       int          =   {
            """input_ids""": input_ids,
            """attention_mask""": input_mask,
            """token_type_ids""": token_type_ids,
        }
        A :       Union[str, Any]          =   model(lowerCamelCase__		)
        self.parent.assertEqual(result.logits.shape,		(self.batch_size, self.seq_length, self.num_labels)		)






    def 			_lowerCAmelCase					(       self,		lowerCamelCase__,		lowerCamelCase__,		lowerCamelCase__,		lowerCamelCase__,		lowerCamelCase__,		lowerCamelCase__,		lowerCamelCase__		):
        A :       Tuple          =   TFDebertaVaForQuestionAnswering(config=lowerCamelCase__		)
        A :       Optional[Any]          =   {
            """input_ids""": input_ids,
            """attention_mask""": input_mask,
            """token_type_ids""": token_type_ids,
        }

        A :       Optional[Any]          =   model(lowerCamelCase__		)
        self.parent.assertEqual(result.start_logits.shape,		(self.batch_size, self.seq_length)		)
        self.parent.assertEqual(result.end_logits.shape,		(self.batch_size, self.seq_length)		)






    def 			_lowerCAmelCase					(       self		):
        A :       Any          =   self.prepare_config_and_inputs()
        (
            (
            A
        )     ,							(
            A
        )     ,							(
            A
        )     ,							(
            A
        )     ,							(
            A
        )     ,							(
            A
        )     ,							(
            A
        )     ,							
        ) :       List[Any]          =   config_and_inputs
        A :       str          =   {"""input_ids""": input_ids, """token_type_ids""": token_type_ids, """attention_mask""": input_mask}
        return config, inputs_dict






@require_tf
class SCREAMING_SNAKE_CASE__     (		SCREAMING_SNAKE_CASE__ ,						SCREAMING_SNAKE_CASE__ ,						unittest.TestCase    ):




    '''simple docstring'''






    __lowerCamelCase			: Dict					=      (
        (
            TFDebertaVaModel,
            TFDebertaVaForMaskedLM,
            TFDebertaVaForQuestionAnswering,
            TFDebertaVaForSequenceClassification,
            TFDebertaVaForTokenClassification,
        )
        if is_tf_available()
        else ()
    )
    __lowerCamelCase			: Tuple					=      (
        {
            "feature-extraction": TFDebertaVaModel,
            "fill-mask": TFDebertaVaForMaskedLM,
            "question-answering": TFDebertaVaForQuestionAnswering,
            "text-classification": TFDebertaVaForSequenceClassification,
            "token-classification": TFDebertaVaForTokenClassification,
            "zero-shot": TFDebertaVaForSequenceClassification,
        }
        if is_tf_available()
        else {}
    )

    __lowerCamelCase			: Tuple					=      False
    __lowerCamelCase			: str					=      False






    def 			_lowerCAmelCase					(       self		):
        A :       List[Any]          =   TFDebertaVaModelTester(self		)
        A :       Optional[Any]          =   ConfigTester(self,		config_class=lowerCamelCase__,		hidden_size=37		)






    def 			_lowerCAmelCase					(       self		):
        self.config_tester.run_common_tests()






    def 			_lowerCAmelCase					(       self		):
        A :       List[Any]          =   self.model_tester.prepare_config_and_inputs()
        self.model_tester.create_and_check_model(*lowerCamelCase__		)






    def 			_lowerCAmelCase					(       self		):
        A :       Tuple          =   self.model_tester.prepare_config_and_inputs()
        self.model_tester.create_and_check_for_masked_lm(*lowerCamelCase__		)






    def 			_lowerCAmelCase					(       self		):
        A :       List[str]          =   self.model_tester.prepare_config_and_inputs()
        self.model_tester.create_and_check_for_question_answering(*lowerCamelCase__		)






    def 			_lowerCAmelCase					(       self		):
        A :       Any          =   self.model_tester.prepare_config_and_inputs()
        self.model_tester.create_and_check_for_sequence_classification(*lowerCamelCase__		)






    def 			_lowerCAmelCase					(       self		):
        A :       str          =   self.model_tester.prepare_config_and_inputs()
        self.model_tester.create_and_check_for_token_classification(*lowerCamelCase__		)






    @slow
    def 			_lowerCAmelCase					(       self		):
        A :       Tuple          =   TFDebertaVaModel.from_pretrained("""kamalkraj/deberta-v2-xlarge"""		)
        self.assertIsNotNone(lowerCamelCase__		)






@require_tf
class SCREAMING_SNAKE_CASE__     (		unittest.TestCase    ):




    '''simple docstring'''






    @unittest.skip(reason="""Model not available yet"""		)
    def 			_lowerCAmelCase					(       self		):
        pass






    @slow
    def 			_lowerCAmelCase					(       self		):
        A :       Optional[int]          =   TFDebertaVaModel.from_pretrained("""kamalkraj/deberta-v2-xlarge"""		)
        A :       Dict          =   tf.constant([[0, 3_1414, 232, 328, 740, 1140, 1_2695, 69, 4_6078, 1588, 2]]		)
        A :       Tuple          =   tf.constant([[0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]]		)
        A :       Optional[Any]          =   model(lowerCamelCase__,		attention_mask=lowerCamelCase__		)[0]

        A :       List[str]          =   tf.constant(
            [[[0.2356, 0.1948, 0.0369], [-0.1063, 0.3586, -0.5152], [-0.6399, -0.0259, -0.2525]]]		)
        tf.debugging.assert_near(output[:, 1:4, 1:4],		lowerCamelCase__,		atol=1e-4		)







from dataclasses import dataclass
from typing import Dict, Optional, Union

import torch
import torch.nn.functional as F
from torch import nn

from ..configuration_utils import ConfigMixin, register_to_config
from ..utils import BaseOutput
from .attention import BasicTransformerBlock
from .attention_processor import AttentionProcessor, AttnProcessor
from .embeddings import TimestepEmbedding, Timesteps
from .modeling_utils import ModelMixin






@dataclass
class SCREAMING_SNAKE_CASE__     (		SCREAMING_SNAKE_CASE__    ):




    '''simple docstring'''






    __lowerCamelCase			: torch.FloatTensor






class SCREAMING_SNAKE_CASE__     (		SCREAMING_SNAKE_CASE__ ,						SCREAMING_SNAKE_CASE__    ):




    '''simple docstring'''






    @register_to_config
    def __init__(       self,		lowerCamelCase__ = 32,		lowerCamelCase__ = 64,		lowerCamelCase__ = 20,		lowerCamelCase__ = 768,		lowerCamelCase__=77,		lowerCamelCase__=4,		lowerCamelCase__ = 0.0,		lowerCamelCase__ = "silu",		lowerCamelCase__ = None,		lowerCamelCase__ = None,		lowerCamelCase__ = "linear",		lowerCamelCase__ = "prd",		lowerCamelCase__ = None,		lowerCamelCase__ = None,		lowerCamelCase__ = None,		):
        super().__init__()
        A :       int          =   num_attention_heads
        A :       Tuple          =   attention_head_dim
        A :       Optional[int]          =   num_attention_heads * attention_head_dim
        A :       List[str]          =   additional_embeddings

        A :       int          =   time_embed_dim or inner_dim
        A :       Tuple          =   embedding_proj_dim or embedding_dim
        A :       Dict          =   clip_embed_dim or embedding_dim

        A :       List[str]          =   Timesteps(lowerCamelCase__,		lowerCamelCase__,		0		)
        A :       Any          =   TimestepEmbedding(lowerCamelCase__,		lowerCamelCase__,		out_dim=lowerCamelCase__,		act_fn=lowerCamelCase__		)

        A :       List[Any]          =   nn.Linear(lowerCamelCase__,		lowerCamelCase__		)

        if embedding_proj_norm_type is None:
            A :       Union[str, Any]          =   None
        elif embedding_proj_norm_type == "layer":
            A :       str          =   nn.LayerNorm(lowerCamelCase__		)
        else:
            raise ValueError(f'''unsupported embedding_proj_norm_type: {embedding_proj_norm_type}'''		)

        A :       Optional[Any]          =   nn.Linear(lowerCamelCase__,		lowerCamelCase__		)

        if encoder_hid_proj_type is None:
            A :       Dict          =   None
        elif encoder_hid_proj_type == "linear":
            A :       Union[str, Any]          =   nn.Linear(lowerCamelCase__,		lowerCamelCase__		)
        else:
            raise ValueError(f'''unsupported encoder_hid_proj_type: {encoder_hid_proj_type}'''		)

        A :       List[str]          =   nn.Parameter(torch.zeros(1,		num_embeddings + additional_embeddings,		lowerCamelCase__		)		)

        if added_emb_type == "prd":
            A :       Union[str, Any]          =   nn.Parameter(torch.zeros(1,		1,		lowerCamelCase__		)		)
        elif added_emb_type is None:
            A :       int          =   None
        else:
            raise ValueError(
                f'''`added_emb_type`: {added_emb_type} is not supported. Make sure to choose one of `\'prd\'` or `None`.'''		)

        A :       int          =   nn.ModuleList(
            [
                BasicTransformerBlock(
                    lowerCamelCase__,		lowerCamelCase__,		lowerCamelCase__,		dropout=lowerCamelCase__,		activation_fn="""gelu""",		attention_bias=lowerCamelCase__,		)
                for d in range(lowerCamelCase__		)
            ]		)

        if norm_in_type == "layer":
            A :       int          =   nn.LayerNorm(lowerCamelCase__		)
        elif norm_in_type is None:
            A :       Tuple          =   None
        else:
            raise ValueError(f'''Unsupported norm_in_type: {norm_in_type}.'''		)

        A :       Optional[Any]          =   nn.LayerNorm(lowerCamelCase__		)

        A :       str          =   nn.Linear(lowerCamelCase__,		lowerCamelCase__		)

        A :       Union[str, Any]          =   torch.full(
            [num_embeddings + additional_embeddings, num_embeddings + additional_embeddings],		-1_0000.0		)
        causal_attention_mask.triu_(1		)
        A :       Optional[Any]          =   causal_attention_mask[None, ...]
        self.register_buffer("""causal_attention_mask""",		lowerCamelCase__,		persistent=lowerCamelCase__		)

        A :       Dict          =   nn.Parameter(torch.zeros(1,		lowerCamelCase__		)		)
        A :       Tuple          =   nn.Parameter(torch.zeros(1,		lowerCamelCase__		)		)






    @property
    # Copied from diffusers.models.unet_2d_condition.UNet2DConditionModel.attn_processors
    def 			_lowerCAmelCase					(       self		):
        A :       Optional[int]          =   {}

        def fn_recursive_add_processors(lowerCamelCase__,		lowerCamelCase__,		lowerCamelCase__		):
            if hasattr(lowerCamelCase__,		"""set_processor"""		):
                A :       Tuple          =   module.processor

            for sub_name, child in module.named_children():
                fn_recursive_add_processors(f'''{name}.{sub_name}''',		lowerCamelCase__,		lowerCamelCase__		)

            return processors

        for name, module in self.named_children():
            fn_recursive_add_processors(lowerCamelCase__,		lowerCamelCase__,		lowerCamelCase__		)

        return processors






    def 			_lowerCAmelCase					(       self,		lowerCamelCase__		):
        A :       Optional[Any]          =   len(self.attn_processors.keys()		)

        if isinstance(lowerCamelCase__,		lowerCamelCase__		) and len(lowerCamelCase__		) != count:
            raise ValueError(
                f'''A dict of processors was passed, but the number of processors {len(lowerCamelCase__		)} does not match the'''
                f''' number of attention layers: {count}. Please make sure to pass {count} processor classes.'''		)

        def fn_recursive_attn_processor(lowerCamelCase__,		lowerCamelCase__,		lowerCamelCase__		):
            if hasattr(lowerCamelCase__,		"""set_processor"""		):
                if not isinstance(lowerCamelCase__,		lowerCamelCase__		):
                    module.set_processor(lowerCamelCase__		)
                else:
                    module.set_processor(processor.pop(f'''{name}.processor'''		)		)

            for sub_name, child in module.named_children():
                fn_recursive_attn_processor(f'''{name}.{sub_name}''',		lowerCamelCase__,		lowerCamelCase__		)

        for name, module in self.named_children():
            fn_recursive_attn_processor(lowerCamelCase__,		lowerCamelCase__,		lowerCamelCase__		)






    def 			_lowerCAmelCase					(       self		):
        self.set_attn_processor(AttnProcessor()		)






    def 			_lowerCAmelCase					(       self,		lowerCamelCase__,		lowerCamelCase__,		lowerCamelCase__,		lowerCamelCase__ = None,		lowerCamelCase__ = None,		lowerCamelCase__ = True,		):
        A :       Optional[int]          =   hidden_states.shape[0]

        A :       Tuple          =   timestep
        if not torch.is_tensor(lowerCamelCase__		):
            A :       List[Any]          =   torch.tensor([timesteps],		dtype=torch.long,		device=hidden_states.device		)
        elif torch.is_tensor(lowerCamelCase__		) and len(timesteps.shape		) == 0:
            A :       Optional[int]          =   timesteps[None].to(hidden_states.device		)

        # broadcast to batch dimension in a way that's compatible with ONNX/Core ML
        A :       Optional[Any]          =   timesteps * torch.ones(lowerCamelCase__,		dtype=timesteps.dtype,		device=timesteps.device		)

        A :       str          =   self.time_proj(lowerCamelCase__		)

        # timesteps does not contain any weights and will always return f32 tensors
        # but time_embedding might be fp16, so we need to cast here.
        A :       Union[str, Any]          =   timesteps_projected.to(dtype=self.dtype		)
        A :       Tuple          =   self.time_embedding(lowerCamelCase__		)

        if self.embedding_proj_norm is not None:
            A :       Optional[Any]          =   self.embedding_proj_norm(lowerCamelCase__		)

        A :       Any          =   self.embedding_proj(lowerCamelCase__		)
        if self.encoder_hidden_states_proj is not None and encoder_hidden_states is not None:
            A :       List[str]          =   self.encoder_hidden_states_proj(lowerCamelCase__		)
        elif self.encoder_hidden_states_proj is not None and encoder_hidden_states is None:
            raise ValueError("""`encoder_hidden_states_proj` requires `encoder_hidden_states` to be set"""		)

        A :       Tuple          =   self.proj_in(lowerCamelCase__		)

        A :       Dict          =   self.positional_embedding.to(hidden_states.dtype		)

        A :       List[str]          =   []
        A :       Optional[Any]          =   0

        if encoder_hidden_states is not None:
            additional_embeds.append(lowerCamelCase__		)
            additional_embeddings_len += encoder_hidden_states.shape[1]

        if len(proj_embeddings.shape		) == 2:
            A :       Optional[Any]          =   proj_embeddings[:, None, :]

        if len(hidden_states.shape		) == 2:
            A :       List[str]          =   hidden_states[:, None, :]

        A :       Dict          =   additional_embeds + [
            proj_embeddings,
            time_embeddings[:, None, :],
            hidden_states,
        ]

        if self.prd_embedding is not None:
            A :       List[Any]          =   self.prd_embedding.to(hidden_states.dtype		).expand(lowerCamelCase__,		-1,		-1		)
            additional_embeds.append(lowerCamelCase__		)

        A :       Union[str, Any]          =   torch.cat(
            lowerCamelCase__,		dim=1,		)

        # Allow positional_embedding to not include the `addtional_embeddings` and instead pad it with zeros for these additional tokens
        A :       Dict          =   additional_embeddings_len + proj_embeddings.shape[1] + 1
        if positional_embeddings.shape[1] < hidden_states.shape[1]:
            A :       Any          =   F.pad(
                lowerCamelCase__,		(
                    0,
                    0,
                    additional_embeddings_len,
                    self.prd_embedding.shape[1] if self.prd_embedding is not None else 0,
                ),		value=0.0,		)

        A :       Union[str, Any]          =   hidden_states + positional_embeddings

        if attention_mask is not None:
            A :       List[Any]          =   (1 - attention_mask.to(hidden_states.dtype		)) * -1_0000.0
            A :       Tuple          =   F.pad(lowerCamelCase__,		(0, self.additional_embeddings),		value=0.0		)
            A :       List[Any]          =   (attention_mask[:, None, :] + self.causal_attention_mask).to(hidden_states.dtype		)
            A :       Any          =   attention_mask.repeat_interleave(self.config.num_attention_heads,		dim=0		)

        if self.norm_in is not None:
            A :       str          =   self.norm_in(lowerCamelCase__		)

        for block in self.transformer_blocks:
            A :       Optional[Any]          =   block(lowerCamelCase__,		attention_mask=lowerCamelCase__		)

        A :       Optional[int]          =   self.norm_out(lowerCamelCase__		)

        if self.prd_embedding is not None:
            A :       Dict          =   hidden_states[:, -1]
        else:
            A :       Optional[int]          =   hidden_states[:, additional_embeddings_len:]

        A :       Optional[Any]          =   self.proj_to_clip_embeddings(lowerCamelCase__		)

        if not return_dict:
            return (predicted_image_embedding,)

        return PriorTransformerOutput(predicted_image_embedding=lowerCamelCase__		)






    def 			_lowerCAmelCase					(       self,		lowerCamelCase__		):
        A :       str          =   (prior_latents * self.clip_std) + self.clip_mean
        return prior_latents


