





"""simple docstring"""





from typing import Any, Dict, Optional

import torch
import torch.nn.functional as F
from torch import nn

from ..utils import maybe_allow_in_graph
from .activations import get_activation
from .attention_processor import Attention
from .embeddings import CombinedTimestepLabelEmbeddings






@maybe_allow_in_graph
class 					_lowerCAmelCase   (     nn.Module       ):
       """simple docstring"""




       def __init__(    self    , __UpperCAmelCase    , __UpperCAmelCase    , __UpperCAmelCase    , __UpperCAmelCase=0.0    , __UpperCAmelCase = None    , __UpperCAmelCase = "geglu"    , __UpperCAmelCase = None    , __UpperCAmelCase = False    , __UpperCAmelCase = False    , __UpperCAmelCase = False    , __UpperCAmelCase = False    , __UpperCAmelCase = True    , __UpperCAmelCase = "layer_norm"    , __UpperCAmelCase = False    , ):



         '''simple docstring'''

         super().__init__()
         lowerCAmelCase__			:Dict											=							only_cross_attention

         lowerCAmelCase__			:Any											=							(num_embeds_ada_norm is not None) and norm_type == 'ada_norm_zero'
         lowerCAmelCase__			:Union[str, Any]											=							(num_embeds_ada_norm is not None) and norm_type == 'ada_norm'

         if norm_type in ("ada_norm", "ada_norm_zero") and num_embeds_ada_norm is None:
           raise ValueError(
               F"`norm_type` is set to {norm_type}, but `num_embeds_ada_norm` is not defined. Please make sure to"
               F" define `num_embeds_ada_norm` if setting `norm_type` to {norm_type}."      )

         # Define 3 blocks. Each block has its own normalization layer.
         # 1. Self-Attn
         if self.use_ada_layer_norm:
           lowerCAmelCase__			:Optional[Any]											=							AdaLayerNorm(__UpperCAmelCase    , __UpperCAmelCase      )
         elif self.use_ada_layer_norm_zero:
           lowerCAmelCase__			:List[str]											=							AdaLayerNormZero(__UpperCAmelCase    , __UpperCAmelCase      )
         else:
           lowerCAmelCase__			:Dict											=							nn.LayerNorm(__UpperCAmelCase    , elementwise_affine=__UpperCAmelCase      )
         lowerCAmelCase__			:Optional[int]											=							Attention(
             query_dim=__UpperCAmelCase    , heads=__UpperCAmelCase    , dim_head=__UpperCAmelCase    , dropout=__UpperCAmelCase    , bias=__UpperCAmelCase    , cross_attention_dim=cross_attention_dim if only_cross_attention else None    , upcast_attention=__UpperCAmelCase    , )

         # 2. Cross-Attn
         if cross_attention_dim is not None or double_self_attention:
           # We currently only use AdaLayerNormZero for self attention where there will only be one attention block.
           # I.e. the number of returned modulation chunks from AdaLayerZero would not make sense if returned during
           # the second cross attention block.
           lowerCAmelCase__			:Dict											=							(
               AdaLayerNorm(__UpperCAmelCase    , __UpperCAmelCase      )
               if self.use_ada_layer_norm
               else nn.LayerNorm(__UpperCAmelCase    , elementwise_affine=__UpperCAmelCase      )
           )
           lowerCAmelCase__			:List[Any]											=							Attention(
               query_dim=__UpperCAmelCase    , cross_attention_dim=cross_attention_dim if not double_self_attention else None    , heads=__UpperCAmelCase    , dim_head=__UpperCAmelCase    , dropout=__UpperCAmelCase    , bias=__UpperCAmelCase    , upcast_attention=__UpperCAmelCase    , )  # is self-attn if encoder_hidden_states is none
         else:
           lowerCAmelCase__			:List[str]											=							None
           lowerCAmelCase__			:List[Any]											=							None

         # 3. Feed-forward
         lowerCAmelCase__			:Union[str, Any]											=							nn.LayerNorm(__UpperCAmelCase    , elementwise_affine=__UpperCAmelCase      )
         lowerCAmelCase__			:Optional[Any]											=							FeedForward(__UpperCAmelCase    , dropout=__UpperCAmelCase    , activation_fn=__UpperCAmelCase    , final_dropout=__UpperCAmelCase      )

         # let chunk size default to None
         lowerCAmelCase__			:Tuple											=							None
         lowerCAmelCase__			:Dict											=							0






       def      snake_case						(    self    , __UpperCAmelCase    , __UpperCAmelCase      ):



         '''simple docstring'''

         lowerCAmelCase__			:str											=							chunk_size
         lowerCAmelCase__			:Optional[int]											=							dim






       def      snake_case						(    self    , __UpperCAmelCase    , __UpperCAmelCase = None    , __UpperCAmelCase = None    , __UpperCAmelCase = None    , __UpperCAmelCase = None    , __UpperCAmelCase = None    , __UpperCAmelCase = None    , ):



         '''simple docstring'''

         if self.use_ada_layer_norm:
           lowerCAmelCase__			:Tuple											=							self.norma(__UpperCAmelCase    , __UpperCAmelCase      )
         elif self.use_ada_layer_norm_zero:
           lowerCAmelCase__							,				lowerCAmelCase__							,				lowerCAmelCase__							,				lowerCAmelCase__							,				lowerCAmelCase__			:Union[str, Any]											=							self.norma(
               __UpperCAmelCase    , __UpperCAmelCase    , __UpperCAmelCase    , hidden_dtype=hidden_states.dtype      )
         else:
           lowerCAmelCase__			:Optional[int]											=							self.norma(__UpperCAmelCase      )

         lowerCAmelCase__			:Any											=							cross_attention_kwargs if cross_attention_kwargs is not None else {}

         lowerCAmelCase__			:Optional[int]											=							self.attna(
             __UpperCAmelCase    , encoder_hidden_states=encoder_hidden_states if self.only_cross_attention else None    , attention_mask=__UpperCAmelCase    , **__UpperCAmelCase    , )
         if self.use_ada_layer_norm_zero:
           lowerCAmelCase__			:Any											=							gate_msa.unsqueeze(1      ) * attn_output
         lowerCAmelCase__			:Dict											=							attn_output + hidden_states

         # 2. Cross-Attention
         if self.attna is not None:
           lowerCAmelCase__			:List[Any]											=							(
               self.norma(__UpperCAmelCase    , __UpperCAmelCase      ) if self.use_ada_layer_norm else self.norma(__UpperCAmelCase      )
           )

           lowerCAmelCase__			:List[Any]											=							self.attna(
               __UpperCAmelCase    , encoder_hidden_states=__UpperCAmelCase    , attention_mask=__UpperCAmelCase    , **__UpperCAmelCase    , )
           lowerCAmelCase__			:Union[str, Any]											=							attn_output + hidden_states

         # 3. Feed-forward
         lowerCAmelCase__			:str											=							self.norma(__UpperCAmelCase      )

         if self.use_ada_layer_norm_zero:
           lowerCAmelCase__			:List[str]											=							norm_hidden_states * (1 + scale_mlp[:, None]) + shift_mlp[:, None]

         if self._chunk_size is not None:
           # "feed_forward_chunk_size" can be used to save memory
           if norm_hidden_states.shape[self._chunk_dim] % self._chunk_size != 0:
             raise ValueError(
                 F"`hidden_states` dimension to be chunked: {norm_hidden_states.shape[self._chunk_dim]} has to be divisible by chunk size: {self._chunk_size}. Make sure to set an appropriate `chunk_size` when calling `unet.enable_forward_chunking`."      )

           lowerCAmelCase__			:Union[str, Any]											=							norm_hidden_states.shape[self._chunk_dim] // self._chunk_size
           lowerCAmelCase__			:Dict											=							torch.cat(
               [self.ff(__UpperCAmelCase      ) for hid_slice in norm_hidden_states.chunk(__UpperCAmelCase    , dim=self._chunk_dim      )]    , dim=self._chunk_dim    , )
         else:
           lowerCAmelCase__			:str											=							self.ff(__UpperCAmelCase      )

         if self.use_ada_layer_norm_zero:
           lowerCAmelCase__			:Tuple											=							gate_mlp.unsqueeze(1      ) * ff_output

         lowerCAmelCase__			:List[Any]											=							ff_output + hidden_states

         return hidden_states






class 					_lowerCAmelCase   (     nn.Module       ):
       """simple docstring"""




       def __init__(    self    , __UpperCAmelCase    , __UpperCAmelCase = None    , __UpperCAmelCase = 4    , __UpperCAmelCase = 0.0    , __UpperCAmelCase = "geglu"    , __UpperCAmelCase = False    , ):



         '''simple docstring'''

         super().__init__()
         lowerCAmelCase__			:Any											=							int(dim * mult      )
         lowerCAmelCase__			:str											=							dim_out if dim_out is not None else dim

         if activation_fn == "gelu":
           lowerCAmelCase__			:Optional[Any]											=							GELU(__UpperCAmelCase    , __UpperCAmelCase      )
         if activation_fn == "gelu-approximate":
           lowerCAmelCase__			:Union[str, Any]											=							GELU(__UpperCAmelCase    , __UpperCAmelCase    , approximate='tanh'      )
         elif activation_fn == "geglu":
           lowerCAmelCase__			:Optional[Any]											=							GEGLU(__UpperCAmelCase    , __UpperCAmelCase      )
         elif activation_fn == "geglu-approximate":
           lowerCAmelCase__			:Union[str, Any]											=							ApproximateGELU(__UpperCAmelCase    , __UpperCAmelCase      )

         lowerCAmelCase__			:int											=							nn.ModuleList([]      )
         # project in
         self.net.append(__UpperCAmelCase      )
         # project dropout
         self.net.append(nn.Dropout(__UpperCAmelCase      )      )
         # project out
         self.net.append(nn.Linear(__UpperCAmelCase    , __UpperCAmelCase      )      )
         # FF as used in Vision Transformer, MLP-Mixer, etc. have a final dropout
         if final_dropout:
           self.net.append(nn.Dropout(__UpperCAmelCase      )      )






       def      snake_case						(    self    , __UpperCAmelCase      ):



         '''simple docstring'''

         for module in self.net:
           lowerCAmelCase__			:Optional[int]											=							module(__UpperCAmelCase      )
         return hidden_states






class 					_lowerCAmelCase   (     nn.Module       ):
       """simple docstring"""




       def __init__(    self    , __UpperCAmelCase    , __UpperCAmelCase    , __UpperCAmelCase = "none"      ):



         '''simple docstring'''

         super().__init__()
         lowerCAmelCase__			:str											=							nn.Linear(__UpperCAmelCase    , __UpperCAmelCase      )
         lowerCAmelCase__			:List[str]											=							approximate






       def      snake_case						(    self    , __UpperCAmelCase      ):



         '''simple docstring'''

         if gate.device.type != "mps":
           return F.gelu(__UpperCAmelCase    , approximate=self.approximate      )
         # mps: gelu is not implemented for float16
         return F.gelu(gate.to(dtype=torch.floataa      )    , approximate=self.approximate      ).to(dtype=gate.dtype      )






       def      snake_case						(    self    , __UpperCAmelCase      ):



         '''simple docstring'''

         lowerCAmelCase__			:List[str]											=							self.proj(__UpperCAmelCase      )
         lowerCAmelCase__			:Dict											=							self.gelu(__UpperCAmelCase      )
         return hidden_states






class 					_lowerCAmelCase   (     nn.Module       ):
       """simple docstring"""




       def __init__(    self    , __UpperCAmelCase    , __UpperCAmelCase      ):



         '''simple docstring'''

         super().__init__()
         lowerCAmelCase__			:int											=							nn.Linear(__UpperCAmelCase    , dim_out * 2      )






       def      snake_case						(    self    , __UpperCAmelCase      ):



         '''simple docstring'''

         if gate.device.type != "mps":
           return F.gelu(__UpperCAmelCase      )
         # mps: gelu is not implemented for float16
         return F.gelu(gate.to(dtype=torch.floataa      )      ).to(dtype=gate.dtype      )






       def      snake_case						(    self    , __UpperCAmelCase      ):



         '''simple docstring'''

         lowerCAmelCase__							,				lowerCAmelCase__			:int											=							self.proj(__UpperCAmelCase      ).chunk(2    , dim=-1      )
         return hidden_states * self.gelu(__UpperCAmelCase      )






class 					_lowerCAmelCase   (     nn.Module       ):
       """simple docstring"""




       def __init__(    self    , __UpperCAmelCase    , __UpperCAmelCase      ):



         '''simple docstring'''

         super().__init__()
         lowerCAmelCase__			:Dict											=							nn.Linear(__UpperCAmelCase    , __UpperCAmelCase      )






       def      snake_case						(    self    , __UpperCAmelCase      ):



         '''simple docstring'''

         lowerCAmelCase__			:List[str]											=							self.proj(__UpperCAmelCase      )
         return x * torch.sigmoid(1.7_02 * x      )






class 					_lowerCAmelCase   (     nn.Module       ):
       """simple docstring"""




       def __init__(    self    , __UpperCAmelCase    , __UpperCAmelCase      ):



         '''simple docstring'''

         super().__init__()
         lowerCAmelCase__			:List[str]											=							nn.Embedding(__UpperCAmelCase    , __UpperCAmelCase      )
         lowerCAmelCase__			:List[Any]											=							nn.SiLU()
         lowerCAmelCase__			:Union[str, Any]											=							nn.Linear(__UpperCAmelCase    , embedding_dim * 2      )
         lowerCAmelCase__			:List[Any]											=							nn.LayerNorm(__UpperCAmelCase    , elementwise_affine=__UpperCAmelCase      )






       def      snake_case						(    self    , __UpperCAmelCase    , __UpperCAmelCase      ):



         '''simple docstring'''

         lowerCAmelCase__			:List[str]											=							self.linear(self.silu(self.emb(__UpperCAmelCase      )      )      )
         lowerCAmelCase__							,				lowerCAmelCase__			:Optional[Any]											=							torch.chunk(__UpperCAmelCase    , 2      )
         lowerCAmelCase__			:Union[str, Any]											=							self.norm(__UpperCAmelCase      ) * (1 + scale) + shift
         return x






class 					_lowerCAmelCase   (     nn.Module       ):
       """simple docstring"""




       def __init__(    self    , __UpperCAmelCase    , __UpperCAmelCase      ):



         '''simple docstring'''

         super().__init__()

         lowerCAmelCase__			:int											=							CombinedTimestepLabelEmbeddings(__UpperCAmelCase    , __UpperCAmelCase      )

         lowerCAmelCase__			:Optional[Any]											=							nn.SiLU()
         lowerCAmelCase__			:Optional[int]											=							nn.Linear(__UpperCAmelCase    , 6 * embedding_dim    , bias=__UpperCAmelCase      )
         lowerCAmelCase__			:int											=							nn.LayerNorm(__UpperCAmelCase    , elementwise_affine=__UpperCAmelCase    , eps=1E-6      )






       def      snake_case						(    self    , __UpperCAmelCase    , __UpperCAmelCase    , __UpperCAmelCase    , __UpperCAmelCase=None      ):



         '''simple docstring'''

         lowerCAmelCase__			:Optional[int]											=							self.linear(self.silu(self.emb(__UpperCAmelCase    , __UpperCAmelCase    , hidden_dtype=__UpperCAmelCase      )      )      )
         lowerCAmelCase__							,				lowerCAmelCase__							,				lowerCAmelCase__							,				lowerCAmelCase__							,				lowerCAmelCase__							,				lowerCAmelCase__			:Dict											=							emb.chunk(6    , dim=1      )
         lowerCAmelCase__			:Any											=							self.norm(__UpperCAmelCase      ) * (1 + scale_msa[:, None]) + shift_msa[:, None]
         return x, gate_msa, shift_mlp, scale_mlp, gate_mlp






class 					_lowerCAmelCase   (     nn.Module       ):
       """simple docstring"""




       def __init__(    self    , __UpperCAmelCase    , __UpperCAmelCase    , __UpperCAmelCase    , __UpperCAmelCase = None    , __UpperCAmelCase = 1E-5      ):



         '''simple docstring'''

         super().__init__()
         lowerCAmelCase__			:Any											=							num_groups
         lowerCAmelCase__			:str											=							eps

         if act_fn is None:
           lowerCAmelCase__			:List[Any]											=							None
         else:
           lowerCAmelCase__			:Optional[Any]											=							get_activation(__UpperCAmelCase      )

         lowerCAmelCase__			:Optional[Any]											=							nn.Linear(__UpperCAmelCase    , out_dim * 2      )






       def      snake_case						(    self    , __UpperCAmelCase    , __UpperCAmelCase      ):



         '''simple docstring'''

         if self.act:
           lowerCAmelCase__			:Optional[Any]											=							self.act(__UpperCAmelCase      )
         lowerCAmelCase__			:Union[str, Any]											=							self.linear(__UpperCAmelCase      )
         lowerCAmelCase__			:Dict											=							emb[:, :, None, None]
         lowerCAmelCase__							,				lowerCAmelCase__			:Tuple											=							emb.chunk(2    , dim=1      )

         lowerCAmelCase__			:List[Any]											=							F.group_norm(__UpperCAmelCase    , self.num_groups    , eps=self.eps      )
         lowerCAmelCase__			:Union[str, Any]											=							x * (1 + scale) + shift
         return x







"""simple docstring"""





def  __A   (_SCREAMING_SNAKE_CASE     ,   _SCREAMING_SNAKE_CASE			)					->bool:





  """simple docstring"""


  return numa ^ numa < 0


if __name__ == "__main__":
  import doctest

  doctest.testmod()
