from typing import Any, Dict, Optional

import torch
import torch.nn.functional as F
from torch import nn

from ..utils import maybe_allow_in_graph
from .activations import get_activation
from .attention_processor import Attention
from .embeddings import CombinedTimestepLabelEmbeddings





@maybe_allow_in_graph
class 	A				(							nn.Module	):


   def __init__(self	:		str			,    __UpperCAmelCase	:		Optional[int]			,    __UpperCAmelCase	:		Tuple			,    __UpperCAmelCase	:		Dict			,    __UpperCAmelCase	:		Tuple=0.0			,    __UpperCAmelCase	:		Union[str, Any] = None			,    __UpperCAmelCase	:		Union[str, Any] = "geglu"			,    __UpperCAmelCase	:		int = None			,    __UpperCAmelCase	:		Any = False			,    __UpperCAmelCase	:		Union[str, Any] = False			,    __UpperCAmelCase	:		Tuple = False			,    __UpperCAmelCase	:		List[str] = False			,    __UpperCAmelCase	:		List[str] = True			,    __UpperCAmelCase	:		Dict = "layer_norm"			,    __UpperCAmelCase	:		Optional[Any] = False			,    )		->							Optional[int]:
         """simple docstring"""



         super().__init__()
         UpperCAmelCase__								=    only_cross_attention

         UpperCAmelCase__								=    (num_embeds_ada_norm is not None) and norm_type == "ada_norm_zero"
         UpperCAmelCase__								=    (num_embeds_ada_norm is not None) and norm_type == "ada_norm"

         if norm_type in ("ada_norm", "ada_norm_zero") and num_embeds_ada_norm is None:
               raise ValueError(
                   f"""`norm_type` is set to {norm_type}, but `num_embeds_ada_norm` is not defined. Please make sure to"""
                   f""" define `num_embeds_ada_norm` if setting `norm_type` to {norm_type}."""						)

         # Define 3 blocks. Each block has its own normalization layer.
         # 1. Self-Attn
         if self.use_ada_layer_norm:
               UpperCAmelCase__								=    AdaLayerNorm(lowerCAmelCase__			,    lowerCAmelCase__						)
         elif self.use_ada_layer_norm_zero:
               UpperCAmelCase__								=    AdaLayerNormZero(lowerCAmelCase__			,    lowerCAmelCase__						)
         else:
               UpperCAmelCase__								=    nn.LayerNorm(lowerCAmelCase__			,    elementwise_affine=lowerCAmelCase__						)
         UpperCAmelCase__								=    Attention(
             query_dim=lowerCAmelCase__			,    heads=lowerCAmelCase__			,    dim_head=lowerCAmelCase__			,    dropout=lowerCAmelCase__			,    bias=lowerCAmelCase__			,    cross_attention_dim=cross_attention_dim if only_cross_attention else None			,    upcast_attention=lowerCAmelCase__			,    )

         # 2. Cross-Attn
         if cross_attention_dim is not None or double_self_attention:
               # We currently only use AdaLayerNormZero for self attention where there will only be one attention block.
               # I.e. the number of returned modulation chunks from AdaLayerZero would not make sense if returned during
               # the second cross attention block.
               UpperCAmelCase__								=    (
                   AdaLayerNorm(lowerCAmelCase__			,    lowerCAmelCase__						)
                   if self.use_ada_layer_norm
                   else nn.LayerNorm(lowerCAmelCase__			,    elementwise_affine=lowerCAmelCase__						)
               )
               UpperCAmelCase__								=    Attention(
                   query_dim=lowerCAmelCase__			,    cross_attention_dim=cross_attention_dim if not double_self_attention else None			,    heads=lowerCAmelCase__			,    dim_head=lowerCAmelCase__			,    dropout=lowerCAmelCase__			,    bias=lowerCAmelCase__			,    upcast_attention=lowerCAmelCase__			,    )  # is self-attn if encoder_hidden_states is none
         else:
               UpperCAmelCase__								=    None
               UpperCAmelCase__								=    None

         # 3. Feed-forward
         UpperCAmelCase__								=    nn.LayerNorm(lowerCAmelCase__			,    elementwise_affine=lowerCAmelCase__						)
         UpperCAmelCase__								=    FeedForward(lowerCAmelCase__			,    dropout=lowerCAmelCase__			,    activation_fn=lowerCAmelCase__			,    final_dropout=lowerCAmelCase__						)

         # let chunk size default to None
         UpperCAmelCase__								=    None
         UpperCAmelCase__								=    0


   def      lowercase_ (self	:		List[str]			,    __UpperCAmelCase	:		Tuple			,    __UpperCAmelCase	:		Union[str, Any]						)		->							Optional[Any]:
         """simple docstring"""



         UpperCAmelCase__								=    chunk_size
         UpperCAmelCase__								=    dim





   def      lowercase_ (self	:		Union[str, Any]			,    __UpperCAmelCase	:		str			,    __UpperCAmelCase	:		Optional[Any] = None			,    __UpperCAmelCase	:		Optional[int] = None			,    __UpperCAmelCase	:		int = None			,    __UpperCAmelCase	:		Dict = None			,    __UpperCAmelCase	:		Any = None			,    __UpperCAmelCase	:		Dict = None			,    )		->							Any:
         """simple docstring"""



         if self.use_ada_layer_norm:
               UpperCAmelCase__								=    self.norma(lowerCAmelCase__			,    lowerCAmelCase__						)
         elif self.use_ada_layer_norm_zero:
               UpperCAmelCase__								=    self.norma(
                   lowerCAmelCase__			,    lowerCAmelCase__			,    lowerCAmelCase__			,    hidden_dtype=hidden_states.dtype						)
         else:
               UpperCAmelCase__								=    self.norma(lowerCAmelCase__						)

         UpperCAmelCase__								=    cross_attention_kwargs if cross_attention_kwargs is not None else {}

         UpperCAmelCase__								=    self.attna(
             lowerCAmelCase__			,    encoder_hidden_states=encoder_hidden_states if self.only_cross_attention else None			,    attention_mask=lowerCAmelCase__			,    **lowerCAmelCase__			,    )
         if self.use_ada_layer_norm_zero:
               UpperCAmelCase__								=    gate_msa.unsqueeze(1						) * attn_output
         UpperCAmelCase__								=    attn_output + hidden_states

         # 2. Cross-Attention
         if self.attna is not None:
               UpperCAmelCase__								=    (
                   self.norma(lowerCAmelCase__			,    lowerCAmelCase__						) if self.use_ada_layer_norm else self.norma(lowerCAmelCase__						)
               )

               UpperCAmelCase__								=    self.attna(
                   lowerCAmelCase__			,    encoder_hidden_states=lowerCAmelCase__			,    attention_mask=lowerCAmelCase__			,    **lowerCAmelCase__			,    )
               UpperCAmelCase__								=    attn_output + hidden_states

         # 3. Feed-forward
         UpperCAmelCase__								=    self.norma(lowerCAmelCase__						)

         if self.use_ada_layer_norm_zero:
               UpperCAmelCase__								=    norm_hidden_states * (1 + scale_mlp[:, None]) + shift_mlp[:, None]

         if self._chunk_size is not None:
               # "feed_forward_chunk_size" can be used to save memory
               if norm_hidden_states.shape[self._chunk_dim] % self._chunk_size != 0:
                     raise ValueError(
                         f"""`hidden_states` dimension to be chunked: {norm_hidden_states.shape[self._chunk_dim]} has to be divisible by chunk size: {self._chunk_size}. Make sure to set an appropriate `chunk_size` when calling `unet.enable_forward_chunking`."""						)

               UpperCAmelCase__								=    norm_hidden_states.shape[self._chunk_dim] // self._chunk_size
               UpperCAmelCase__								=    torch.cat(
                   [self.ff(lowerCAmelCase__						) for hid_slice in norm_hidden_states.chunk(lowerCAmelCase__			,    dim=self._chunk_dim						)]			,    dim=self._chunk_dim			,    )
         else:
               UpperCAmelCase__								=    self.ff(lowerCAmelCase__						)

         if self.use_ada_layer_norm_zero:
               UpperCAmelCase__								=    gate_mlp.unsqueeze(1						) * ff_output

         UpperCAmelCase__								=    ff_output + hidden_states

         return hidden_states





class 	A				(							nn.Module	):


   def __init__(self	:		Optional[Any]			,    __UpperCAmelCase	:		Optional[Any]			,    __UpperCAmelCase	:		Any = None			,    __UpperCAmelCase	:		Optional[int] = 4			,    __UpperCAmelCase	:		Any = 0.0			,    __UpperCAmelCase	:		Tuple = "geglu"			,    __UpperCAmelCase	:		Any = False			,    )		->							List[str]:
         """simple docstring"""



         super().__init__()
         UpperCAmelCase__								=    int(dim * mult						)
         UpperCAmelCase__								=    dim_out if dim_out is not None else dim

         if activation_fn == "gelu":
               UpperCAmelCase__								=    GELU(lowerCAmelCase__			,    lowerCAmelCase__						)
         if activation_fn == "gelu-approximate":
               UpperCAmelCase__								=    GELU(lowerCAmelCase__			,    lowerCAmelCase__			,    approximate="tanh"						)
         elif activation_fn == "geglu":
               UpperCAmelCase__								=    GEGLU(lowerCAmelCase__			,    lowerCAmelCase__						)
         elif activation_fn == "geglu-approximate":
               UpperCAmelCase__								=    ApproximateGELU(lowerCAmelCase__			,    lowerCAmelCase__						)

         UpperCAmelCase__								=    nn.ModuleList([]						)
         # project in
         self.net.append(lowerCAmelCase__						)
         # project dropout
         self.net.append(nn.Dropout(lowerCAmelCase__						)						)
         # project out
         self.net.append(nn.Linear(lowerCAmelCase__			,    lowerCAmelCase__						)						)
         # FF as used in Vision Transformer, MLP-Mixer, etc. have a final dropout
         if final_dropout:
               self.net.append(nn.Dropout(lowerCAmelCase__						)						)





   def      lowercase_ (self	:		str			,    __UpperCAmelCase	:		Tuple						)		->							int:
         """simple docstring"""



         for module in self.net:
               UpperCAmelCase__								=    module(lowerCAmelCase__						)
         return hidden_states





class 	A				(							nn.Module	):


   def __init__(self	:		Union[str, Any]			,    __UpperCAmelCase	:		List[Any]			,    __UpperCAmelCase	:		Dict			,    __UpperCAmelCase	:		Any = "none"						)		->							Optional[Any]:
         """simple docstring"""



         super().__init__()
         UpperCAmelCase__								=    nn.Linear(lowerCAmelCase__			,    lowerCAmelCase__						)
         UpperCAmelCase__								=    approximate


   def      lowercase_ (self	:		Tuple			,    __UpperCAmelCase	:		int						)		->							Tuple:
         """simple docstring"""



         if gate.device.type != "mps":
               return F.gelu(lowerCAmelCase__			,    approximate=self.approximate						)
         # mps: gelu is not implemented for float16
         return F.gelu(gate.to(dtype=torch.floataa						)			,    approximate=self.approximate						).to(dtype=gate.dtype						)





   def      lowercase_ (self	:		Any			,    __UpperCAmelCase	:		List[Any]						)		->							List[Any]:
         """simple docstring"""



         UpperCAmelCase__								=    self.proj(lowerCAmelCase__						)
         UpperCAmelCase__								=    self.gelu(lowerCAmelCase__						)
         return hidden_states





class 	A				(							nn.Module	):


   def __init__(self	:		Union[str, Any]			,    __UpperCAmelCase	:		Optional[int]			,    __UpperCAmelCase	:		int						)		->							List[str]:
         """simple docstring"""



         super().__init__()
         UpperCAmelCase__								=    nn.Linear(lowerCAmelCase__			,    dim_out * 2						)


   def      lowercase_ (self	:		Dict			,    __UpperCAmelCase	:		Union[str, Any]						)		->							Optional[Any]:
         """simple docstring"""



         if gate.device.type != "mps":
               return F.gelu(lowerCAmelCase__						)
         # mps: gelu is not implemented for float16
         return F.gelu(gate.to(dtype=torch.floataa						)						).to(dtype=gate.dtype						)





   def      lowercase_ (self	:		List[Any]			,    __UpperCAmelCase	:		int						)		->							List[str]:
         """simple docstring"""



         UpperCAmelCase__								=    self.proj(lowerCAmelCase__						).chunk(2			,    dim=-1						)
         return hidden_states * self.gelu(lowerCAmelCase__						)





class 	A				(							nn.Module	):


   def __init__(self	:		Dict			,    __UpperCAmelCase	:		int			,    __UpperCAmelCase	:		List[str]						)		->							str:
         """simple docstring"""



         super().__init__()
         UpperCAmelCase__								=    nn.Linear(lowerCAmelCase__			,    lowerCAmelCase__						)





   def      lowercase_ (self	:		str			,    __UpperCAmelCase	:		Tuple						)		->							List[Any]:
         """simple docstring"""



         UpperCAmelCase__								=    self.proj(lowerCAmelCase__						)
         return x * torch.sigmoid(1.702 * x						)





class 	A				(							nn.Module	):


   def __init__(self	:		Optional[int]			,    __UpperCAmelCase	:		Any			,    __UpperCAmelCase	:		Union[str, Any]						)		->							str:
         """simple docstring"""



         super().__init__()
         UpperCAmelCase__								=    nn.Embedding(lowerCAmelCase__			,    lowerCAmelCase__						)
         UpperCAmelCase__								=    nn.SiLU()
         UpperCAmelCase__								=    nn.Linear(lowerCAmelCase__			,    embedding_dim * 2						)
         UpperCAmelCase__								=    nn.LayerNorm(lowerCAmelCase__			,    elementwise_affine=lowerCAmelCase__						)





   def      lowercase_ (self	:		str			,    __UpperCAmelCase	:		Optional[Any]			,    __UpperCAmelCase	:		Tuple						)		->							int:
         """simple docstring"""



         UpperCAmelCase__								=    self.linear(self.silu(self.emb(lowerCAmelCase__						)						)						)
         UpperCAmelCase__								=    torch.chunk(lowerCAmelCase__			,    2						)
         UpperCAmelCase__								=    self.norm(lowerCAmelCase__						) * (1 + scale) + shift
         return x





class 	A				(							nn.Module	):


   def __init__(self	:		int			,    __UpperCAmelCase	:		int			,    __UpperCAmelCase	:		Optional[Any]						)		->							Union[str, Any]:
         """simple docstring"""



         super().__init__()

         UpperCAmelCase__								=    CombinedTimestepLabelEmbeddings(lowerCAmelCase__			,    lowerCAmelCase__						)

         UpperCAmelCase__								=    nn.SiLU()
         UpperCAmelCase__								=    nn.Linear(lowerCAmelCase__			,    6 * embedding_dim			,    bias=lowerCAmelCase__						)
         UpperCAmelCase__								=    nn.LayerNorm(lowerCAmelCase__			,    elementwise_affine=lowerCAmelCase__			,    eps=1E-6						)





   def      lowercase_ (self	:		str			,    __UpperCAmelCase	:		Optional[int]			,    __UpperCAmelCase	:		int			,    __UpperCAmelCase	:		Optional[Any]			,    __UpperCAmelCase	:		Dict=None						)		->							int:
         """simple docstring"""



         UpperCAmelCase__								=    self.linear(self.silu(self.emb(lowerCAmelCase__			,    lowerCAmelCase__			,    hidden_dtype=lowerCAmelCase__						)						)						)
         UpperCAmelCase__								=    emb.chunk(6			,    dim=1						)
         UpperCAmelCase__								=    self.norm(lowerCAmelCase__						) * (1 + scale_msa[:, None]) + shift_msa[:, None]
         return x, gate_msa, shift_mlp, scale_mlp, gate_mlp





class 	A				(							nn.Module	):


   def __init__(self	:		List[Any]			,    __UpperCAmelCase	:		int			,    __UpperCAmelCase	:		List[Any]			,    __UpperCAmelCase	:		Any			,    __UpperCAmelCase	:		Any = None			,    __UpperCAmelCase	:		Dict = 1E-5						)		->							List[str]:
         """simple docstring"""



         super().__init__()
         UpperCAmelCase__								=    num_groups
         UpperCAmelCase__								=    eps

         if act_fn is None:
               UpperCAmelCase__								=    None
         else:
               UpperCAmelCase__								=    get_activation(lowerCAmelCase__						)

         UpperCAmelCase__								=    nn.Linear(lowerCAmelCase__			,    out_dim * 2						)





   def      lowercase_ (self	:		Tuple			,    __UpperCAmelCase	:		Tuple			,    __UpperCAmelCase	:		Any						)		->							Optional[Any]:
         """simple docstring"""



         if self.act:
               UpperCAmelCase__								=    self.act(lowerCAmelCase__						)
         UpperCAmelCase__								=    self.linear(lowerCAmelCase__						)
         UpperCAmelCase__								=    emb[:, :, None, None]
         UpperCAmelCase__								=    emb.chunk(2			,    dim=1						)

         UpperCAmelCase__								=    F.group_norm(lowerCAmelCase__			,    self.num_groups			,    eps=self.eps						)
         UpperCAmelCase__								=    x * (1 + scale) + shift
         return x










"""simple docstring"""




def  _SCREAMING_SNAKE_CASE						(      _lowercase		:       list[int]   )					->float:






	'''simple docstring'''






	if not nums:  # Makes sure that the list is not empty
		raise ValueError("List is empty"   )

	a :						Dict      =							sum(_lowercase   ) / len(_lowercase   )  # Calculate the average
	return sum(abs(x - average   ) for x in nums   ) / len(_lowercase   )


if __name__ == "__main__":
		import doctest

		doctest.testmod()


