





from typing import Any, Dict, Optional

import torch
import torch.nn.functional as F
from torch import nn

from ..utils import maybe_allow_in_graph
from .activations import get_activation
from .attention_processor import Attention
from .embeddings import CombinedTimestepLabelEmbeddings



@maybe_allow_in_graph
class    A__  (			nn.Module     ):





      def __init__(					self			,				A_			,				A_			,				A_			,				A_=0.0			,				A_ = None			,				A_ = "geglu"			,				A_ = None			,				A_ = False			,				A_ = False			,				A_ = False			,				A_ = False			,				A_ = True			,				A_ = "layer_norm"			,				A_ = False			,				):
       '''simple docstring'''




       super().__init__()
       UpperCamelCase      :   Optional[Any]			      =  only_cross_attention

       UpperCamelCase      :   Any			      =  (num_embeds_ada_norm is not None) and norm_type == "ada_norm_zero"
       UpperCamelCase      :   Tuple			      =  (num_embeds_ada_norm is not None) and norm_type == "ada_norm"

       if norm_type in ("ada_norm", "ada_norm_zero") and num_embeds_ada_norm is None:
        raise ValueError(
            F"""`norm_type` is set to {norm_type}, but `num_embeds_ada_norm` is not defined. Please make sure to"""
            F""" define `num_embeds_ada_norm` if setting `norm_type` to {norm_type}."""  )

       # Define 3 blocks. Each block has its own normalization layer.
       # 1. Self-Attn
       if self.use_ada_layer_norm:
        UpperCamelCase      :   Optional[Any]			      =  AdaLayerNorm(A_			,				A_  )
       elif self.use_ada_layer_norm_zero:
        UpperCamelCase      :   Any			      =  AdaLayerNormZero(A_			,				A_  )
       else:
        UpperCamelCase      :   List[str]			      =  nn.LayerNorm(A_			,				elementwise_affine=A_  )
       UpperCamelCase      :   Tuple			      =  Attention(
           query_dim=A_			,				heads=A_			,				dim_head=A_			,				dropout=A_			,				bias=A_			,				cross_attention_dim=cross_attention_dim if only_cross_attention else None			,				upcast_attention=A_			,				)

       # 2. Cross-Attn
       if cross_attention_dim is not None or double_self_attention:
        # We currently only use AdaLayerNormZero for self attention where there will only be one attention block.
        # I.e. the number of returned modulation chunks from AdaLayerZero would not make sense if returned during
        # the second cross attention block.
        UpperCamelCase      :   Tuple			      =  (
            AdaLayerNorm(A_			,				A_  )
            if self.use_ada_layer_norm
            else nn.LayerNorm(A_			,				elementwise_affine=A_  )
        )
        UpperCamelCase      :   Optional[int]			      =  Attention(
            query_dim=A_			,				cross_attention_dim=cross_attention_dim if not double_self_attention else None			,				heads=A_			,				dim_head=A_			,				dropout=A_			,				bias=A_			,				upcast_attention=A_			,				)  # is self-attn if encoder_hidden_states is none
       else:
        UpperCamelCase      :   Optional[Any]			      =  None
        UpperCamelCase      :   List[str]			      =  None

       # 3. Feed-forward
       UpperCamelCase      :   Tuple			      =  nn.LayerNorm(A_			,				elementwise_affine=A_  )
       UpperCamelCase      :   str			      =  FeedForward(A_			,				dropout=A_			,				activation_fn=A_			,				final_dropout=A_  )

       # let chunk size default to None
       UpperCamelCase      :   Any			      =  None
       UpperCamelCase      :   Dict			      =  0





      def 			__UpperCamelCase(					self			,				A_			,				A_  ):
       '''simple docstring'''




       UpperCamelCase      :   str			      =  chunk_size
       UpperCamelCase      :   Dict			      =  dim





      def 			__UpperCamelCase(					self			,				A_			,				A_ = None			,				A_ = None			,				A_ = None			,				A_ = None			,				A_ = None			,				A_ = None			,				):
       '''simple docstring'''




       if self.use_ada_layer_norm:
        UpperCamelCase      :   List[Any]			      =  self.norma(A_			,				A_  )
       elif self.use_ada_layer_norm_zero:
        UpperCamelCase      ,       UpperCamelCase      ,       UpperCamelCase      ,       UpperCamelCase      ,       UpperCamelCase      :   Optional[int]			      =  self.norma(
            A_			,				A_			,				A_			,				hidden_dtype=hidden_states.dtype  )
       else:
        UpperCamelCase      :   int			      =  self.norma(A_  )

       UpperCamelCase      :   Tuple			      =  cross_attention_kwargs if cross_attention_kwargs is not None else {}

       UpperCamelCase      :   Optional[Any]			      =  self.attna(
           A_			,				encoder_hidden_states=encoder_hidden_states if self.only_cross_attention else None			,				attention_mask=A_			,				**A_			,				)
       if self.use_ada_layer_norm_zero:
        UpperCamelCase      :   List[Any]			      =  gate_msa.unsqueeze(1  ) * attn_output
       UpperCamelCase      :   str			      =  attn_output + hidden_states

       # 2. Cross-Attention
       if self.attna is not None:
        UpperCamelCase      :   List[Any]			      =  (
            self.norma(A_			,				A_  ) if self.use_ada_layer_norm else self.norma(A_  )
        )

        UpperCamelCase      :   Optional[int]			      =  self.attna(
            A_			,				encoder_hidden_states=A_			,				attention_mask=A_			,				**A_			,				)
        UpperCamelCase      :   Tuple			      =  attn_output + hidden_states

       # 3. Feed-forward
       UpperCamelCase      :   Any			      =  self.norma(A_  )

       if self.use_ada_layer_norm_zero:
        UpperCamelCase      :   int			      =  norm_hidden_states * (1 + scale_mlp[:, None]) + shift_mlp[:, None]

       if self._chunk_size is not None:
        # "feed_forward_chunk_size" can be used to save memory
        if norm_hidden_states.shape[self._chunk_dim] % self._chunk_size != 0:
         raise ValueError(
             F"""`hidden_states` dimension to be chunked: {norm_hidden_states.shape[self._chunk_dim]} has to be divisible by chunk size: {self._chunk_size}. Make sure to set an appropriate `chunk_size` when calling `unet.enable_forward_chunking`."""  )

        UpperCamelCase      :   str			      =  norm_hidden_states.shape[self._chunk_dim] // self._chunk_size
        UpperCamelCase      :   Tuple			      =  torch.cat(
            [self.ff(A_  ) for hid_slice in norm_hidden_states.chunk(A_			,				dim=self._chunk_dim  )]			,				dim=self._chunk_dim			,				)
       else:
        UpperCamelCase      :   List[str]			      =  self.ff(A_  )

       if self.use_ada_layer_norm_zero:
        UpperCamelCase      :   Union[str, Any]			      =  gate_mlp.unsqueeze(1  ) * ff_output

       UpperCamelCase      :   Optional[Any]			      =  ff_output + hidden_states

       return hidden_states



class    A__  (			nn.Module     ):





      def __init__(					self			,				A_			,				A_ = None			,				A_ = 4			,				A_ = 0.0			,				A_ = "geglu"			,				A_ = False			,				):
       '''simple docstring'''




       super().__init__()
       UpperCamelCase      :   int			      =  int(dim * mult  )
       UpperCamelCase      :   Dict			      =  dim_out if dim_out is not None else dim

       if activation_fn == "gelu":
        UpperCamelCase      :   str			      =  GELU(A_			,				A_  )
       if activation_fn == "gelu-approximate":
        UpperCamelCase      :   Any			      =  GELU(A_			,				A_			,				approximate="tanh"  )
       elif activation_fn == "geglu":
        UpperCamelCase      :   Optional[int]			      =  GEGLU(A_			,				A_  )
       elif activation_fn == "geglu-approximate":
        UpperCamelCase      :   Any			      =  ApproximateGELU(A_			,				A_  )

       UpperCamelCase      :   Optional[Any]			      =  nn.ModuleList([]  )
       # project in
       self.net.append(A_  )
       # project dropout
       self.net.append(nn.Dropout(A_  )  )
       # project out
       self.net.append(nn.Linear(A_			,				A_  )  )
       # FF as used in Vision Transformer, MLP-Mixer, etc. have a final dropout
       if final_dropout:
        self.net.append(nn.Dropout(A_  )  )





      def 			__UpperCamelCase(					self			,				A_  ):
       '''simple docstring'''




       for module in self.net:
        UpperCamelCase      :   Tuple			      =  module(A_  )
       return hidden_states



class    A__  (			nn.Module     ):





      def __init__(					self			,				A_			,				A_			,				A_ = "none"  ):
       '''simple docstring'''




       super().__init__()
       UpperCamelCase      :   List[str]			      =  nn.Linear(A_			,				A_  )
       UpperCamelCase      :   int			      =  approximate





      def 			__UpperCamelCase(					self			,				A_  ):
       '''simple docstring'''




       if gate.device.type != "mps":
        return F.gelu(A_			,				approximate=self.approximate  )
       # mps: gelu is not implemented for float16
       return F.gelu(gate.to(dtype=torch.floataa  )			,				approximate=self.approximate  ).to(dtype=gate.dtype  )





      def 			__UpperCamelCase(					self			,				A_  ):
       '''simple docstring'''




       UpperCamelCase      :   List[str]			      =  self.proj(A_  )
       UpperCamelCase      :   Tuple			      =  self.gelu(A_  )
       return hidden_states



class    A__  (			nn.Module     ):





      def __init__(					self			,				A_			,				A_  ):
       '''simple docstring'''




       super().__init__()
       UpperCamelCase      :   Optional[int]			      =  nn.Linear(A_			,				dim_out * 2  )





      def 			__UpperCamelCase(					self			,				A_  ):
       '''simple docstring'''




       if gate.device.type != "mps":
        return F.gelu(A_  )
       # mps: gelu is not implemented for float16
       return F.gelu(gate.to(dtype=torch.floataa  )  ).to(dtype=gate.dtype  )





      def 			__UpperCamelCase(					self			,				A_  ):
       '''simple docstring'''




       UpperCamelCase      ,       UpperCamelCase      :   int			      =  self.proj(A_  ).chunk(2			,				dim=-1  )
       return hidden_states * self.gelu(A_  )



class    A__  (			nn.Module     ):





      def __init__(					self			,				A_			,				A_  ):
       '''simple docstring'''




       super().__init__()
       UpperCamelCase      :   Optional[int]			      =  nn.Linear(A_			,				A_  )





      def 			__UpperCamelCase(					self			,				A_  ):
       '''simple docstring'''




       UpperCamelCase      :   List[str]			      =  self.proj(A_  )
       return x * torch.sigmoid(1.7_02 * x  )



class    A__  (			nn.Module     ):





      def __init__(					self			,				A_			,				A_  ):
       '''simple docstring'''




       super().__init__()
       UpperCamelCase      :   Any			      =  nn.Embedding(A_			,				A_  )
       UpperCamelCase      :   Dict			      =  nn.SiLU()
       UpperCamelCase      :   Optional[Any]			      =  nn.Linear(A_			,				embedding_dim * 2  )
       UpperCamelCase      :   List[Any]			      =  nn.LayerNorm(A_			,				elementwise_affine=A_  )





      def 			__UpperCamelCase(					self			,				A_			,				A_  ):
       '''simple docstring'''




       UpperCamelCase      :   List[str]			      =  self.linear(self.silu(self.emb(A_  )  )  )
       UpperCamelCase      ,       UpperCamelCase      :   Union[str, Any]			      =  torch.chunk(A_			,				2  )
       UpperCamelCase      :   Tuple			      =  self.norm(A_  ) * (1 + scale) + shift
       return x



class    A__  (			nn.Module     ):





      def __init__(					self			,				A_			,				A_  ):
       '''simple docstring'''




       super().__init__()

       UpperCamelCase      :   int			      =  CombinedTimestepLabelEmbeddings(A_			,				A_  )

       UpperCamelCase      :   int			      =  nn.SiLU()
       UpperCamelCase      :   Dict			      =  nn.Linear(A_			,				6 * embedding_dim			,				bias=A_  )
       UpperCamelCase      :   List[Any]			      =  nn.LayerNorm(A_			,				elementwise_affine=A_			,				eps=1e-6  )





      def 			__UpperCamelCase(					self			,				A_			,				A_			,				A_			,				A_=None  ):
       '''simple docstring'''




       UpperCamelCase      :   Optional[Any]			      =  self.linear(self.silu(self.emb(A_			,				A_			,				hidden_dtype=A_  )  )  )
       UpperCamelCase      ,       UpperCamelCase      ,       UpperCamelCase      ,       UpperCamelCase      ,       UpperCamelCase      ,       UpperCamelCase      :   Union[str, Any]			      =  emb.chunk(6			,				dim=1  )
       UpperCamelCase      :   List[str]			      =  self.norm(A_  ) * (1 + scale_msa[:, None]) + shift_msa[:, None]
       return x, gate_msa, shift_mlp, scale_mlp, gate_mlp



class    A__  (			nn.Module     ):





      def __init__(					self			,				A_			,				A_			,				A_			,				A_ = None			,				A_ = 1e-5  ):
       '''simple docstring'''




       super().__init__()
       UpperCamelCase      :   List[str]			      =  num_groups
       UpperCamelCase      :   Optional[int]			      =  eps

       if act_fn is None:
        UpperCamelCase      :   Optional[Any]			      =  None
       else:
        UpperCamelCase      :   Any			      =  get_activation(A_  )

       UpperCamelCase      :   Optional[int]			      =  nn.Linear(A_			,				out_dim * 2  )





      def 			__UpperCamelCase(					self			,				A_			,				A_  ):
       '''simple docstring'''




       if self.act:
        UpperCamelCase      :   List[Any]			      =  self.act(A_  )
       UpperCamelCase      :   Optional[int]			      =  self.linear(A_  )
       UpperCamelCase      :   Optional[int]			      =  emb[:, :, None, None]
       UpperCamelCase      ,       UpperCamelCase      :   List[Any]			      =  emb.chunk(2			,				dim=1  )

       UpperCamelCase      :   Optional[Any]			      =  F.group_norm(A_			,				self.num_groups			,				eps=self.eps  )
       UpperCamelCase      :   List[str]			      =  x * (1 + scale) + shift
       return x







from typing import Any



def 						A_							(	_lowerCAmelCase	)		->					list[Any]:
 if not input_list:
  return []
 UpperCamelCase      :   List[str]			      =  [input_list.count(_lowerCAmelCase	) for value in input_list]
 UpperCamelCase      :   Dict			      =  max(_lowerCAmelCase	)  # Gets the maximum count in the input list.
 # Gets values of modes
 return sorted({input_list[i] for i, value in enumerate(_lowerCAmelCase	) if value == y}	)


if __name__ == "__main__":
    import doctest

    doctest.testmod()
