from typing import Any, Dict, Optional

import torch
import torch.nn.functional as F
from torch import nn

from ..utils import maybe_allow_in_graph
from .activations import get_activation
from .attention_processor import Attention
from .embeddings import CombinedTimestepLabelEmbeddings



@maybe_allow_in_graph
class 			SCREAMING_SNAKE_CASE_			(       nn.Module      ):
  '''simple docstring'''


  def __init__(					self     :       Any   ,						SCREAMING_SNAKE_CASE__     :       List[str]   ,						SCREAMING_SNAKE_CASE__     :       Optional[Any]   ,						SCREAMING_SNAKE_CASE__     :       List[Any]   ,						SCREAMING_SNAKE_CASE__     :       Any=0.0   ,						SCREAMING_SNAKE_CASE__     :       Union[str, Any] = None   ,						SCREAMING_SNAKE_CASE__     :       Union[str, Any] = "geglu"   ,						SCREAMING_SNAKE_CASE__     :       Dict = None   ,						SCREAMING_SNAKE_CASE__     :       Dict = False   ,						SCREAMING_SNAKE_CASE__     :       List[Any] = False   ,						SCREAMING_SNAKE_CASE__     :       List[str] = False   ,						SCREAMING_SNAKE_CASE__     :       Union[str, Any] = False   ,						SCREAMING_SNAKE_CASE__     :       Dict = True   ,						SCREAMING_SNAKE_CASE__     :       str = "layer_norm"   ,						SCREAMING_SNAKE_CASE__     :       List[Any] = False   ,						)     ->					int:
         super().__init__()
         A    :  Optional[Any]								=only_cross_attention

         A    :  Any								=(num_embeds_ada_norm is not None) and norm_type == "ada_norm_zero"
         A    :  Tuple								=(num_embeds_ada_norm is not None) and norm_type == "ada_norm"

         if norm_type in ("ada_norm", "ada_norm_zero") and num_embeds_ada_norm is None:
                raise ValueError(
                    f'`norm_type` is set to {norm_type}, but `num_embeds_ada_norm` is not defined. Please make sure to'
                    f' define `num_embeds_ada_norm` if setting `norm_type` to {norm_type}.'  )

         # Define 3 blocks. Each block has its own normalization layer.
         # 1. Self-Attn
         if self.use_ada_layer_norm:
                A    :  Optional[Any]								=AdaLayerNorm(A_   ,						A_  )
         elif self.use_ada_layer_norm_zero:
                A    :  Any								=AdaLayerNormZero(A_   ,						A_  )
         else:
                A    :  List[str]								=nn.LayerNorm(A_   ,						elementwise_affine=A_  )
         A    :  Tuple								=Attention(
             query_dim=A_   ,						heads=A_   ,						dim_head=A_   ,						dropout=A_   ,						bias=A_   ,						cross_attention_dim=cross_attention_dim if only_cross_attention else None   ,						upcast_attention=A_   ,						)

         # 2. Cross-Attn
         if cross_attention_dim is not None or double_self_attention:
                # We currently only use AdaLayerNormZero for self attention where there will only be one attention block.
                # I.e. the number of returned modulation chunks from AdaLayerZero would not make sense if returned during
                # the second cross attention block.
                A    :  Tuple								=(
                    AdaLayerNorm(A_   ,						A_  )
                    if self.use_ada_layer_norm
                    else nn.LayerNorm(A_   ,						elementwise_affine=A_  )
                )
                A    :  Optional[int]								=Attention(
                    query_dim=A_   ,						cross_attention_dim=cross_attention_dim if not double_self_attention else None   ,						heads=A_   ,						dim_head=A_   ,						dropout=A_   ,						bias=A_   ,						upcast_attention=A_   ,						)  # is self-attn if encoder_hidden_states is none
         else:
                A    :  Optional[Any]								=None
                A    :  List[str]								=None

         # 3. Feed-forward
         A    :  Tuple								=nn.LayerNorm(A_   ,						elementwise_affine=A_  )
         A    :  str								=FeedForward(A_   ,						dropout=A_   ,						activation_fn=A_   ,						final_dropout=A_  )

         # let chunk size default to None
         A    :  Any								=None
         A    :  Dict								=0







  def 		SCREAMING_SNAKE_CASE_		(					self     :       Any   ,						SCREAMING_SNAKE_CASE__     :       str   ,						SCREAMING_SNAKE_CASE__     :       Union[str, Any]  )     ->					Any:
         A    :  str								=chunk_size
         A    :  Dict								=dim







  def 		SCREAMING_SNAKE_CASE_		(					self     :       Optional[Any]   ,						SCREAMING_SNAKE_CASE__     :       Dict   ,						SCREAMING_SNAKE_CASE__     :       List[str] = None   ,						SCREAMING_SNAKE_CASE__     :       Tuple = None   ,						SCREAMING_SNAKE_CASE__     :       List[str] = None   ,						SCREAMING_SNAKE_CASE__     :       Any = None   ,						SCREAMING_SNAKE_CASE__     :       int = None   ,						SCREAMING_SNAKE_CASE__     :       Optional[Any] = None   ,						)     ->					int:




         if self.use_ada_layer_norm:
                A    :  List[Any]								=self.norma(A_   ,						A_  )
         elif self.use_ada_layer_norm_zero:
                A    :  Optional[int]								=self.norma(
                    A_   ,						A_   ,						A_   ,						hidden_dtype=hidden_states.dtype  )
         else:
                A    :  int								=self.norma(A_  )

         A    :  Tuple								=cross_attention_kwargs if cross_attention_kwargs is not None else {}

         A    :  Optional[Any]								=self.attna(
             A_   ,						encoder_hidden_states=encoder_hidden_states if self.only_cross_attention else None   ,						attention_mask=A_   ,						**A_   ,						)
         if self.use_ada_layer_norm_zero:
                A    :  List[Any]								=gate_msa.unsqueeze(1  ) * attn_output
         A    :  str								=attn_output + hidden_states

         # 2. Cross-Attention
         if self.attna is not None:
                A    :  List[Any]								=(
                    self.norma(A_   ,						A_  ) if self.use_ada_layer_norm else self.norma(A_  )
                )

                A    :  Optional[int]								=self.attna(
                    A_   ,						encoder_hidden_states=A_   ,						attention_mask=A_   ,						**A_   ,						)
                A    :  Tuple								=attn_output + hidden_states

         # 3. Feed-forward
         A    :  Any								=self.norma(A_  )

         if self.use_ada_layer_norm_zero:
                A    :  int								=norm_hidden_states * (1 + scale_mlp[:, None]) + shift_mlp[:, None]

         if self._chunk_size is not None:
                # "feed_forward_chunk_size" can be used to save memory
                if norm_hidden_states.shape[self._chunk_dim] % self._chunk_size != 0:
                       raise ValueError(
                           f'`hidden_states` dimension to be chunked: {norm_hidden_states.shape[self._chunk_dim]} has to be divisible by chunk size: {self._chunk_size}. Make sure to set an appropriate `chunk_size` when calling `unet.enable_forward_chunking`.'  )

                A    :  str								=norm_hidden_states.shape[self._chunk_dim] // self._chunk_size
                A    :  Tuple								=torch.cat(
                    [self.ff(A_  ) for hid_slice in norm_hidden_states.chunk(A_   ,						dim=self._chunk_dim  )]   ,						dim=self._chunk_dim   ,						)
         else:
                A    :  List[str]								=self.ff(A_  )

         if self.use_ada_layer_norm_zero:
                A    :  Union[str, Any]								=gate_mlp.unsqueeze(1  ) * ff_output

         A    :  Optional[Any]								=ff_output + hidden_states

         return hidden_states







class 			SCREAMING_SNAKE_CASE_			(       nn.Module      ):
  '''simple docstring'''


  def __init__(					self     :       str   ,						SCREAMING_SNAKE_CASE__     :       Tuple   ,						SCREAMING_SNAKE_CASE__     :       int = None   ,						SCREAMING_SNAKE_CASE__     :       Dict = 4   ,						SCREAMING_SNAKE_CASE__     :       Optional[Any] = 0.0   ,						SCREAMING_SNAKE_CASE__     :       int = "geglu"   ,						SCREAMING_SNAKE_CASE__     :       List[Any] = False   ,						)     ->					int:
         super().__init__()
         A    :  int								=int(dim * mult  )
         A    :  Dict								=dim_out if dim_out is not None else dim

         if activation_fn == "gelu":
                A    :  str								=GELU(A_   ,						A_  )
         if activation_fn == "gelu-approximate":
                A    :  Any								=GELU(A_   ,						A_   ,						approximate='tanh'  )
         elif activation_fn == "geglu":
                A    :  Optional[int]								=GEGLU(A_   ,						A_  )
         elif activation_fn == "geglu-approximate":
                A    :  Any								=ApproximateGELU(A_   ,						A_  )

         A    :  Optional[Any]								=nn.ModuleList([]  )
         # project in
         self.net.append(A_  )
         # project dropout
         self.net.append(nn.Dropout(A_  )  )
         # project out
         self.net.append(nn.Linear(A_   ,						A_  )  )
         # FF as used in Vision Transformer, MLP-Mixer, etc. have a final dropout
         if final_dropout:
                self.net.append(nn.Dropout(A_  )  )







  def 		SCREAMING_SNAKE_CASE_		(					self     :       Tuple   ,						SCREAMING_SNAKE_CASE__     :       Tuple  )     ->					Dict:




         for module in self.net:
                A    :  Tuple								=module(A_  )
         return hidden_states







class 			SCREAMING_SNAKE_CASE_			(       nn.Module      ):
  '''simple docstring'''


  def __init__(					self     :       List[str]   ,						SCREAMING_SNAKE_CASE__     :       Optional[Any]   ,						SCREAMING_SNAKE_CASE__     :       Optional[Any]   ,						SCREAMING_SNAKE_CASE__     :       List[str] = "none"  )     ->					List[str]:
         super().__init__()
         A    :  List[str]								=nn.Linear(A_   ,						A_  )
         A    :  int								=approximate







  def 		SCREAMING_SNAKE_CASE_		(					self     :       Union[str, Any]   ,						SCREAMING_SNAKE_CASE__     :       str  )     ->					Union[str, Any]:




         if gate.device.type != "mps":
                return F.gelu(A_   ,						approximate=self.approximate  )
         # mps: gelu is not implemented for float16
         return F.gelu(gate.to(dtype=torch.floataa  )   ,						approximate=self.approximate  ).to(dtype=gate.dtype  )







  def 		SCREAMING_SNAKE_CASE_		(					self     :       Union[str, Any]   ,						SCREAMING_SNAKE_CASE__     :       Dict  )     ->					List[Any]:
         A    :  List[str]								=self.proj(A_  )
         A    :  Tuple								=self.gelu(A_  )
         return hidden_states







class 			SCREAMING_SNAKE_CASE_			(       nn.Module      ):
  '''simple docstring'''


  def __init__(					self     :       List[Any]   ,						SCREAMING_SNAKE_CASE__     :       Optional[Any]   ,						SCREAMING_SNAKE_CASE__     :       Dict  )     ->					str:
         super().__init__()
         A    :  Optional[int]								=nn.Linear(A_   ,						dim_out * 2  )







  def 		SCREAMING_SNAKE_CASE_		(					self     :       List[Any]   ,						SCREAMING_SNAKE_CASE__     :       Union[str, Any]  )     ->					Union[str, Any]:




         if gate.device.type != "mps":
                return F.gelu(A_  )
         # mps: gelu is not implemented for float16
         return F.gelu(gate.to(dtype=torch.floataa  )  ).to(dtype=gate.dtype  )







  def 		SCREAMING_SNAKE_CASE_		(					self     :       Optional[int]   ,						SCREAMING_SNAKE_CASE__     :       List[Any]  )     ->					List[str]:
         A    :  int								=self.proj(A_  ).chunk(2   ,						dim=-1  )
         return hidden_states * self.gelu(A_  )







class 			SCREAMING_SNAKE_CASE_			(       nn.Module      ):
  '''simple docstring'''


  def __init__(					self     :       List[str]   ,						SCREAMING_SNAKE_CASE__     :       Optional[Any]   ,						SCREAMING_SNAKE_CASE__     :       Optional[Any]  )     ->					Optional[Any]:
         super().__init__()
         A    :  Optional[int]								=nn.Linear(A_   ,						A_  )







  def 		SCREAMING_SNAKE_CASE_		(					self     :       Dict   ,						SCREAMING_SNAKE_CASE__     :       Optional[int]  )     ->					Any:
         A    :  List[str]								=self.proj(A_  )
         return x * torch.sigmoid(1.7_0_2 * x  )







class 			SCREAMING_SNAKE_CASE_			(       nn.Module      ):
  '''simple docstring'''


  def __init__(					self     :       List[Any]   ,						SCREAMING_SNAKE_CASE__     :       Tuple   ,						SCREAMING_SNAKE_CASE__     :       str  )     ->					Tuple:
         super().__init__()
         A    :  Any								=nn.Embedding(A_   ,						A_  )
         A    :  Dict								=nn.SiLU()
         A    :  Optional[Any]								=nn.Linear(A_   ,						embedding_dim * 2  )
         A    :  List[Any]								=nn.LayerNorm(A_   ,						elementwise_affine=A_  )







  def 		SCREAMING_SNAKE_CASE_		(					self     :       Dict   ,						SCREAMING_SNAKE_CASE__     :       List[str]   ,						SCREAMING_SNAKE_CASE__     :       List[str]  )     ->					Tuple:
         A    :  List[str]								=self.linear(self.silu(self.emb(A_  )  )  )
         A    :  Union[str, Any]								=torch.chunk(A_   ,						2  )
         A    :  Tuple								=self.norm(A_  ) * (1 + scale) + shift
         return x







class 			SCREAMING_SNAKE_CASE_			(       nn.Module      ):
  '''simple docstring'''


  def __init__(					self     :       List[Any]   ,						SCREAMING_SNAKE_CASE__     :       Any   ,						SCREAMING_SNAKE_CASE__     :       List[Any]  )     ->					List[str]:
         super().__init__()

         A    :  int								=CombinedTimestepLabelEmbeddings(A_   ,						A_  )

         A    :  int								=nn.SiLU()
         A    :  Dict								=nn.Linear(A_   ,						6 * embedding_dim   ,						bias=A_  )
         A    :  List[Any]								=nn.LayerNorm(A_   ,						elementwise_affine=A_   ,						eps=1e-6  )







  def 		SCREAMING_SNAKE_CASE_		(					self     :       Any   ,						SCREAMING_SNAKE_CASE__     :       Tuple   ,						SCREAMING_SNAKE_CASE__     :       Optional[Any]   ,						SCREAMING_SNAKE_CASE__     :       Any   ,						SCREAMING_SNAKE_CASE__     :       Optional[int]=None  )     ->					List[Any]:
         A    :  Optional[Any]								=self.linear(self.silu(self.emb(A_   ,						A_   ,						hidden_dtype=A_  )  )  )
         A    :  Union[str, Any]								=emb.chunk(6   ,						dim=1  )
         A    :  List[str]								=self.norm(A_  ) * (1 + scale_msa[:, None]) + shift_msa[:, None]
         return x, gate_msa, shift_mlp, scale_mlp, gate_mlp







class 			SCREAMING_SNAKE_CASE_			(       nn.Module      ):
  '''simple docstring'''


  def __init__(					self     :       Optional[int]   ,						SCREAMING_SNAKE_CASE__     :       int   ,						SCREAMING_SNAKE_CASE__     :       Any   ,						SCREAMING_SNAKE_CASE__     :       int   ,						SCREAMING_SNAKE_CASE__     :       Tuple = None   ,						SCREAMING_SNAKE_CASE__     :       Tuple = 1e-5  )     ->					str:
         super().__init__()
         A    :  List[str]								=num_groups
         A    :  Optional[int]								=eps

         if act_fn is None:
                A    :  Optional[Any]								=None
         else:
                A    :  Any								=get_activation(A_  )

         A    :  Optional[int]								=nn.Linear(A_   ,						out_dim * 2  )







  def 		SCREAMING_SNAKE_CASE_		(					self     :       List[str]   ,						SCREAMING_SNAKE_CASE__     :       Any   ,						SCREAMING_SNAKE_CASE__     :       List[Any]  )     ->					str:




         if self.act:
                A    :  List[Any]								=self.act(A_  )
         A    :  Optional[int]								=self.linear(A_  )
         A    :  Optional[int]								=emb[:, :, None, None]
         A    :  List[Any]								=emb.chunk(2   ,						dim=1  )

         A    :  Optional[Any]								=F.group_norm(A_   ,						self.num_groups   ,						eps=self.eps  )
         A    :  List[str]								=x * (1 + scale) + shift
         return x













from typing import TYPE_CHECKING

from ...utils import OptionalDependencyNotAvailable, _LazyModule, is_torch_available, is_vision_available


__lowerCamelCase			:	List[Any]					    =	{
    """configuration_chinese_clip""": [
        """CHINESE_CLIP_PRETRAINED_CONFIG_ARCHIVE_MAP""",
        """ChineseCLIPConfig""",
        """ChineseCLIPOnnxConfig""",
        """ChineseCLIPTextConfig""",
        """ChineseCLIPVisionConfig""",
    ],
    """processing_chinese_clip""": ["""ChineseCLIPProcessor"""],
}

try:
    if not is_vision_available():
        raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
    pass
else:
    __lowerCamelCase			:	Tuple					    =	["""ChineseCLIPFeatureExtractor"""]
    __lowerCamelCase			:	List[str]					    =	["""ChineseCLIPImageProcessor"""]

try:
    if not is_torch_available():
        raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
    pass
else:
    __lowerCamelCase			:	str					    =	[
        """CHINESE_CLIP_PRETRAINED_MODEL_ARCHIVE_LIST""",
        """ChineseCLIPModel""",
        """ChineseCLIPPreTrainedModel""",
        """ChineseCLIPTextModel""",
        """ChineseCLIPVisionModel""",
    ]

if TYPE_CHECKING:
    from .configuration_chinese_clip import (
        CHINESE_CLIP_PRETRAINED_CONFIG_ARCHIVE_MAP,
        ChineseCLIPConfig,
        ChineseCLIPOnnxConfig,
        ChineseCLIPTextConfig,
        ChineseCLIPVisionConfig,
    )
    from .processing_chinese_clip import ChineseCLIPProcessor

    try:
        if not is_vision_available():
            raise OptionalDependencyNotAvailable()
    except OptionalDependencyNotAvailable:
        pass
    else:
        from .feature_extraction_chinese_clip import ChineseCLIPFeatureExtractor, ChineseCLIPImageProcessor

    try:
        if not is_torch_available():
            raise OptionalDependencyNotAvailable()
    except OptionalDependencyNotAvailable:
        pass
    else:
        from .modeling_chinese_clip import (
            CHINESE_CLIP_PRETRAINED_MODEL_ARCHIVE_LIST,
            ChineseCLIPModel,
            ChineseCLIPPreTrainedModel,
            ChineseCLIPTextModel,
            ChineseCLIPVisionModel,
        )

else:
    import sys

    __lowerCamelCase			:	Any					    =	_LazyModule(__name__, globals()["""__file__"""], _import_structure, module_spec=__spec__)
