




from typing import Any, Dict, Optional

import torch
import torch.nn.functional as F
from torch import nn

from ..utils import maybe_allow_in_graph
from .activations import get_activation
from .attention_processor import Attention
from .embeddings import CombinedTimestepLabelEmbeddings


@maybe_allow_in_graph
class  A		(  nn.Module     ):



		def __init__(     self,	UpperCamelCase__,	UpperCamelCase__,	UpperCamelCase__,	UpperCamelCase__=0.0,	UpperCamelCase__ = None,	UpperCamelCase__ = "geglu",	UpperCamelCase__ = None,	UpperCamelCase__ = False,	UpperCamelCase__ = False,	UpperCamelCase__ = False,	UpperCamelCase__ = False,	UpperCamelCase__ = True,	UpperCamelCase__ = "layer_norm",	UpperCamelCase__ = False,	):







								"""simple docstring"""



								super().__init__()
								lowerCAmelCase_      			=  only_cross_attention

								lowerCAmelCase_      			=  (num_embeds_ada_norm is not None) and norm_type == '''ada_norm_zero'''
								lowerCAmelCase_      			=  (num_embeds_ada_norm is not None) and norm_type == '''ada_norm'''

								if norm_type in ("ada_norm", "ada_norm_zero") and num_embeds_ada_norm is None:
														raise ValueError(
														    f"`norm_type` is set to {norm_type}, but `num_embeds_ada_norm` is not defined. Please make sure to"
														    f" define `num_embeds_ada_norm` if setting `norm_type` to {norm_type}."   )

								# Define 3 blocks. Each block has its own normalization layer.
								# 1. Self-Attn
								if self.use_ada_layer_norm:
														lowerCAmelCase_      			=  AdaLayerNorm(UpperCamelCase__,	UpperCamelCase__   )
								elif self.use_ada_layer_norm_zero:
														lowerCAmelCase_      			=  AdaLayerNormZero(UpperCamelCase__,	UpperCamelCase__   )
								else:
														lowerCAmelCase_      			=  nn.LayerNorm(UpperCamelCase__,	elementwise_affine=UpperCamelCase__   )
								lowerCAmelCase_      			=  Attention(
								    query_dim=UpperCamelCase__,	heads=UpperCamelCase__,	dim_head=UpperCamelCase__,	dropout=UpperCamelCase__,	bias=UpperCamelCase__,	cross_attention_dim=cross_attention_dim if only_cross_attention else None,	upcast_attention=UpperCamelCase__,	)

								# 2. Cross-Attn
								if cross_attention_dim is not None or double_self_attention:
														# We currently only use AdaLayerNormZero for self attention where there will only be one attention block.
														# I.e. the number of returned modulation chunks from AdaLayerZero would not make sense if returned during
														# the second cross attention block.
														lowerCAmelCase_      			=  (
														    AdaLayerNorm(UpperCamelCase__,	UpperCamelCase__   )
														    if self.use_ada_layer_norm
														    else nn.LayerNorm(UpperCamelCase__,	elementwise_affine=UpperCamelCase__   )
														)
														lowerCAmelCase_      			=  Attention(
														    query_dim=UpperCamelCase__,	cross_attention_dim=cross_attention_dim if not double_self_attention else None,	heads=UpperCamelCase__,	dim_head=UpperCamelCase__,	dropout=UpperCamelCase__,	bias=UpperCamelCase__,	upcast_attention=UpperCamelCase__,	)  # is self-attn if encoder_hidden_states is none
								else:
														lowerCAmelCase_      			=  None
														lowerCAmelCase_      			=  None

								# 3. Feed-forward
								lowerCAmelCase_      			=  nn.LayerNorm(UpperCamelCase__,	elementwise_affine=UpperCamelCase__   )
								lowerCAmelCase_      			=  FeedForward(UpperCamelCase__,	dropout=UpperCamelCase__,	activation_fn=UpperCamelCase__,	final_dropout=UpperCamelCase__   )

								# let chunk size default to None
								lowerCAmelCase_      			=  None
								lowerCAmelCase_      			=  0



		def        SCREAMING_SNAKE_CASE__			(     self,	UpperCamelCase__,	UpperCamelCase__   ):







								"""simple docstring"""



								lowerCAmelCase_      			=  chunk_size
								lowerCAmelCase_      			=  dim







		def        SCREAMING_SNAKE_CASE__			(     self,	UpperCamelCase__,	UpperCamelCase__ = None,	UpperCamelCase__ = None,	UpperCamelCase__ = None,	UpperCamelCase__ = None,	UpperCamelCase__ = None,	UpperCamelCase__ = None,	):







								"""simple docstring"""



								if self.use_ada_layer_norm:
														lowerCAmelCase_      			=  self.norma(UpperCamelCase__,	UpperCamelCase__   )
								elif self.use_ada_layer_norm_zero:
														lowerCAmelCase_			,					lowerCAmelCase_			,					lowerCAmelCase_			,					lowerCAmelCase_			,					lowerCAmelCase_      			=  self.norma(
														    UpperCamelCase__,	UpperCamelCase__,	UpperCamelCase__,	hidden_dtype=hidden_states.dtype   )
								else:
														lowerCAmelCase_      			=  self.norma(UpperCamelCase__   )

								lowerCAmelCase_      			=  cross_attention_kwargs if cross_attention_kwargs is not None else {}

								lowerCAmelCase_      			=  self.attna(
								    UpperCamelCase__,	encoder_hidden_states=encoder_hidden_states if self.only_cross_attention else None,	attention_mask=UpperCamelCase__,	**UpperCamelCase__,	)
								if self.use_ada_layer_norm_zero:
														lowerCAmelCase_      			=  gate_msa.unsqueeze(1   ) * attn_output
								lowerCAmelCase_      			=  attn_output + hidden_states

								# 2. Cross-Attention
								if self.attna is not None:
														lowerCAmelCase_      			=  (
														    self.norma(UpperCamelCase__,	UpperCamelCase__   ) if self.use_ada_layer_norm else self.norma(UpperCamelCase__   )
														)

														lowerCAmelCase_      			=  self.attna(
														    UpperCamelCase__,	encoder_hidden_states=UpperCamelCase__,	attention_mask=UpperCamelCase__,	**UpperCamelCase__,	)
														lowerCAmelCase_      			=  attn_output + hidden_states

								# 3. Feed-forward
								lowerCAmelCase_      			=  self.norma(UpperCamelCase__   )

								if self.use_ada_layer_norm_zero:
														lowerCAmelCase_      			=  norm_hidden_states * (1 + scale_mlp[:, None]) + shift_mlp[:, None]

								if self._chunk_size is not None:
														# "feed_forward_chunk_size" can be used to save memory
														if norm_hidden_states.shape[self._chunk_dim] % self._chunk_size != 0:
																				raise ValueError(
																				    f"`hidden_states` dimension to be chunked: {norm_hidden_states.shape[self._chunk_dim]} has to be divisible by chunk size: {self._chunk_size}. Make sure to set an appropriate `chunk_size` when calling `unet.enable_forward_chunking`."   )

														lowerCAmelCase_      			=  norm_hidden_states.shape[self._chunk_dim] // self._chunk_size
														lowerCAmelCase_      			=  torch.cat(
														    [self.ff(UpperCamelCase__   ) for hid_slice in norm_hidden_states.chunk(UpperCamelCase__,	dim=self._chunk_dim   )],	dim=self._chunk_dim,	)
								else:
														lowerCAmelCase_      			=  self.ff(UpperCamelCase__   )

								if self.use_ada_layer_norm_zero:
														lowerCAmelCase_      			=  gate_mlp.unsqueeze(1   ) * ff_output

								lowerCAmelCase_      			=  ff_output + hidden_states

								return hidden_states







class  A		(  nn.Module     ):



		def __init__(     self,	UpperCamelCase__,	UpperCamelCase__ = None,	UpperCamelCase__ = 4,	UpperCamelCase__ = 0.0,	UpperCamelCase__ = "geglu",	UpperCamelCase__ = False,	):







								"""simple docstring"""



								super().__init__()
								lowerCAmelCase_      			=  int(dim * mult   )
								lowerCAmelCase_      			=  dim_out if dim_out is not None else dim

								if activation_fn == "gelu":
														lowerCAmelCase_      			=  GELU(UpperCamelCase__,	UpperCamelCase__   )
								if activation_fn == "gelu-approximate":
														lowerCAmelCase_      			=  GELU(UpperCamelCase__,	UpperCamelCase__,	approximate='''tanh'''   )
								elif activation_fn == "geglu":
														lowerCAmelCase_      			=  GEGLU(UpperCamelCase__,	UpperCamelCase__   )
								elif activation_fn == "geglu-approximate":
														lowerCAmelCase_      			=  ApproximateGELU(UpperCamelCase__,	UpperCamelCase__   )

								lowerCAmelCase_      			=  nn.ModuleList([]   )
								# project in
								self.net.append(UpperCamelCase__   )
								# project dropout
								self.net.append(nn.Dropout(UpperCamelCase__   )   )
								# project out
								self.net.append(nn.Linear(UpperCamelCase__,	UpperCamelCase__   )   )
								# FF as used in Vision Transformer, MLP-Mixer, etc. have a final dropout
								if final_dropout:
														self.net.append(nn.Dropout(UpperCamelCase__   )   )







		def        SCREAMING_SNAKE_CASE__			(     self,	UpperCamelCase__   ):







								"""simple docstring"""



								for module in self.net:
														lowerCAmelCase_      			=  module(UpperCamelCase__   )
								return hidden_states







class  A		(  nn.Module     ):



		def __init__(     self,	UpperCamelCase__,	UpperCamelCase__,	UpperCamelCase__ = "none"   ):







								"""simple docstring"""



								super().__init__()
								lowerCAmelCase_      			=  nn.Linear(UpperCamelCase__,	UpperCamelCase__   )
								lowerCAmelCase_      			=  approximate



		def        SCREAMING_SNAKE_CASE__			(     self,	UpperCamelCase__   ):







								"""simple docstring"""



								if gate.device.type != "mps":
														return F.gelu(UpperCamelCase__,	approximate=self.approximate   )
								# mps: gelu is not implemented for float16
								return F.gelu(gate.to(dtype=torch.floataa   ),	approximate=self.approximate   ).to(dtype=gate.dtype   )







		def        SCREAMING_SNAKE_CASE__			(     self,	UpperCamelCase__   ):







								"""simple docstring"""



								lowerCAmelCase_      			=  self.proj(UpperCamelCase__   )
								lowerCAmelCase_      			=  self.gelu(UpperCamelCase__   )
								return hidden_states







class  A		(  nn.Module     ):



		def __init__(     self,	UpperCamelCase__,	UpperCamelCase__   ):







								"""simple docstring"""



								super().__init__()
								lowerCAmelCase_      			=  nn.Linear(UpperCamelCase__,	dim_out * 2   )



		def        SCREAMING_SNAKE_CASE__			(     self,	UpperCamelCase__   ):







								"""simple docstring"""



								if gate.device.type != "mps":
														return F.gelu(UpperCamelCase__   )
								# mps: gelu is not implemented for float16
								return F.gelu(gate.to(dtype=torch.floataa   )   ).to(dtype=gate.dtype   )







		def        SCREAMING_SNAKE_CASE__			(     self,	UpperCamelCase__   ):







								"""simple docstring"""



								lowerCAmelCase_			,					lowerCAmelCase_      			=  self.proj(UpperCamelCase__   ).chunk(2,	dim=-1   )
								return hidden_states * self.gelu(UpperCamelCase__   )







class  A		(  nn.Module     ):



		def __init__(     self,	UpperCamelCase__,	UpperCamelCase__   ):







								"""simple docstring"""



								super().__init__()
								lowerCAmelCase_      			=  nn.Linear(UpperCamelCase__,	UpperCamelCase__   )







		def        SCREAMING_SNAKE_CASE__			(     self,	UpperCamelCase__   ):







								"""simple docstring"""



								lowerCAmelCase_      			=  self.proj(UpperCamelCase__   )
								return x * torch.sigmoid(1.702 * x   )







class  A		(  nn.Module     ):



		def __init__(     self,	UpperCamelCase__,	UpperCamelCase__   ):







								"""simple docstring"""



								super().__init__()
								lowerCAmelCase_      			=  nn.Embedding(UpperCamelCase__,	UpperCamelCase__   )
								lowerCAmelCase_      			=  nn.SiLU()
								lowerCAmelCase_      			=  nn.Linear(UpperCamelCase__,	embedding_dim * 2   )
								lowerCAmelCase_      			=  nn.LayerNorm(UpperCamelCase__,	elementwise_affine=UpperCamelCase__   )







		def        SCREAMING_SNAKE_CASE__			(     self,	UpperCamelCase__,	UpperCamelCase__   ):







								"""simple docstring"""



								lowerCAmelCase_      			=  self.linear(self.silu(self.emb(UpperCamelCase__   )   )   )
								lowerCAmelCase_			,					lowerCAmelCase_      			=  torch.chunk(UpperCamelCase__,	2   )
								lowerCAmelCase_      			=  self.norm(UpperCamelCase__   ) * (1 + scale) + shift
								return x







class  A		(  nn.Module     ):



		def __init__(     self,	UpperCamelCase__,	UpperCamelCase__   ):







								"""simple docstring"""



								super().__init__()

								lowerCAmelCase_      			=  CombinedTimestepLabelEmbeddings(UpperCamelCase__,	UpperCamelCase__   )

								lowerCAmelCase_      			=  nn.SiLU()
								lowerCAmelCase_      			=  nn.Linear(UpperCamelCase__,	6 * embedding_dim,	bias=UpperCamelCase__   )
								lowerCAmelCase_      			=  nn.LayerNorm(UpperCamelCase__,	elementwise_affine=UpperCamelCase__,	eps=1E-6   )







		def        SCREAMING_SNAKE_CASE__			(     self,	UpperCamelCase__,	UpperCamelCase__,	UpperCamelCase__,	UpperCamelCase__=None   ):







								"""simple docstring"""



								lowerCAmelCase_      			=  self.linear(self.silu(self.emb(UpperCamelCase__,	UpperCamelCase__,	hidden_dtype=UpperCamelCase__   )   )   )
								lowerCAmelCase_			,					lowerCAmelCase_			,					lowerCAmelCase_			,					lowerCAmelCase_			,					lowerCAmelCase_			,					lowerCAmelCase_      			=  emb.chunk(6,	dim=1   )
								lowerCAmelCase_      			=  self.norm(UpperCamelCase__   ) * (1 + scale_msa[:, None]) + shift_msa[:, None]
								return x, gate_msa, shift_mlp, scale_mlp, gate_mlp







class  A		(  nn.Module     ):



		def __init__(     self,	UpperCamelCase__,	UpperCamelCase__,	UpperCamelCase__,	UpperCamelCase__ = None,	UpperCamelCase__ = 1E-5   ):







								"""simple docstring"""



								super().__init__()
								lowerCAmelCase_      			=  num_groups
								lowerCAmelCase_      			=  eps

								if act_fn is None:
														lowerCAmelCase_      			=  None
								else:
														lowerCAmelCase_      			=  get_activation(UpperCamelCase__   )

								lowerCAmelCase_      			=  nn.Linear(UpperCamelCase__,	out_dim * 2   )







		def        SCREAMING_SNAKE_CASE__			(     self,	UpperCamelCase__,	UpperCamelCase__   ):







								"""simple docstring"""



								if self.act:
														lowerCAmelCase_      			=  self.act(UpperCamelCase__   )
								lowerCAmelCase_      			=  self.linear(UpperCamelCase__   )
								lowerCAmelCase_      			=  emb[:, :, None, None]
								lowerCAmelCase_			,					lowerCAmelCase_      			=  emb.chunk(2,	dim=1   )

								lowerCAmelCase_      			=  F.group_norm(UpperCamelCase__,	self.num_groups,	eps=self.eps   )
								lowerCAmelCase_      			=  x * (1 + scale) + shift
								return x










from ...utils import (
    OptionalDependencyNotAvailable,
    is_torch_available,
    is_transformers_available,
    is_transformers_version,
)


try:
						if not (is_transformers_available() and is_torch_available() and is_transformers_version('''>=''', '''4.25.0''')):
												raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
						from ...utils.dummy_torch_and_transformers_objects import UnCLIPImageVariationPipeline, UnCLIPPipeline
else:
						from .pipeline_unclip import UnCLIPPipeline
						from .pipeline_unclip_image_variation import UnCLIPImageVariationPipeline
						from .text_proj import UnCLIPTextProjModel




