'''simple docstring'''




from typing import Any, Dict, Optional

import torch
import torch.nn.functional as F
from torch import nn

from ..utils import maybe_allow_in_graph
from .activations import get_activation
from .attention_processor import Attention
from .embeddings import CombinedTimestepLabelEmbeddings






@maybe_allow_in_graph
class 				a__(					nn.Module	):


			'''simple docstring'''






			def __init__(   self       ,    __lowerCAmelCase       ,    __lowerCAmelCase       ,    __lowerCAmelCase       ,    __lowerCAmelCase=0.0       ,    __lowerCAmelCase = None       ,    __lowerCAmelCase = "geglu"       ,    __lowerCAmelCase = None       ,    __lowerCAmelCase = False       ,    __lowerCAmelCase = False       ,    __lowerCAmelCase = False       ,    __lowerCAmelCase = False       ,    __lowerCAmelCase = True       ,    __lowerCAmelCase = "layer_norm"       ,    __lowerCAmelCase = False       ,    ):




					"""simple docstring"""


					super().__init__()
					lowerCAmelCase   					=  only_cross_attention

					lowerCAmelCase   					=  (num_embeds_ada_norm is not None) and norm_type == """ada_norm_zero"""
					lowerCAmelCase   					=  (num_embeds_ada_norm is not None) and norm_type == """ada_norm"""

					if norm_type in ("ada_norm", "ada_norm_zero") and num_embeds_ada_norm is None:
							raise ValueError(
							    f"`norm_type` is set to {norm_type}, but `num_embeds_ada_norm` is not defined. Please make sure to"
							    f" define `num_embeds_ada_norm` if setting `norm_type` to {norm_type}.")

					# Define 3 blocks. Each block has its own normalization layer.
					# 1. Self-Attn
					if self.use_ada_layer_norm:
							lowerCAmelCase   					=  AdaLayerNorm(__lowerCAmelCase       ,    __lowerCAmelCase)
					elif self.use_ada_layer_norm_zero:
							lowerCAmelCase   					=  AdaLayerNormZero(__lowerCAmelCase       ,    __lowerCAmelCase)
					else:
							lowerCAmelCase   					=  nn.LayerNorm(__lowerCAmelCase       ,    elementwise_affine=__lowerCAmelCase)
					lowerCAmelCase   					=  Attention(
					    query_dim=__lowerCAmelCase       ,    heads=__lowerCAmelCase       ,    dim_head=__lowerCAmelCase       ,    dropout=__lowerCAmelCase       ,    bias=__lowerCAmelCase       ,    cross_attention_dim=cross_attention_dim if only_cross_attention else None       ,    upcast_attention=__lowerCAmelCase       ,    )

					# 2. Cross-Attn
					if cross_attention_dim is not None or double_self_attention:
							# We currently only use AdaLayerNormZero for self attention where there will only be one attention block.
							# I.e. the number of returned modulation chunks from AdaLayerZero would not make sense if returned during
							# the second cross attention block.
							lowerCAmelCase   					=  (
							    AdaLayerNorm(__lowerCAmelCase       ,    __lowerCAmelCase)
							    if self.use_ada_layer_norm
							    else nn.LayerNorm(__lowerCAmelCase       ,    elementwise_affine=__lowerCAmelCase)
							)
							lowerCAmelCase   					=  Attention(
							    query_dim=__lowerCAmelCase       ,    cross_attention_dim=cross_attention_dim if not double_self_attention else None       ,    heads=__lowerCAmelCase       ,    dim_head=__lowerCAmelCase       ,    dropout=__lowerCAmelCase       ,    bias=__lowerCAmelCase       ,    upcast_attention=__lowerCAmelCase       ,    )  # is self-attn if encoder_hidden_states is none
					else:
							lowerCAmelCase   					=  None
							lowerCAmelCase   					=  None

					# 3. Feed-forward
					lowerCAmelCase   					=  nn.LayerNorm(__lowerCAmelCase       ,    elementwise_affine=__lowerCAmelCase)
					lowerCAmelCase   					=  FeedForward(__lowerCAmelCase       ,    dropout=__lowerCAmelCase       ,    activation_fn=__lowerCAmelCase       ,    final_dropout=__lowerCAmelCase)

					# let chunk size default to None
					lowerCAmelCase   					=  None
					lowerCAmelCase   					=  0



			def 							a_       (   self       ,    __lowerCAmelCase       ,    __lowerCAmelCase):




					"""simple docstring"""


					lowerCAmelCase   					=  chunk_size
					lowerCAmelCase   					=  dim







			def 							a_       (   self       ,    __lowerCAmelCase       ,    __lowerCAmelCase = None       ,    __lowerCAmelCase = None       ,    __lowerCAmelCase = None       ,    __lowerCAmelCase = None       ,    __lowerCAmelCase = None       ,    __lowerCAmelCase = None       ,    ):




					"""simple docstring"""


					if self.use_ada_layer_norm:
							lowerCAmelCase   					=  self.norma(__lowerCAmelCase       ,    __lowerCAmelCase)
					elif self.use_ada_layer_norm_zero:
							lowerCAmelCase					,     lowerCAmelCase					,     lowerCAmelCase					,     lowerCAmelCase					,     lowerCAmelCase   					=  self.norma(
							    __lowerCAmelCase       ,    __lowerCAmelCase       ,    __lowerCAmelCase       ,    hidden_dtype=hidden_states.dtype)
					else:
							lowerCAmelCase   					=  self.norma(__lowerCAmelCase)

					lowerCAmelCase   					=  cross_attention_kwargs if cross_attention_kwargs is not None else {}

					lowerCAmelCase   					=  self.attna(
					    __lowerCAmelCase       ,    encoder_hidden_states=encoder_hidden_states if self.only_cross_attention else None       ,    attention_mask=__lowerCAmelCase       ,    **__lowerCAmelCase       ,    )
					if self.use_ada_layer_norm_zero:
							lowerCAmelCase   					=  gate_msa.unsqueeze(1) * attn_output
					lowerCAmelCase   					=  attn_output + hidden_states

					# 2. Cross-Attention
					if self.attna is not None:
							lowerCAmelCase   					=  (
							    self.norma(__lowerCAmelCase       ,    __lowerCAmelCase) if self.use_ada_layer_norm else self.norma(__lowerCAmelCase)
							)

							lowerCAmelCase   					=  self.attna(
							    __lowerCAmelCase       ,    encoder_hidden_states=__lowerCAmelCase       ,    attention_mask=__lowerCAmelCase       ,    **__lowerCAmelCase       ,    )
							lowerCAmelCase   					=  attn_output + hidden_states

					# 3. Feed-forward
					lowerCAmelCase   					=  self.norma(__lowerCAmelCase)

					if self.use_ada_layer_norm_zero:
							lowerCAmelCase   					=  norm_hidden_states * (1 + scale_mlp[:, None]) + shift_mlp[:, None]

					if self._chunk_size is not None:
							# "feed_forward_chunk_size" can be used to save memory
							if norm_hidden_states.shape[self._chunk_dim] % self._chunk_size != 0:
									raise ValueError(
									    f"`hidden_states` dimension to be chunked: {norm_hidden_states.shape[self._chunk_dim]} has to be divisible by chunk size: {self._chunk_size}. Make sure to set an appropriate `chunk_size` when calling `unet.enable_forward_chunking`.")

							lowerCAmelCase   					=  norm_hidden_states.shape[self._chunk_dim] // self._chunk_size
							lowerCAmelCase   					=  torch.cat(
							    [self.ff(__lowerCAmelCase) for hid_slice in norm_hidden_states.chunk(__lowerCAmelCase       ,    dim=self._chunk_dim)]       ,    dim=self._chunk_dim       ,    )
					else:
							lowerCAmelCase   					=  self.ff(__lowerCAmelCase)

					if self.use_ada_layer_norm_zero:
							lowerCAmelCase   					=  gate_mlp.unsqueeze(1) * ff_output

					lowerCAmelCase   					=  ff_output + hidden_states

					return hidden_states







class 				a__(					nn.Module	):


			'''simple docstring'''






			def __init__(   self       ,    __lowerCAmelCase       ,    __lowerCAmelCase = None       ,    __lowerCAmelCase = 4       ,    __lowerCAmelCase = 0.0       ,    __lowerCAmelCase = "geglu"       ,    __lowerCAmelCase = False       ,    ):




					"""simple docstring"""


					super().__init__()
					lowerCAmelCase   					=  int(dim * mult)
					lowerCAmelCase   					=  dim_out if dim_out is not None else dim

					if activation_fn == "gelu":
							lowerCAmelCase   					=  GELU(__lowerCAmelCase       ,    __lowerCAmelCase)
					if activation_fn == "gelu-approximate":
							lowerCAmelCase   					=  GELU(__lowerCAmelCase       ,    __lowerCAmelCase       ,    approximate="""tanh""")
					elif activation_fn == "geglu":
							lowerCAmelCase   					=  GEGLU(__lowerCAmelCase       ,    __lowerCAmelCase)
					elif activation_fn == "geglu-approximate":
							lowerCAmelCase   					=  ApproximateGELU(__lowerCAmelCase       ,    __lowerCAmelCase)

					lowerCAmelCase   					=  nn.ModuleList([])
					# project in
					self.net.append(__lowerCAmelCase)
					# project dropout
					self.net.append(nn.Dropout(__lowerCAmelCase))
					# project out
					self.net.append(nn.Linear(__lowerCAmelCase       ,    __lowerCAmelCase))
					# FF as used in Vision Transformer, MLP-Mixer, etc. have a final dropout
					if final_dropout:
							self.net.append(nn.Dropout(__lowerCAmelCase))







			def 							a_       (   self       ,    __lowerCAmelCase):




					"""simple docstring"""


					for module in self.net:
							lowerCAmelCase   					=  module(__lowerCAmelCase)
					return hidden_states







class 				a__(					nn.Module	):


			'''simple docstring'''






			def __init__(   self       ,    __lowerCAmelCase       ,    __lowerCAmelCase       ,    __lowerCAmelCase = "none"):




					"""simple docstring"""


					super().__init__()
					lowerCAmelCase   					=  nn.Linear(__lowerCAmelCase       ,    __lowerCAmelCase)
					lowerCAmelCase   					=  approximate



			def 							a_       (   self       ,    __lowerCAmelCase):




					"""simple docstring"""


					if gate.device.type != "mps":
							return F.gelu(__lowerCAmelCase       ,    approximate=self.approximate)
					# mps: gelu is not implemented for float16
					return F.gelu(gate.to(dtype=torch.floataa)       ,    approximate=self.approximate).to(dtype=gate.dtype)







			def 							a_       (   self       ,    __lowerCAmelCase):




					"""simple docstring"""


					lowerCAmelCase   					=  self.proj(__lowerCAmelCase)
					lowerCAmelCase   					=  self.gelu(__lowerCAmelCase)
					return hidden_states







class 				a__(					nn.Module	):


			'''simple docstring'''






			def __init__(   self       ,    __lowerCAmelCase       ,    __lowerCAmelCase):




					"""simple docstring"""


					super().__init__()
					lowerCAmelCase   					=  nn.Linear(__lowerCAmelCase       ,    dim_out * 2)



			def 							a_       (   self       ,    __lowerCAmelCase):




					"""simple docstring"""


					if gate.device.type != "mps":
							return F.gelu(__lowerCAmelCase)
					# mps: gelu is not implemented for float16
					return F.gelu(gate.to(dtype=torch.floataa)).to(dtype=gate.dtype)







			def 							a_       (   self       ,    __lowerCAmelCase):




					"""simple docstring"""


					lowerCAmelCase					,     lowerCAmelCase   					=  self.proj(__lowerCAmelCase).chunk(2       ,    dim=-1)
					return hidden_states * self.gelu(__lowerCAmelCase)







class 				a__(					nn.Module	):


			'''simple docstring'''






			def __init__(   self       ,    __lowerCAmelCase       ,    __lowerCAmelCase):




					"""simple docstring"""


					super().__init__()
					lowerCAmelCase   					=  nn.Linear(__lowerCAmelCase       ,    __lowerCAmelCase)







			def 							a_       (   self       ,    __lowerCAmelCase):




					"""simple docstring"""


					lowerCAmelCase   					=  self.proj(__lowerCAmelCase)
					return x * torch.sigmoid(1.702 * x)







class 				a__(					nn.Module	):


			'''simple docstring'''






			def __init__(   self       ,    __lowerCAmelCase       ,    __lowerCAmelCase):




					"""simple docstring"""


					super().__init__()
					lowerCAmelCase   					=  nn.Embedding(__lowerCAmelCase       ,    __lowerCAmelCase)
					lowerCAmelCase   					=  nn.SiLU()
					lowerCAmelCase   					=  nn.Linear(__lowerCAmelCase       ,    embedding_dim * 2)
					lowerCAmelCase   					=  nn.LayerNorm(__lowerCAmelCase       ,    elementwise_affine=__lowerCAmelCase)







			def 							a_       (   self       ,    __lowerCAmelCase       ,    __lowerCAmelCase):




					"""simple docstring"""


					lowerCAmelCase   					=  self.linear(self.silu(self.emb(__lowerCAmelCase)))
					lowerCAmelCase					,     lowerCAmelCase   					=  torch.chunk(__lowerCAmelCase       ,    2)
					lowerCAmelCase   					=  self.norm(__lowerCAmelCase) * (1 + scale) + shift
					return x







class 				a__(					nn.Module	):


			'''simple docstring'''






			def __init__(   self       ,    __lowerCAmelCase       ,    __lowerCAmelCase):




					"""simple docstring"""


					super().__init__()

					lowerCAmelCase   					=  CombinedTimestepLabelEmbeddings(__lowerCAmelCase       ,    __lowerCAmelCase)

					lowerCAmelCase   					=  nn.SiLU()
					lowerCAmelCase   					=  nn.Linear(__lowerCAmelCase       ,    6 * embedding_dim       ,    bias=__lowerCAmelCase)
					lowerCAmelCase   					=  nn.LayerNorm(__lowerCAmelCase       ,    elementwise_affine=__lowerCAmelCase       ,    eps=1E-6)







			def 							a_       (   self       ,    __lowerCAmelCase       ,    __lowerCAmelCase       ,    __lowerCAmelCase       ,    __lowerCAmelCase=None):




					"""simple docstring"""


					lowerCAmelCase   					=  self.linear(self.silu(self.emb(__lowerCAmelCase       ,    __lowerCAmelCase       ,    hidden_dtype=__lowerCAmelCase)))
					lowerCAmelCase					,     lowerCAmelCase					,     lowerCAmelCase					,     lowerCAmelCase					,     lowerCAmelCase					,     lowerCAmelCase   					=  emb.chunk(6       ,    dim=1)
					lowerCAmelCase   					=  self.norm(__lowerCAmelCase) * (1 + scale_msa[:, None]) + shift_msa[:, None]
					return x, gate_msa, shift_mlp, scale_mlp, gate_mlp







class 				a__(					nn.Module	):


			'''simple docstring'''






			def __init__(   self       ,    __lowerCAmelCase       ,    __lowerCAmelCase       ,    __lowerCAmelCase       ,    __lowerCAmelCase = None       ,    __lowerCAmelCase = 1E-5):




					"""simple docstring"""


					super().__init__()
					lowerCAmelCase   					=  num_groups
					lowerCAmelCase   					=  eps

					if act_fn is None:
							lowerCAmelCase   					=  None
					else:
							lowerCAmelCase   					=  get_activation(__lowerCAmelCase)

					lowerCAmelCase   					=  nn.Linear(__lowerCAmelCase       ,    out_dim * 2)







			def 							a_       (   self       ,    __lowerCAmelCase       ,    __lowerCAmelCase):




					"""simple docstring"""


					if self.act:
							lowerCAmelCase   					=  self.act(__lowerCAmelCase)
					lowerCAmelCase   					=  self.linear(__lowerCAmelCase)
					lowerCAmelCase   					=  emb[:, :, None, None]
					lowerCAmelCase					,     lowerCAmelCase   					=  emb.chunk(2       ,    dim=1)

					lowerCAmelCase   					=  F.group_norm(__lowerCAmelCase       ,    self.num_groups       ,    eps=self.eps)
					lowerCAmelCase   					=  x * (1 + scale) + shift
					return x





'''simple docstring'''




from typing import TYPE_CHECKING

from ...utils import OptionalDependencyNotAvailable, _LazyModule, is_tokenizers_available, is_torch_available


__lowercase          =  {
    '''configuration_nezha''': ['''NEZHA_PRETRAINED_CONFIG_ARCHIVE_MAP''', '''NezhaConfig'''],
}

try:
						if not is_torch_available():
												raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
						pass
else:
						__lowercase          =  [
						    '''NEZHA_PRETRAINED_MODEL_ARCHIVE_LIST''',
						    '''NezhaForNextSentencePrediction''',
						    '''NezhaForMaskedLM''',
						    '''NezhaForPreTraining''',
						    '''NezhaForMultipleChoice''',
						    '''NezhaForQuestionAnswering''',
						    '''NezhaForSequenceClassification''',
						    '''NezhaForTokenClassification''',
						    '''NezhaModel''',
						    '''NezhaPreTrainedModel''',
						]


if TYPE_CHECKING:
						from .configuration_nezha import NEZHA_PRETRAINED_CONFIG_ARCHIVE_MAP, NezhaConfig

						try:
												if not is_torch_available():
																		raise OptionalDependencyNotAvailable()
						except OptionalDependencyNotAvailable:
												pass
						else:
												from .modeling_nezha import (
												    NEZHA_PRETRAINED_MODEL_ARCHIVE_LIST,
												    NezhaForMaskedLM,
												    NezhaForMultipleChoice,
												    NezhaForNextSentencePrediction,
												    NezhaForPreTraining,
												    NezhaForQuestionAnswering,
												    NezhaForSequenceClassification,
												    NezhaForTokenClassification,
												    NezhaModel,
												    NezhaPreTrainedModel,
												)


else:
						import sys

						__lowercase          =  _LazyModule(__name__, globals()['''__file__'''], _import_structure, module_spec=__spec__)




