


import math

import torch
from torch import nn

from ..configuration_utils import ConfigMixin, register_to_config
from .attention_processor import Attention
from .embeddings import get_timestep_embedding
from .modeling_utils import ModelMixin
class 	UpperCAmelCase_					(					UpperCamelCase		, UpperCamelCase    ):

   '''simple docstring'''




   @register_to_config
   def __init__(		self						,				__A = 128						,				__A = 256						,				__A = 2000.0						,				__A = 768						,				__A = 12						,				__A = 12						,				__A = 64						,				__A = 2048						,				__A = 0.1						,				):






     """simple docstring"""





     super().__init__()

     lowerCamelCase :					List[str]    =				nn.Sequential(
         nn.Linear(__A						,				d_model * 4						,				bias=__A )						,				nn.SiLU()						,				nn.Linear(d_model * 4						,				d_model * 4						,				bias=__A )						,				nn.SiLU()						,				)

     lowerCamelCase :					Optional[Any]    =				nn.Embedding(__A						,				__A )
     lowerCamelCase :					Any    =				False

     lowerCamelCase :					Union[str, Any]    =				nn.Linear(__A						,				__A						,				bias=__A )

     lowerCamelCase :					int    =				nn.Dropout(p=__A )

     lowerCamelCase :					int    =				nn.ModuleList()
     for lyr_num in range(__A ):
       # FiLM conditional T5 decoder
       lowerCamelCase :					List[Any]    =				DecoderLayer(d_model=__A						,				d_kv=__A						,				num_heads=__A						,				d_ff=__A						,				dropout_rate=__A )
       self.decoders.append(__A )

     lowerCamelCase :					Dict    =				TaLayerNorm(__A )

     lowerCamelCase :					Any    =				nn.Dropout(p=__A )
     lowerCamelCase :					Optional[int]    =				nn.Linear(__A						,				__A						,				bias=__A )






   def _snake_case    (		self						,				__A						,				__A ):






     """simple docstring"""





     lowerCamelCase :					int    =				torch.mul(query_input.unsqueeze(-1 )						,				key_input.unsqueeze(-2 ) )
     return mask.unsqueeze(-3 )






   def _snake_case    (		self						,				__A						,				__A						,				__A ):






     """simple docstring"""





     lowerCamelCase    ,	lowerCamelCase    ,	lowerCamelCase :					List[Any]    =				decoder_input_tokens.shape
     assert decoder_noise_time.shape == (batch,)

     # decoder_noise_time is in [0, 1), so rescale to expected timing range.
     lowerCamelCase :					List[Any]    =				get_timestep_embedding(
         decoder_noise_time * self.config.max_decoder_noise_time						,				embedding_dim=self.config.d_model						,				max_period=self.config.max_decoder_noise_time						,				).to(dtype=self.dtype )

     lowerCamelCase :					Union[str, Any]    =				self.conditioning_emb(__A ).unsqueeze(1 )

     assert conditioning_emb.shape == (batch, 1, self.config.d_model * 4)

     lowerCamelCase :					int    =				decoder_input_tokens.shape[1]

     # If we want to use relative positions for audio context, we can just offset
     # this sequence by the length of encodings_and_masks.
     lowerCamelCase :					int    =				torch.broadcast_to(
         torch.arange(__A						,				device=decoder_input_tokens.device )						,				(batch, seq_length)						,				)

     lowerCamelCase :					str    =				self.position_encoding(__A )

     lowerCamelCase :					Tuple    =				self.continuous_inputs_projection(__A )
     inputs += position_encodings
     lowerCamelCase :					List[str]    =				self.dropout(__A )

     # decoder: No padding present.
     lowerCamelCase :					Optional[int]    =				torch.ones(
         decoder_input_tokens.shape[:2]						,				device=decoder_input_tokens.device						,				dtype=inputs.dtype )

     # Translate encoding masks to encoder-decoder masks.
     lowerCamelCase :					List[Any]    =				[(x, self.encoder_decoder_mask(__A						,				__A )) for x, y in encodings_and_masks]

     # cross attend style: concat encodings
     lowerCamelCase :					Optional[Any]    =				torch.cat([x[0] for x in encodings_and_encdec_masks]						,				dim=1 )
     lowerCamelCase :					Union[str, Any]    =				torch.cat([x[1] for x in encodings_and_encdec_masks]						,				dim=-1 )

     for lyr in self.decoders:
       lowerCamelCase :					List[str]    =				lyr(
           __A						,				conditioning_emb=__A						,				encoder_hidden_states=__A						,				encoder_attention_mask=__A						,				)[0]

     lowerCamelCase :					Optional[int]    =				self.decoder_norm(__A )
     lowerCamelCase :					Any    =				self.post_dropout(__A )

     lowerCamelCase :					Optional[Any]    =				self.spec_out(__A )
     return spec_out



class 	UpperCAmelCase_					(					nn.Module    ):

   '''simple docstring'''




   def __init__(		self						,				__A						,				__A						,				__A						,				__A						,				__A						,				__A=1e-6 ):






     """simple docstring"""





     super().__init__()
     lowerCamelCase :					Optional[int]    =				nn.ModuleList()

     # cond self attention: layer 0
     self.layer.append(
         TaLayerSelfAttentionCond(d_model=__A						,				d_kv=__A						,				num_heads=__A						,				dropout_rate=__A ) )

     # cross attention: layer 1
     self.layer.append(
         TaLayerCrossAttention(
             d_model=__A						,				d_kv=__A						,				num_heads=__A						,				dropout_rate=__A						,				layer_norm_epsilon=__A						,				) )

     # Film Cond MLP + dropout: last layer
     self.layer.append(
         TaLayerFFCond(d_model=__A						,				d_ff=__A						,				dropout_rate=__A						,				layer_norm_epsilon=__A ) )






   def _snake_case    (		self						,				__A						,				__A=None						,				__A=None						,				__A=None						,				__A=None						,				__A=None						,				):






     """simple docstring"""





     lowerCamelCase :					Union[str, Any]    =				self.layer[0](
         __A						,				conditioning_emb=__A						,				attention_mask=__A						,				)

     if encoder_hidden_states is not None:
       lowerCamelCase :					List[Any]    =				torch.where(encoder_attention_mask > 0						,				0						,				-1e10 ).to(
           encoder_hidden_states.dtype )

       lowerCamelCase :					Optional[Any]    =				self.layer[1](
           __A						,				key_value_states=__A						,				attention_mask=__A						,				)

     # Apply Film Conditional Feed Forward layer
     lowerCamelCase :					List[str]    =				self.layer[-1](__A						,				__A )

     return (hidden_states,)



class 	UpperCAmelCase_					(					nn.Module    ):

   '''simple docstring'''




   def __init__(		self						,				__A						,				__A						,				__A						,				__A ):






     """simple docstring"""





     super().__init__()
     lowerCamelCase :					Optional[Any]    =				TaLayerNorm(__A )
     lowerCamelCase :					Dict    =				TaFiLMLayer(in_features=d_model * 4						,				out_features=__A )
     lowerCamelCase :					str    =				Attention(query_dim=__A						,				heads=__A						,				dim_head=__A						,				out_bias=__A						,				scale_qk=__A )
     lowerCamelCase :					Optional[Any]    =				nn.Dropout(__A )






   def _snake_case    (		self						,				__A						,				__A=None						,				__A=None						,				):






     """simple docstring"""





     lowerCamelCase :					Tuple    =				self.layer_norm(__A )

     if conditioning_emb is not None:
       lowerCamelCase :					Dict    =				self.FiLMLayer(__A						,				__A )

     # Self-attention block
     lowerCamelCase :					Union[str, Any]    =				self.attention(__A )

     lowerCamelCase :					Dict    =				hidden_states + self.dropout(__A )

     return hidden_states



class 	UpperCAmelCase_					(					nn.Module    ):

   '''simple docstring'''




   def __init__(		self						,				__A						,				__A						,				__A						,				__A						,				__A ):






     """simple docstring"""





     super().__init__()
     lowerCamelCase :					Union[str, Any]    =				Attention(query_dim=__A						,				heads=__A						,				dim_head=__A						,				out_bias=__A						,				scale_qk=__A )
     lowerCamelCase :					Tuple    =				TaLayerNorm(__A						,				eps=__A )
     lowerCamelCase :					Union[str, Any]    =				nn.Dropout(__A )






   def _snake_case    (		self						,				__A						,				__A=None						,				__A=None						,				):






     """simple docstring"""





     lowerCamelCase :					Optional[Any]    =				self.layer_norm(__A )
     lowerCamelCase :					Dict    =				self.attention(
         __A						,				encoder_hidden_states=__A						,				attention_mask=attention_mask.squeeze(1 )						,				)
     lowerCamelCase :					List[Any]    =				hidden_states + self.dropout(__A )
     return layer_output



class 	UpperCAmelCase_					(					nn.Module    ):

   '''simple docstring'''




   def __init__(		self						,				__A						,				__A						,				__A						,				__A ):






     """simple docstring"""





     super().__init__()
     lowerCamelCase :					Any    =				TaDenseGatedActDense(d_model=__A						,				d_ff=__A						,				dropout_rate=__A )
     lowerCamelCase :					Optional[int]    =				TaFiLMLayer(in_features=d_model * 4						,				out_features=__A )
     lowerCamelCase :					str    =				TaLayerNorm(__A						,				eps=__A )
     lowerCamelCase :					Union[str, Any]    =				nn.Dropout(__A )






   def _snake_case    (		self						,				__A						,				__A=None ):






     """simple docstring"""





     lowerCamelCase :					Any    =				self.layer_norm(__A )
     if conditioning_emb is not None:
       lowerCamelCase :					str    =				self.film(__A						,				__A )

     lowerCamelCase :					Union[str, Any]    =				self.DenseReluDense(__A )
     lowerCamelCase :					Dict    =				hidden_states + self.dropout(__A )
     return hidden_states



class 	UpperCAmelCase_					(					nn.Module    ):

   '''simple docstring'''




   def __init__(		self						,				__A						,				__A						,				__A ):






     """simple docstring"""





     super().__init__()
     lowerCamelCase :					Optional[Any]    =				nn.Linear(__A						,				__A						,				bias=__A )
     lowerCamelCase :					Optional[Any]    =				nn.Linear(__A						,				__A						,				bias=__A )
     lowerCamelCase :					Optional[int]    =				nn.Linear(__A						,				__A						,				bias=__A )
     lowerCamelCase :					Optional[Any]    =				nn.Dropout(__A )
     lowerCamelCase :					int    =				NewGELUActivation()






   def _snake_case    (		self						,				__A ):






     """simple docstring"""





     lowerCamelCase :					List[str]    =				self.act(self.wi_a(__A ) )
     lowerCamelCase :					int    =				self.wi_a(__A )
     lowerCamelCase :					List[Any]    =				hidden_gelu * hidden_linear
     lowerCamelCase :					Dict    =				self.dropout(__A )

     lowerCamelCase :					Tuple    =				self.wo(__A )
     return hidden_states



class 	UpperCAmelCase_					(					nn.Module    ):

   '''simple docstring'''




   def __init__(		self						,				__A						,				__A=1e-6 ):






     """simple docstring"""





     super().__init__()
     lowerCamelCase :					Optional[int]    =				nn.Parameter(torch.ones(__A ) )
     lowerCamelCase :					Optional[Any]    =				eps






   def _snake_case    (		self						,				__A ):






     """simple docstring"""





     lowerCamelCase :					Tuple    =				hidden_states.to(torch.floataa ).pow(2 ).mean(-1						,				keepdim=__A )
     lowerCamelCase :					int    =				hidden_states * torch.rsqrt(variance + self.variance_epsilon )

     # convert into half-precision if necessary
     if self.weight.dtype in [torch.floataa, torch.bfloataa]:
       lowerCamelCase :					Any    =				hidden_states.to(self.weight.dtype )

     return self.weight * hidden_states



class 	UpperCAmelCase_					(					nn.Module    ):

   '''simple docstring'''




   def _snake_case    (		self						,				__A ):






     """simple docstring"""





     return 0.5 * input * (1.0 + torch.tanh(math.sqrt(2.0 / math.pi ) * (input + 0.044715 * torch.pow(__A						,				3.0 )) ))



class 	UpperCAmelCase_					(					nn.Module    ):

   '''simple docstring'''




   def __init__(		self						,				__A						,				__A ):






     """simple docstring"""





     super().__init__()
     lowerCamelCase :					List[Any]    =				nn.Linear(__A						,				out_features * 2						,				bias=__A )






   def _snake_case    (		self						,				__A						,				__A ):






     """simple docstring"""





     lowerCamelCase :					Optional[Any]    =				self.scale_bias(__A )
     lowerCamelCase    ,	lowerCamelCase :					Optional[Any]    =				torch.chunk(__A						,				2						,				-1 )
     lowerCamelCase :					Dict    =				x * (1 + scale) + shift
     return x







import argparse

import intel_extension_for_pytorch as ipex
import torch

from diffusers import DPMSolverMultistepScheduler, StableDiffusionPipeline


_snake_case             =     argparse.ArgumentParser('''Stable Diffusion script with intel optimization''', add_help=False)
parser.add_argument('''--dpm''', action='''store_true''', help='''Enable DPMSolver or not''')
parser.add_argument('''--steps''', default=None, type=int, help='''Num inference steps''')
_snake_case             =     parser.parse_args()


_snake_case             =     '''cpu'''
_snake_case             =     '''a lovely <dicoo> in red dress and hat, in the snowly and brightly night, with many brighly buildings'''

_snake_case             =     '''path-to-your-trained-model'''
_snake_case             =     StableDiffusionPipeline.from_pretrained(model_id)
if args.dpm:
   _snake_case             =     DPMSolverMultistepScheduler.from_config(pipe.scheduler.config)
_snake_case             =     pipe.to(device)

# to channels last
_snake_case             =     pipe.unet.to(memory_format=torch.channels_last)
_snake_case             =     pipe.vae.to(memory_format=torch.channels_last)
_snake_case             =     pipe.text_encoder.to(memory_format=torch.channels_last)
if pipe.requires_safety_checker:
   _snake_case             =     pipe.safety_checker.to(memory_format=torch.channels_last)

# optimize with ipex
_snake_case             =     torch.randn(2, 4, 64, 64)
_snake_case             =     torch.rand(1) * 9_99
_snake_case             =     torch.randn(2, 77, 7_68)
_snake_case             =     (sample, timestep, encoder_hidden_status)
try:
   _snake_case             =     ipex.optimize(pipe.unet.eval(), dtype=torch.bfloataa, inplace=True, sample_input=input_example)
except Exception:
   _snake_case             =     ipex.optimize(pipe.unet.eval(), dtype=torch.bfloataa, inplace=True)
_snake_case             =     ipex.optimize(pipe.vae.eval(), dtype=torch.bfloataa, inplace=True)
_snake_case             =     ipex.optimize(pipe.text_encoder.eval(), dtype=torch.bfloataa, inplace=True)
if pipe.requires_safety_checker:
   _snake_case             =     ipex.optimize(pipe.safety_checker.eval(), dtype=torch.bfloataa, inplace=True)

# compute
_snake_case             =     6_66
_snake_case             =     torch.Generator(device).manual_seed(seed)
_snake_case             =     {'''generator''': generator}
if args.steps is not None:
   _snake_case             =     args.steps

with torch.cpu.amp.autocast(enabled=True, dtype=torch.bfloataa):
   _snake_case             =     pipe(prompt, **generate_kwargs).images[0]

# save image
image.save('''generated.png''')



