



'''simple docstring'''

import random
import unittest

import numpy as np

import transformers
from transformers import is_flax_available, is_torch_available
from transformers.testing_utils import is_pt_flax_cross_test, require_flax


if is_flax_available():
	import os

	import jax.numpy as jnp
	from jax import jit

	from transformers import AutoTokenizer, FlaxAutoModelForCausalLM
	from transformers.modeling_flax_pytorch_utils import load_flax_weights_in_pytorch_model

	snake_case_      =  """0.12"""  # assumed parallelism: 8


if is_torch_available():
	import torch






def    __lowercase	(_SCREAMING_SNAKE_CASE   :Dict     ,						_SCREAMING_SNAKE_CASE   :Union[str, Any]     ,						_SCREAMING_SNAKE_CASE   :str=None							):
				if rng is None:
								SCREAMING_SNAKE_CASE	:    List[Any]          =			random.Random()

				SCREAMING_SNAKE_CASE	:    List[Any]          =			1
				for dim in shape:
								total_dims *= dim

				SCREAMING_SNAKE_CASE	:    Any          =			[]
				for _ in range(_SCREAMING_SNAKE_CASE							):
								values.append(rng.randint(0     ,						vocab_size - 1							)							)

				SCREAMING_SNAKE_CASE	:    Dict          =			np.array(_SCREAMING_SNAKE_CASE     ,						dtype=jnp.intaa							).reshape(_SCREAMING_SNAKE_CASE							)

				return output






def    __lowercase	(_SCREAMING_SNAKE_CASE   :Optional[int]     ,						_SCREAMING_SNAKE_CASE   :Any=None							):
				SCREAMING_SNAKE_CASE	:    List[Any]          =			ids_tensor(_SCREAMING_SNAKE_CASE     ,						vocab_size=2     ,						rng=_SCREAMING_SNAKE_CASE							)
				# make sure that at least one token is attended to for each batch
				SCREAMING_SNAKE_CASE	:    Dict          =			1
				return attn_mask


@require_flax
class 						a__  :
	__magic_name__		:     Optional[int]						     =  None
	__magic_name__		:     Any						     =  ()


	def 					lowercase__	(self			:			Optional[int]   )  ->      Any:

					"""simple docstring"""

					SCREAMING_SNAKE_CASE	:    List[Any]          =			self.model_tester.prepare_config_and_inputs_for_common()

					# cut to half length & take max batch_size 3
					SCREAMING_SNAKE_CASE	:    Dict          =			2
					SCREAMING_SNAKE_CASE	:    Optional[Any]          =			inputs["input_ids"].shape[-1] // 2
					SCREAMING_SNAKE_CASE	:    Union[str, Any]          =			inputs["input_ids"][:max_batch_size, :sequence_length]

					SCREAMING_SNAKE_CASE	:    Optional[int]          =			jnp.ones_like(lowerCamelCase__   )
					SCREAMING_SNAKE_CASE	:    List[Any]          =			attention_mask[:max_batch_size, :sequence_length]

					# generate max 5 tokens
					SCREAMING_SNAKE_CASE	:    Dict          =			input_ids.shape[-1] + 5
					if config.eos_token_id is not None and config.pad_token_id is None:
									# hack to allow generate for models such as GPT2 as is done in `generate()`
									SCREAMING_SNAKE_CASE	:    Union[str, Any]          =			config.eos_token_id
					return config, input_ids, attention_mask, max_length


	@is_pt_flax_cross_test
	def 					lowercase__	(self			:			Dict   )  ->      Optional[int]:

					"""simple docstring"""

					SCREAMING_SNAKE_CASE	:    Dict          =			self._get_input_ids_and_config()
					SCREAMING_SNAKE_CASE	:    Optional[int]          =			False
					SCREAMING_SNAKE_CASE	:    List[str]          =			max_length
					SCREAMING_SNAKE_CASE	:    str          =			0

					for model_class in self.all_generative_model_classes:
									SCREAMING_SNAKE_CASE	:    int          =			model_class(lowerCamelCase__   )

									SCREAMING_SNAKE_CASE	:    List[Any]          =			model_class.__name__[4:]  # Skip the "Flax" at the beginning
									SCREAMING_SNAKE_CASE	:    Optional[int]          =			getattr(lowerCamelCase__,       lowerCamelCase__   )
									SCREAMING_SNAKE_CASE	:    int          =			pt_model_class(lowerCamelCase__   ).eval()
									SCREAMING_SNAKE_CASE	:    Dict          =			load_flax_weights_in_pytorch_model(lowerCamelCase__,       flax_model.params   )

									SCREAMING_SNAKE_CASE	:    Union[str, Any]          =			flax_model.generate(lowerCamelCase__   ).sequences
									SCREAMING_SNAKE_CASE	:    int          =			pt_model.generate(torch.tensor(lowerCamelCase__,       dtype=torch.long   )   )

									if flax_generation_outputs.shape[-1] > pt_generation_outputs.shape[-1]:
													SCREAMING_SNAKE_CASE	:    Optional[int]          =			flax_generation_outputs[:, : pt_generation_outputs.shape[-1]]

									self.assertListEqual(pt_generation_outputs.numpy().tolist(),       flax_generation_outputs.tolist()   )


	def 					lowercase__	(self			:			Union[str, Any]   )  ->      Optional[Any]:

					"""simple docstring"""

					SCREAMING_SNAKE_CASE	:    Tuple          =			self._get_input_ids_and_config()
					SCREAMING_SNAKE_CASE	:    List[str]          =			False
					SCREAMING_SNAKE_CASE	:    str          =			max_length

					for model_class in self.all_generative_model_classes:
									SCREAMING_SNAKE_CASE	:    int          =			model_class(lowerCamelCase__   )

									SCREAMING_SNAKE_CASE	:    List[str]          =			model.generate(lowerCamelCase__   ).sequences
									self.assertEqual(generation_outputs.shape[-1],       lowerCamelCase__   )

									SCREAMING_SNAKE_CASE	:    List[Any]          =			jit(model.generate   )
									SCREAMING_SNAKE_CASE	:    Union[str, Any]          =			jit_generate(lowerCamelCase__   ).sequences

									self.assertListEqual(generation_outputs.tolist(),       jit_generation_outputs.tolist()   )


	def 					lowercase__	(self			:			Dict   )  ->      Tuple:

					"""simple docstring"""

					SCREAMING_SNAKE_CASE	:    Optional[Any]          =			self._get_input_ids_and_config()
					SCREAMING_SNAKE_CASE	:    Dict          =			True
					SCREAMING_SNAKE_CASE	:    Tuple          =			max_length

					for model_class in self.all_generative_model_classes:
									SCREAMING_SNAKE_CASE	:    Union[str, Any]          =			model_class(lowerCamelCase__   )

									SCREAMING_SNAKE_CASE	:    Dict          =			model.generate(lowerCamelCase__   ).sequences
									self.assertEqual(generation_outputs.shape[-1],       lowerCamelCase__   )

									SCREAMING_SNAKE_CASE	:    List[Any]          =			jit(model.generate   )
									SCREAMING_SNAKE_CASE	:    Union[str, Any]          =			jit_generate(lowerCamelCase__   ).sequences

									self.assertListEqual(generation_outputs.tolist(),       jit_generation_outputs.tolist()   )


	def 					lowercase__	(self			:			Union[str, Any]   )  ->      Dict:

					"""simple docstring"""

					SCREAMING_SNAKE_CASE	:    str          =			self._get_input_ids_and_config()
					SCREAMING_SNAKE_CASE	:    Dict          =			False
					SCREAMING_SNAKE_CASE	:    Optional[Any]          =			max_length
					SCREAMING_SNAKE_CASE	:    List[Any]          =			2

					for model_class in self.all_generative_model_classes:
									SCREAMING_SNAKE_CASE	:    Any          =			model_class(lowerCamelCase__   )

									SCREAMING_SNAKE_CASE	:    str          =			model.generate(lowerCamelCase__   ).sequences
									self.assertEqual(generation_outputs.shape[-1],       lowerCamelCase__   )

									SCREAMING_SNAKE_CASE	:    Optional[Any]          =			jit(model.generate   )
									SCREAMING_SNAKE_CASE	:    Union[str, Any]          =			jit_generate(lowerCamelCase__   ).sequences

									self.assertListEqual(generation_outputs.tolist(),       jit_generation_outputs.tolist()   )


	def 					lowercase__	(self			:			Any   )  ->      Dict:

					"""simple docstring"""

					SCREAMING_SNAKE_CASE	:    Union[str, Any]          =			self._get_input_ids_and_config()
					SCREAMING_SNAKE_CASE	:    Optional[Any]          =			False
					SCREAMING_SNAKE_CASE	:    Optional[Any]          =			max_length
					SCREAMING_SNAKE_CASE	:    Dict          =			2
					SCREAMING_SNAKE_CASE	:    Optional[Any]          =			2

					for model_class in self.all_generative_model_classes:
									SCREAMING_SNAKE_CASE	:    int          =			model_class(lowerCamelCase__   )

									SCREAMING_SNAKE_CASE	:    Optional[Any]          =			model.generate(lowerCamelCase__   ).sequences
									self.assertEqual(generation_outputs.shape[0],       input_ids.shape[0] * config.num_return_sequences   )


	def 					lowercase__	(self			:			int   )  ->      str:

					"""simple docstring"""

					SCREAMING_SNAKE_CASE	:    Any          =			self._get_input_ids_and_config()
					SCREAMING_SNAKE_CASE	:    str          =			True
					SCREAMING_SNAKE_CASE	:    Dict          =			max_length
					SCREAMING_SNAKE_CASE	:    Tuple          =			0.8
					SCREAMING_SNAKE_CASE	:    int          =			10
					SCREAMING_SNAKE_CASE	:    Dict          =			0.3
					SCREAMING_SNAKE_CASE	:    int          =			1
					SCREAMING_SNAKE_CASE	:    Tuple          =			8
					SCREAMING_SNAKE_CASE	:    Tuple          =			9

					for model_class in self.all_generative_model_classes:
									SCREAMING_SNAKE_CASE	:    List[Any]          =			model_class(lowerCamelCase__   )

									SCREAMING_SNAKE_CASE	:    Any          =			model.generate(lowerCamelCase__   ).sequences
									self.assertEqual(generation_outputs.shape[-1],       lowerCamelCase__   )

									SCREAMING_SNAKE_CASE	:    List[str]          =			jit(model.generate   )
									SCREAMING_SNAKE_CASE	:    Any          =			jit_generate(lowerCamelCase__   ).sequences

									self.assertListEqual(generation_outputs.tolist(),       jit_generation_outputs.tolist()   )


	def 					lowercase__	(self			:			int   )  ->      List[Any]:

					"""simple docstring"""

					SCREAMING_SNAKE_CASE	:    Union[str, Any]          =			self._get_input_ids_and_config()
					SCREAMING_SNAKE_CASE	:    Any          =			max_length
					SCREAMING_SNAKE_CASE	:    List[str]          =			1
					SCREAMING_SNAKE_CASE	:    Any          =			8
					SCREAMING_SNAKE_CASE	:    Optional[int]          =			9

					for model_class in self.all_generative_model_classes:
									SCREAMING_SNAKE_CASE	:    Optional[Any]          =			model_class(lowerCamelCase__   )

									SCREAMING_SNAKE_CASE	:    Union[str, Any]          =			model.generate(lowerCamelCase__   ).sequences
									self.assertEqual(generation_outputs.shape[-1],       lowerCamelCase__   )

									SCREAMING_SNAKE_CASE	:    List[Any]          =			jit(model.generate   )
									SCREAMING_SNAKE_CASE	:    Tuple          =			jit_generate(lowerCamelCase__   ).sequences

									self.assertListEqual(generation_outputs.tolist(),       jit_generation_outputs.tolist()   )


	def 					lowercase__	(self			:			Dict   )  ->      Optional[int]:

					"""simple docstring"""

					SCREAMING_SNAKE_CASE	:    Optional[Any]          =			self._get_input_ids_and_config()
					SCREAMING_SNAKE_CASE	:    Optional[Any]          =			max_length
					SCREAMING_SNAKE_CASE	:    int          =			2
					SCREAMING_SNAKE_CASE	:    List[str]          =			1
					SCREAMING_SNAKE_CASE	:    Optional[Any]          =			8
					SCREAMING_SNAKE_CASE	:    List[str]          =			9

					for model_class in self.all_generative_model_classes:
									SCREAMING_SNAKE_CASE	:    List[str]          =			model_class(lowerCamelCase__   )

									SCREAMING_SNAKE_CASE	:    List[Any]          =			model.generate(lowerCamelCase__   ).sequences
									self.assertEqual(generation_outputs.shape[-1],       lowerCamelCase__   )

									SCREAMING_SNAKE_CASE	:    Dict          =			jit(model.generate   )
									SCREAMING_SNAKE_CASE	:    Tuple          =			jit_generate(lowerCamelCase__   ).sequences

									self.assertListEqual(generation_outputs.tolist(),       jit_generation_outputs.tolist()   )


	def 					lowercase__	(self			:			Any   )  ->      Union[str, Any]:

					"""simple docstring"""

					SCREAMING_SNAKE_CASE	:    Dict          =			self._get_input_ids_and_config()

					# pad attention mask on the left
					SCREAMING_SNAKE_CASE	:    Union[str, Any]          =			attention_mask.at[(0, 0)].set(0   )

					SCREAMING_SNAKE_CASE	:    Optional[int]          =			False
					SCREAMING_SNAKE_CASE	:    str          =			max_length

					for model_class in self.all_generative_model_classes:
									SCREAMING_SNAKE_CASE	:    List[Any]          =			model_class(lowerCamelCase__   )

									SCREAMING_SNAKE_CASE	:    Union[str, Any]          =			model.generate(lowerCamelCase__,       attention_mask=lowerCamelCase__   ).sequences
									self.assertEqual(generation_outputs.shape[-1],       lowerCamelCase__   )

									SCREAMING_SNAKE_CASE	:    str          =			jit(model.generate   )
									SCREAMING_SNAKE_CASE	:    str          =			jit_generate(lowerCamelCase__,       attention_mask=lowerCamelCase__   ).sequences

									self.assertListEqual(generation_outputs.tolist(),       jit_generation_outputs.tolist()   )


	def 					lowercase__	(self			:			Union[str, Any]   )  ->      Tuple:

					"""simple docstring"""

					SCREAMING_SNAKE_CASE	:    List[Any]          =			self._get_input_ids_and_config()

					# pad attention mask on the left
					SCREAMING_SNAKE_CASE	:    Dict          =			attention_mask.at[(0, 0)].set(0   )

					SCREAMING_SNAKE_CASE	:    List[Any]          =			True
					SCREAMING_SNAKE_CASE	:    Union[str, Any]          =			max_length

					for model_class in self.all_generative_model_classes:
									SCREAMING_SNAKE_CASE	:    List[Any]          =			model_class(lowerCamelCase__   )

									SCREAMING_SNAKE_CASE	:    Optional[Any]          =			model.generate(lowerCamelCase__,       attention_mask=lowerCamelCase__   ).sequences
									self.assertEqual(generation_outputs.shape[-1],       lowerCamelCase__   )

									SCREAMING_SNAKE_CASE	:    Union[str, Any]          =			jit(model.generate   )
									SCREAMING_SNAKE_CASE	:    List[str]          =			jit_generate(lowerCamelCase__,       attention_mask=lowerCamelCase__   ).sequences

									self.assertListEqual(generation_outputs.tolist(),       jit_generation_outputs.tolist()   )






	def 					lowercase__	(self			:			List[str]   )  ->      Tuple:

					"""simple docstring"""

					SCREAMING_SNAKE_CASE	:    List[str]          =			self._get_input_ids_and_config()

					# pad attention mask on the left
					SCREAMING_SNAKE_CASE	:    Union[str, Any]          =			attention_mask.at[(0, 0)].set(0   )

					SCREAMING_SNAKE_CASE	:    Any          =			2
					SCREAMING_SNAKE_CASE	:    int          =			max_length

					for model_class in self.all_generative_model_classes:
									SCREAMING_SNAKE_CASE	:    Dict          =			model_class(lowerCamelCase__   )

									SCREAMING_SNAKE_CASE	:    Tuple          =			model.generate(lowerCamelCase__,       attention_mask=lowerCamelCase__   ).sequences
									self.assertEqual(generation_outputs.shape[-1],       lowerCamelCase__   )

									SCREAMING_SNAKE_CASE	:    Any          =			jit(model.generate   )
									SCREAMING_SNAKE_CASE	:    Optional[int]          =			jit_generate(lowerCamelCase__,       attention_mask=lowerCamelCase__   ).sequences

									self.assertListEqual(generation_outputs.tolist(),       jit_generation_outputs.tolist()   )



@require_flax
class 						a__  (					unittest.TestCase						):


	def 					lowercase__	(self			:			Union[str, Any]   )  ->      int:

					"""simple docstring"""

					SCREAMING_SNAKE_CASE	:    Union[str, Any]          =			AutoTokenizer.from_pretrained('''hf-internal-testing/tiny-bert'''   )
					SCREAMING_SNAKE_CASE	:    int          =			FlaxAutoModelForCausalLM.from_pretrained('''hf-internal-testing/tiny-bert-flax-only'''   )

					SCREAMING_SNAKE_CASE	:    Union[str, Any]          =			"Hello world"
					SCREAMING_SNAKE_CASE	:    List[str]          =			tokenizer(lowerCamelCase__,       return_tensors='''np'''   ).input_ids

					# typos are quickly detected (the correct argument is `do_sample`)
					with self.assertRaisesRegex(lowerCamelCase__,       '''do_samples'''   ):
									model.generate(lowerCamelCase__,       do_samples=lowerCamelCase__   )

					# arbitrary arguments that will not be used anywhere are also not accepted
					with self.assertRaisesRegex(lowerCamelCase__,       '''foo'''   ):
									SCREAMING_SNAKE_CASE	:    Any          =			{"foo": "bar"}
									model.generate(lowerCamelCase__,       **lowerCamelCase__   )





from dataclasses import dataclass
from typing import Optional, Tuple, Union

import torch
import torch.nn as nn

from ..configuration_utils import ConfigMixin, register_to_config
from ..utils import BaseOutput
from .embeddings import GaussianFourierProjection, TimestepEmbedding, Timesteps
from .modeling_utils import ModelMixin
from .unet_ad_blocks import get_down_block, get_mid_block, get_out_block, get_up_block

@dataclass
class 					_lowercase    (     UpperCAmelCase__			):






    '''simple docstring'''
    SCREAMING_SNAKE_CASE:				torch.FloatTensor





class 					_lowercase    (     UpperCAmelCase__ ,  UpperCAmelCase__			):






    '''simple docstring'''




    @register_to_config
    def __init__(    self       ,						lowerCamelCase__ = 65_536       ,						lowerCamelCase__ = None       ,						lowerCamelCase__ = 2       ,						lowerCamelCase__ = 2       ,						lowerCamelCase__ = 0       ,						lowerCamelCase__ = "fourier"       ,						lowerCamelCase__ = True       ,						lowerCamelCase__ = False       ,						lowerCamelCase__ = 0.0       ,						lowerCamelCase__ = ("DownBlock1DNoSkip", "DownBlock1D", "AttnDownBlock1D")       ,						lowerCamelCase__ = ("AttnUpBlock1D", "UpBlock1D", "UpBlock1DNoSkip")       ,						lowerCamelCase__ = "UNetMidBlock1D"       ,						lowerCamelCase__ = None       ,						lowerCamelCase__ = (32, 32, 64)       ,						lowerCamelCase__ = None       ,						lowerCamelCase__ = 8       ,						lowerCamelCase__ = 1       ,						lowerCamelCase__ = False       ,						):
       super().__init__()
       lowerCAmelCase_:							Optional[Any]       =	sample_size

       # time
       if time_embedding_type == "fourier":
          lowerCAmelCase_:							Dict       =	GaussianFourierProjection(
              embedding_size=8       ,						set_W_to_weight=lowerCamelCase__       ,						log=lowerCamelCase__       ,						flip_sin_to_cos=lowerCamelCase__ )
          lowerCAmelCase_:							List[str]       =	2 * block_out_channels[0]
       elif time_embedding_type == "positional":
          lowerCAmelCase_:							Tuple       =	Timesteps(
              block_out_channels[0]       ,						flip_sin_to_cos=lowerCamelCase__       ,						downscale_freq_shift=lowerCamelCase__ )
          lowerCAmelCase_:							Dict       =	block_out_channels[0]

       if use_timestep_embedding:
          lowerCAmelCase_:							Tuple       =	block_out_channels[0] * 4
          lowerCAmelCase_:							Any       =	TimestepEmbedding(
              in_channels=lowerCamelCase__       ,						time_embed_dim=lowerCamelCase__       ,						act_fn=lowerCamelCase__       ,						out_dim=block_out_channels[0]       ,						)

       lowerCAmelCase_:							str       =	nn.ModuleList([] )
       lowerCAmelCase_:							Dict       =	None
       lowerCAmelCase_:							Optional[Any]       =	nn.ModuleList([] )
       lowerCAmelCase_:							int       =	None

       # down
       lowerCAmelCase_:							List[str]       =	in_channels
       for i, down_block_type in enumerate(lowerCamelCase__ ):
          lowerCAmelCase_:							Optional[int]       =	output_channel
          lowerCAmelCase_:							Optional[Any]       =	block_out_channels[i]

          if i == 0:
             input_channel += extra_in_channels

          lowerCAmelCase_:							List[str]       =	i == len(lowerCamelCase__ ) - 1

          lowerCAmelCase_:							List[str]       =	get_down_block(
              lowerCamelCase__       ,						num_layers=lowerCamelCase__       ,						in_channels=lowerCamelCase__       ,						out_channels=lowerCamelCase__       ,						temb_channels=block_out_channels[0]       ,						add_downsample=not is_final_block or downsample_each_block       ,						)
          self.down_blocks.append(lowerCamelCase__ )

       # mid
       lowerCAmelCase_:							Optional[int]       =	get_mid_block(
           lowerCamelCase__       ,						in_channels=block_out_channels[-1]       ,						mid_channels=block_out_channels[-1]       ,						out_channels=block_out_channels[-1]       ,						embed_dim=block_out_channels[0]       ,						num_layers=lowerCamelCase__       ,						add_downsample=lowerCamelCase__       ,						)

       # up
       lowerCAmelCase_:							Dict       =	list(reversed(lowerCamelCase__ ) )
       lowerCAmelCase_:							Any       =	reversed_block_out_channels[0]
       if out_block_type is None:
          lowerCAmelCase_:							str       =	out_channels
       else:
          lowerCAmelCase_:							Any       =	block_out_channels[0]

       for i, up_block_type in enumerate(lowerCamelCase__ ):
          lowerCAmelCase_:							Dict       =	output_channel
          lowerCAmelCase_:							int       =	(
              reversed_block_out_channels[i + 1] if i < len(lowerCamelCase__ ) - 1 else final_upsample_channels
          )

          lowerCAmelCase_:							Optional[int]       =	i == len(lowerCamelCase__ ) - 1

          lowerCAmelCase_:							Union[str, Any]       =	get_up_block(
              lowerCamelCase__       ,						num_layers=lowerCamelCase__       ,						in_channels=lowerCamelCase__       ,						out_channels=lowerCamelCase__       ,						temb_channels=block_out_channels[0]       ,						add_upsample=not is_final_block       ,						)
          self.up_blocks.append(lowerCamelCase__ )
          lowerCAmelCase_:							str       =	output_channel

       # out
       lowerCAmelCase_:							List[str]       =	norm_num_groups if norm_num_groups is not None else min(block_out_channels[0] // 4       ,						32 )
       lowerCAmelCase_:							int       =	get_out_block(
           out_block_type=lowerCamelCase__       ,						num_groups_out=lowerCamelCase__       ,						embed_dim=block_out_channels[0]       ,						out_channels=lowerCamelCase__       ,						act_fn=lowerCamelCase__       ,						fc_dim=block_out_channels[-1] // 4       ,						)




    def      _a    (    self       ,						lowerCamelCase__       ,						lowerCamelCase__       ,						lowerCamelCase__ = True       ,						):
       lowerCAmelCase_:							Any       =	timestep
       if not torch.is_tensor(lowerCamelCase__ ):
          lowerCAmelCase_:							Optional[Any]       =	torch.tensor([timesteps]       ,						dtype=torch.long       ,						device=sample.device )
       elif torch.is_tensor(lowerCamelCase__ ) and len(timesteps.shape ) == 0:
          lowerCAmelCase_:							List[Any]       =	timesteps[None].to(sample.device )

       lowerCAmelCase_:							Union[str, Any]       =	self.time_proj(lowerCamelCase__ )
       if self.config.use_timestep_embedding:
          lowerCAmelCase_:							Any       =	self.time_mlp(lowerCamelCase__ )
       else:
          lowerCAmelCase_:							Any       =	timestep_embed[..., None]
          lowerCAmelCase_:							str       =	timestep_embed.repeat([1, 1, sample.shape[2]] ).to(sample.dtype )
          lowerCAmelCase_:							Dict       =	timestep_embed.broadcast_to((sample.shape[:1] + timestep_embed.shape[1:]) )

       # 2. down
       lowerCAmelCase_:							Dict       =	()
       for downsample_block in self.down_blocks:
          lowerCAmelCase_  ,	lowerCAmelCase_:							Optional[int]       =	downsample_block(hidden_states=lowerCamelCase__       ,						temb=lowerCamelCase__ )
          down_block_res_samples += res_samples

       # 3. mid
       if self.mid_block:
          lowerCAmelCase_:							int       =	self.mid_block(lowerCamelCase__       ,						lowerCamelCase__ )

       # 4. up
       for i, upsample_block in enumerate(self.up_blocks ):
          lowerCAmelCase_:							Any       =	down_block_res_samples[-1:]
          lowerCAmelCase_:							str       =	down_block_res_samples[:-1]
          lowerCAmelCase_:							List[str]       =	upsample_block(lowerCamelCase__       ,						res_hidden_states_tuple=lowerCamelCase__       ,						temb=lowerCamelCase__ )

       # 5. post-process
       if self.out_block:
          lowerCAmelCase_:							Any       =	self.out_block(lowerCamelCase__       ,						lowerCamelCase__ )

       if not return_dict:
          return (sample,)

       return UNetaDOutput(sample=lowerCamelCase__ )