

import flax.linen as nn
import jax.numpy as jnp

from .attention_flax import FlaxTransformeraDModel
from .resnet_flax import FlaxDownsampleaD, FlaxResnetBlockaD, FlaxUpsampleaD



class   __lowercase				( nn.Module	):




			"""simple docstring"""







			_UpperCAmelCase   :							int
			_UpperCAmelCase   :							int
			_UpperCAmelCase   :							float										=     0.0
			_UpperCAmelCase   :							int										=     1
			_UpperCAmelCase   :							int										=     1
			_UpperCAmelCase   :							bool										=     True
			_UpperCAmelCase   :							bool										=     False
			_UpperCAmelCase   :							bool										=     False
			_UpperCAmelCase   :							bool										=     False
			_UpperCAmelCase   :							jnp.dtype										=     jnp.floataa



			def        _SCREAMING_SNAKE_CASE			( self  :     Any):
									SCREAMING_SNAKE_CASE_: Any  		=       []
									SCREAMING_SNAKE_CASE_: str  		=       []

									for i in range(self.num_layers):
															SCREAMING_SNAKE_CASE_: Optional[Any]  		=       self.in_channels if i == 0 else self.out_channels

															SCREAMING_SNAKE_CASE_: int  		=       FlaxResnetBlockaD(
															    in_channels=lowerCAmelCase__   ,						out_channels=self.out_channels   ,						dropout_prob=self.dropout   ,						dtype=self.dtype   ,						)
															resnets.append(lowerCAmelCase__)

															SCREAMING_SNAKE_CASE_: int  		=       FlaxTransformeraDModel(
															    in_channels=self.out_channels   ,						n_heads=self.num_attention_heads   ,						d_head=self.out_channels // self.num_attention_heads   ,						depth=1   ,						use_linear_projection=self.use_linear_projection   ,						only_cross_attention=self.only_cross_attention   ,						use_memory_efficient_attention=self.use_memory_efficient_attention   ,						dtype=self.dtype   ,						)
															attentions.append(lowerCAmelCase__)

									SCREAMING_SNAKE_CASE_: Optional[Any]  		=       resnets
									SCREAMING_SNAKE_CASE_: str  		=       attentions

									if self.add_downsample:
															SCREAMING_SNAKE_CASE_: List[Any]  		=       FlaxDownsampleaD(self.out_channels   ,						dtype=self.dtype)




			def __call__( self  :     Dict   ,						lowerCAmelCase__  :     List[str]   ,						lowerCAmelCase__  :     Tuple   ,						lowerCAmelCase__  :     Dict   ,						lowerCAmelCase__  :     str=True):
									SCREAMING_SNAKE_CASE_: Optional[Any]  		=       ()

									for resnet, attn in zip(self.resnets   ,						self.attentions):
															SCREAMING_SNAKE_CASE_: int  		=       resnet(lowerCAmelCase__   ,						lowerCAmelCase__   ,						deterministic=lowerCAmelCase__)
															SCREAMING_SNAKE_CASE_: str  		=       attn(lowerCAmelCase__   ,						lowerCAmelCase__   ,						deterministic=lowerCAmelCase__)
															output_states += (hidden_states,)

									if self.add_downsample:
															SCREAMING_SNAKE_CASE_: Optional[Any]  		=       self.downsamplers_a(lowerCAmelCase__)
															output_states += (hidden_states,)

									return hidden_states, output_states







class   __lowercase				( nn.Module	):




			"""simple docstring"""







			_UpperCAmelCase   :							int
			_UpperCAmelCase   :							int
			_UpperCAmelCase   :							float										=     0.0
			_UpperCAmelCase   :							int										=     1
			_UpperCAmelCase   :							bool										=     True
			_UpperCAmelCase   :							jnp.dtype										=     jnp.floataa



			def        _SCREAMING_SNAKE_CASE			( self  :     Optional[int]):
									SCREAMING_SNAKE_CASE_: List[str]  		=       []

									for i in range(self.num_layers):
															SCREAMING_SNAKE_CASE_: Optional[int]  		=       self.in_channels if i == 0 else self.out_channels

															SCREAMING_SNAKE_CASE_: Union[str, Any]  		=       FlaxResnetBlockaD(
															    in_channels=lowerCAmelCase__   ,						out_channels=self.out_channels   ,						dropout_prob=self.dropout   ,						dtype=self.dtype   ,						)
															resnets.append(lowerCAmelCase__)
									SCREAMING_SNAKE_CASE_: Union[str, Any]  		=       resnets

									if self.add_downsample:
															SCREAMING_SNAKE_CASE_: Union[str, Any]  		=       FlaxDownsampleaD(self.out_channels   ,						dtype=self.dtype)




			def __call__( self  :     List[str]   ,						lowerCAmelCase__  :     int   ,						lowerCAmelCase__  :     str   ,						lowerCAmelCase__  :     int=True):
									SCREAMING_SNAKE_CASE_: int  		=       ()

									for resnet in self.resnets:
															SCREAMING_SNAKE_CASE_: Tuple  		=       resnet(lowerCAmelCase__   ,						lowerCAmelCase__   ,						deterministic=lowerCAmelCase__)
															output_states += (hidden_states,)

									if self.add_downsample:
															SCREAMING_SNAKE_CASE_: Any  		=       self.downsamplers_a(lowerCAmelCase__)
															output_states += (hidden_states,)

									return hidden_states, output_states







class   __lowercase				( nn.Module	):




			"""simple docstring"""







			_UpperCAmelCase   :							int
			_UpperCAmelCase   :							int
			_UpperCAmelCase   :							int
			_UpperCAmelCase   :							float										=     0.0
			_UpperCAmelCase   :							int										=     1
			_UpperCAmelCase   :							int										=     1
			_UpperCAmelCase   :							bool										=     True
			_UpperCAmelCase   :							bool										=     False
			_UpperCAmelCase   :							bool										=     False
			_UpperCAmelCase   :							bool										=     False
			_UpperCAmelCase   :							jnp.dtype										=     jnp.floataa



			def        _SCREAMING_SNAKE_CASE			( self  :     Optional[int]):
									SCREAMING_SNAKE_CASE_: int  		=       []
									SCREAMING_SNAKE_CASE_: List[Any]  		=       []

									for i in range(self.num_layers):
															SCREAMING_SNAKE_CASE_: int  		=       self.in_channels if (i == self.num_layers - 1) else self.out_channels
															SCREAMING_SNAKE_CASE_: List[str]  		=       self.prev_output_channel if i == 0 else self.out_channels

															SCREAMING_SNAKE_CASE_: Tuple  		=       FlaxResnetBlockaD(
															    in_channels=resnet_in_channels + res_skip_channels   ,						out_channels=self.out_channels   ,						dropout_prob=self.dropout   ,						dtype=self.dtype   ,						)
															resnets.append(lowerCAmelCase__)

															SCREAMING_SNAKE_CASE_: Tuple  		=       FlaxTransformeraDModel(
															    in_channels=self.out_channels   ,						n_heads=self.num_attention_heads   ,						d_head=self.out_channels // self.num_attention_heads   ,						depth=1   ,						use_linear_projection=self.use_linear_projection   ,						only_cross_attention=self.only_cross_attention   ,						use_memory_efficient_attention=self.use_memory_efficient_attention   ,						dtype=self.dtype   ,						)
															attentions.append(lowerCAmelCase__)

									SCREAMING_SNAKE_CASE_: int  		=       resnets
									SCREAMING_SNAKE_CASE_: Union[str, Any]  		=       attentions

									if self.add_upsample:
															SCREAMING_SNAKE_CASE_: str  		=       FlaxUpsampleaD(self.out_channels   ,						dtype=self.dtype)




			def __call__( self  :     List[str]   ,						lowerCAmelCase__  :     List[str]   ,						lowerCAmelCase__  :     Tuple   ,						lowerCAmelCase__  :     str   ,						lowerCAmelCase__  :     List[str]   ,						lowerCAmelCase__  :     Dict=True):
									for resnet, attn in zip(self.resnets   ,						self.attentions):
															# pop res hidden states
															SCREAMING_SNAKE_CASE_: Tuple  		=       res_hidden_states_tuple[-1]
															SCREAMING_SNAKE_CASE_: int  		=       res_hidden_states_tuple[:-1]
															SCREAMING_SNAKE_CASE_: Tuple  		=       jnp.concatenate((hidden_states, res_hidden_states)   ,						axis=-1)

															SCREAMING_SNAKE_CASE_: Tuple  		=       resnet(lowerCAmelCase__   ,						lowerCAmelCase__   ,						deterministic=lowerCAmelCase__)
															SCREAMING_SNAKE_CASE_: List[Any]  		=       attn(lowerCAmelCase__   ,						lowerCAmelCase__   ,						deterministic=lowerCAmelCase__)

									if self.add_upsample:
															SCREAMING_SNAKE_CASE_: str  		=       self.upsamplers_a(lowerCAmelCase__)

									return hidden_states







class   __lowercase				( nn.Module	):




			"""simple docstring"""







			_UpperCAmelCase   :							int
			_UpperCAmelCase   :							int
			_UpperCAmelCase   :							int
			_UpperCAmelCase   :							float										=     0.0
			_UpperCAmelCase   :							int										=     1
			_UpperCAmelCase   :							bool										=     True
			_UpperCAmelCase   :							jnp.dtype										=     jnp.floataa



			def        _SCREAMING_SNAKE_CASE			( self  :     List[Any]):
									SCREAMING_SNAKE_CASE_: str  		=       []

									for i in range(self.num_layers):
															SCREAMING_SNAKE_CASE_: int  		=       self.in_channels if (i == self.num_layers - 1) else self.out_channels
															SCREAMING_SNAKE_CASE_: List[str]  		=       self.prev_output_channel if i == 0 else self.out_channels

															SCREAMING_SNAKE_CASE_: Dict  		=       FlaxResnetBlockaD(
															    in_channels=resnet_in_channels + res_skip_channels   ,						out_channels=self.out_channels   ,						dropout_prob=self.dropout   ,						dtype=self.dtype   ,						)
															resnets.append(lowerCAmelCase__)

									SCREAMING_SNAKE_CASE_: int  		=       resnets

									if self.add_upsample:
															SCREAMING_SNAKE_CASE_: Dict  		=       FlaxUpsampleaD(self.out_channels   ,						dtype=self.dtype)




			def __call__( self  :     int   ,						lowerCAmelCase__  :     int   ,						lowerCAmelCase__  :     Optional[int]   ,						lowerCAmelCase__  :     List[str]   ,						lowerCAmelCase__  :     Union[str, Any]=True):
									for resnet in self.resnets:
															# pop res hidden states
															SCREAMING_SNAKE_CASE_: Union[str, Any]  		=       res_hidden_states_tuple[-1]
															SCREAMING_SNAKE_CASE_: int  		=       res_hidden_states_tuple[:-1]
															SCREAMING_SNAKE_CASE_: Optional[Any]  		=       jnp.concatenate((hidden_states, res_hidden_states)   ,						axis=-1)

															SCREAMING_SNAKE_CASE_: Dict  		=       resnet(lowerCAmelCase__   ,						lowerCAmelCase__   ,						deterministic=lowerCAmelCase__)

									if self.add_upsample:
															SCREAMING_SNAKE_CASE_: Optional[Any]  		=       self.upsamplers_a(lowerCAmelCase__)

									return hidden_states







class   __lowercase				( nn.Module	):




			"""simple docstring"""







			_UpperCAmelCase   :							int
			_UpperCAmelCase   :							float										=     0.0
			_UpperCAmelCase   :							int										=     1
			_UpperCAmelCase   :							int										=     1
			_UpperCAmelCase   :							bool										=     False
			_UpperCAmelCase   :							bool										=     False
			_UpperCAmelCase   :							jnp.dtype										=     jnp.floataa



			def        _SCREAMING_SNAKE_CASE			( self  :     List[Any]):
									# there is always at least one resnet
									SCREAMING_SNAKE_CASE_: Union[str, Any]  		=       [
									    FlaxResnetBlockaD(
									        in_channels=self.in_channels   ,						out_channels=self.in_channels   ,						dropout_prob=self.dropout   ,						dtype=self.dtype   ,						)
									]

									SCREAMING_SNAKE_CASE_: List[Any]  		=       []

									for _ in range(self.num_layers):
															SCREAMING_SNAKE_CASE_: str  		=       FlaxTransformeraDModel(
															    in_channels=self.in_channels   ,						n_heads=self.num_attention_heads   ,						d_head=self.in_channels // self.num_attention_heads   ,						depth=1   ,						use_linear_projection=self.use_linear_projection   ,						use_memory_efficient_attention=self.use_memory_efficient_attention   ,						dtype=self.dtype   ,						)
															attentions.append(lowerCAmelCase__)

															SCREAMING_SNAKE_CASE_: Any  		=       FlaxResnetBlockaD(
															    in_channels=self.in_channels   ,						out_channels=self.in_channels   ,						dropout_prob=self.dropout   ,						dtype=self.dtype   ,						)
															resnets.append(lowerCAmelCase__)

									SCREAMING_SNAKE_CASE_: Optional[Any]  		=       resnets
									SCREAMING_SNAKE_CASE_: Dict  		=       attentions




			def __call__( self  :     Optional[int]   ,						lowerCAmelCase__  :     int   ,						lowerCAmelCase__  :     Optional[int]   ,						lowerCAmelCase__  :     Optional[Any]   ,						lowerCAmelCase__  :     Tuple=True):
									SCREAMING_SNAKE_CASE_: Optional[int]  		=       self.resnets[0](lowerCAmelCase__   ,						lowerCAmelCase__)
									for attn, resnet in zip(self.attentions   ,						self.resnets[1:]):
															SCREAMING_SNAKE_CASE_: Any  		=       attn(lowerCAmelCase__   ,						lowerCAmelCase__   ,						deterministic=lowerCAmelCase__)
															SCREAMING_SNAKE_CASE_: Any  		=       resnet(lowerCAmelCase__   ,						lowerCAmelCase__   ,						deterministic=lowerCAmelCase__)

									return hidden_states












"""simple docstring"""




import operator as op


lowerCAmelCase							:			Dict     							=							"""scaler.pt"""
lowerCAmelCase							:			Tuple     							=							"""pytorch_model"""
lowerCAmelCase							:			Union[str, Any]     							=							"""random_states"""
lowerCAmelCase							:			Union[str, Any]     							=							"""optimizer"""
lowerCAmelCase							:			Dict     							=							"""scheduler"""
lowerCAmelCase							:			int     							=							"""pytorch_model.bin"""
lowerCAmelCase							:			str     							=							"""pytorch_model.bin.index.json"""
lowerCAmelCase							:			Union[str, Any]     							=							"""model.safetensors"""
lowerCAmelCase							:			List[Any]     							=							"""model.safetensors.index.json"""
lowerCAmelCase							:			List[Any]     							=							"""1.10.2"""
lowerCAmelCase							:			Any     							=							"""py38"""
lowerCAmelCase							:			Optional[int]     							=							"""4.17.0"""
lowerCAmelCase							:			str     							=							["""ml.p3.16xlarge""", """ml.p3dn.24xlarge""", """ml.p4dn.24xlarge"""]
lowerCAmelCase							:			Tuple     							=							["""FULL_SHARD""", """SHARD_GRAD_OP""", """NO_SHARD""", """HYBRID_SHARD""", """HYBRID_SHARD_ZERO2"""]
lowerCAmelCase							:			List[Any]     							=							["""TRANSFORMER_BASED_WRAP""", """SIZE_BASED_WRAP""", """NO_WRAP"""]
lowerCAmelCase							:			List[str]     							=							["""BACKWARD_PRE""", """BACKWARD_POST""", """NO_PREFETCH"""]
lowerCAmelCase							:			List[str]     							=							["""FULL_STATE_DICT""", """LOCAL_STATE_DICT""", """SHARDED_STATE_DICT"""]
lowerCAmelCase							:			Any     							=							"""2.0.1"""
lowerCAmelCase							:			List[Any]     							=							["""pdsh""", """standard""", """openmpi""", """mvapich"""]
lowerCAmelCase							:			Union[str, Any]     							=							["""default""", """reduce-overhead""", """max-autotune"""]

lowerCAmelCase							:			Optional[int]     							=							{""">""": op.gt, """>=""": op.ge, """==""": op.eq, """!=""": op.ne, """<=""": op.le, """<""": op.lt}

# These are the args for `torch.distributed.launch` for pytorch < 1.9
lowerCAmelCase							:			Union[str, Any]     							=							[
    """nnodes""",
    """nproc_per_node""",
    """rdzv_backend""",
    """rdzv_endpoint""",
    """rdzv_id""",
    """rdzv_conf""",
    """standalone""",
    """max_restarts""",
    """monitor_interval""",
    """start_method""",
    """role""",
    """module""",
    """m""",
    """no_python""",
    """run_path""",
    """log_dir""",
    """r""",
    """redirects""",
    """t""",
    """tee""",
    """node_rank""",
    """master_addr""",
    """master_port""",
]

lowerCAmelCase							:			List[str]     							=							["""DEEPSPEED""", """MULTI_GPU""", """FSDP""", """MEGATRON_LM"""]
lowerCAmelCase							:			Optional[Any]     							=							["""DEEPSPEED""", """MULTI_XPU""", """FSDP"""]



