



from typing import List, Optional, Union

import torch
from transformers import (
    XLMRobertaTokenizer,
)

from ...models import UNetaDConditionModel, VQModel
from ...pipelines import DiffusionPipeline
from ...pipelines.pipeline_utils import ImagePipelineOutput
from ...schedulers import DDIMScheduler, DDPMScheduler
from ...utils import (
    is_accelerate_available,
    is_accelerate_version,
    logging,
    randn_tensor,
    replace_example_docstring,
)
from .text_encoder import MultilingualCLIP


__UpperCAmelCase          =       logging.get_logger(__name__)  # pylint: disable=invalid-name

__UpperCAmelCase          =       '''
    Examples:
        ```py
        >>> from diffusers import KandinskyPipeline, KandinskyPriorPipeline
        >>> import torch

        >>> pipe_prior = KandinskyPriorPipeline.from_pretrained("kandinsky-community/Kandinsky-2-1-prior")
        >>> pipe_prior.to("cuda")

        >>> prompt = "red cat, 4k photo"
        >>> out = pipe_prior(prompt)
        >>> image_emb = out.image_embeds
        >>> negative_image_emb = out.negative_image_embeds

        >>> pipe = KandinskyPipeline.from_pretrained("kandinsky-community/kandinsky-2-1")
        >>> pipe.to("cuda")

        >>> image = pipe(
        ...     prompt,
        ...     image_embeds=image_emb,
        ...     negative_image_embeds=negative_image_emb,
        ...     height=768,
        ...     width=768,
        ...     num_inference_steps=100,
        ... ).images

        >>> image[0].save("cat.png")
        ```
'''
def 		UpperCamelCase	(   snake_case__       :    Any       ,     snake_case__       :    Dict       ,     snake_case__       :    Union[str, Any]=8	)	->     Tuple:
   UpperCamelCase    :  List[str]							=	h // scale_factor**2
   if h % scale_factor**2 != 0:
      new_h += 1
   UpperCamelCase    :  Tuple							=	w // scale_factor**2
   if w % scale_factor**2 != 0:
      new_w += 1
   return new_h * scale_factor, new_w * scale_factor



class 				lowerCAmelCase_		(  a__							):
       def __init__(			self,			SCREAMING_SNAKE_CASE_,			SCREAMING_SNAKE_CASE_,			SCREAMING_SNAKE_CASE_,			SCREAMING_SNAKE_CASE_,			SCREAMING_SNAKE_CASE_,			)     ->		Union[str, Any]:
          super().__init__()

          self.register_modules(
              text_encoder=SCREAMING_SNAKE_CASE_,			tokenizer=SCREAMING_SNAKE_CASE_,			unet=SCREAMING_SNAKE_CASE_,			scheduler=SCREAMING_SNAKE_CASE_,			movq=SCREAMING_SNAKE_CASE_,			)
          UpperCamelCase    :  Union[str, Any]							=	2 ** (len(self.movq.config.block_out_channels       ) - 1)
       def 					snake_case_       (			self,			SCREAMING_SNAKE_CASE_,			SCREAMING_SNAKE_CASE_,			SCREAMING_SNAKE_CASE_,			SCREAMING_SNAKE_CASE_,			SCREAMING_SNAKE_CASE_,			SCREAMING_SNAKE_CASE_       )     ->		Optional[Any]:
          if latents is None:
             UpperCamelCase    :  int							=	randn_tensor(SCREAMING_SNAKE_CASE_,			generator=SCREAMING_SNAKE_CASE_,			device=SCREAMING_SNAKE_CASE_,			dtype=SCREAMING_SNAKE_CASE_       )
          else:
             if latents.shape != shape:
                raise ValueError(F"""Unexpected latents shape, got {latents.shape}, expected {shape}"""       )
             UpperCamelCase    :  str							=	latents.to(SCREAMING_SNAKE_CASE_       )

          UpperCamelCase    :  Any							=	latents * scheduler.init_noise_sigma
          return latents
       def 					snake_case_       (			self,			SCREAMING_SNAKE_CASE_,			SCREAMING_SNAKE_CASE_,			SCREAMING_SNAKE_CASE_,			SCREAMING_SNAKE_CASE_,			SCREAMING_SNAKE_CASE_=None,			)     ->		Optional[int]:
          UpperCamelCase    :  Optional[int]							=	len(SCREAMING_SNAKE_CASE_       ) if isinstance(SCREAMING_SNAKE_CASE_,			SCREAMING_SNAKE_CASE_       ) else 1
          # get prompt text embeddings
          UpperCamelCase    :  Union[str, Any]							=	self.tokenizer(
              SCREAMING_SNAKE_CASE_,			padding='max_length',			truncation=SCREAMING_SNAKE_CASE_,			max_length=77,			return_attention_mask=SCREAMING_SNAKE_CASE_,			add_special_tokens=SCREAMING_SNAKE_CASE_,			return_tensors='pt',			)

          UpperCamelCase    :  Any							=	text_inputs.input_ids
          UpperCamelCase    :  Any							=	self.tokenizer(SCREAMING_SNAKE_CASE_,			padding='longest',			return_tensors='pt'       ).input_ids

          if untruncated_ids.shape[-1] >= text_input_ids.shape[-1] and not torch.equal(SCREAMING_SNAKE_CASE_,			SCREAMING_SNAKE_CASE_       ):
             UpperCamelCase    :  Any							=	self.tokenizer.batch_decode(untruncated_ids[:, self.tokenizer.model_max_length - 1 : -1]       )
             logger.warning(
                 'The following part of your input was truncated because CLIP can only handle sequences up to'
                 F""" {self.tokenizer.model_max_length} tokens: {removed_text}"""       )

          UpperCamelCase    :  List[str]							=	text_input_ids.to(SCREAMING_SNAKE_CASE_       )
          UpperCamelCase    :  Union[str, Any]							=	text_inputs.attention_mask.to(SCREAMING_SNAKE_CASE_       )

          UpperCamelCase	,     UpperCamelCase    :  List[str]							=	self.text_encoder(
              input_ids=SCREAMING_SNAKE_CASE_,			attention_mask=SCREAMING_SNAKE_CASE_       )

          UpperCamelCase    :  List[str]							=	prompt_embeds.repeat_interleave(SCREAMING_SNAKE_CASE_,			dim=0       )
          UpperCamelCase    :  List[str]							=	text_encoder_hidden_states.repeat_interleave(SCREAMING_SNAKE_CASE_,			dim=0       )
          UpperCamelCase    :  List[Any]							=	text_mask.repeat_interleave(SCREAMING_SNAKE_CASE_,			dim=0       )

          if do_classifier_free_guidance:
             UpperCamelCase    :  List[str]
             if negative_prompt is None:
                UpperCamelCase    :  List[Any]							=	[''] * batch_size
             elif type(SCREAMING_SNAKE_CASE_       ) is not type(SCREAMING_SNAKE_CASE_       ):
                raise TypeError(
                    F"""`negative_prompt` should be the same type to `prompt`, but got {type(SCREAMING_SNAKE_CASE_       )} !="""
                    F""" {type(SCREAMING_SNAKE_CASE_       )}."""       )
             elif isinstance(SCREAMING_SNAKE_CASE_,			SCREAMING_SNAKE_CASE_       ):
                UpperCamelCase    :  Optional[int]							=	[negative_prompt]
             elif batch_size != len(SCREAMING_SNAKE_CASE_       ):
                raise ValueError(
                    F"""`negative_prompt`: {negative_prompt} has batch size {len(SCREAMING_SNAKE_CASE_       )}, but `prompt`:"""
                    F""" {prompt} has batch size {batch_size}. Please make sure that passed `negative_prompt` matches"""
                    ' the batch size of `prompt`.'       )
             else:
                UpperCamelCase    :  Optional[Any]							=	negative_prompt

             UpperCamelCase    :  int							=	self.tokenizer(
                 SCREAMING_SNAKE_CASE_,			padding='max_length',			max_length=77,			truncation=SCREAMING_SNAKE_CASE_,			return_attention_mask=SCREAMING_SNAKE_CASE_,			add_special_tokens=SCREAMING_SNAKE_CASE_,			return_tensors='pt',			)
             UpperCamelCase    :  Dict							=	uncond_input.input_ids.to(SCREAMING_SNAKE_CASE_       )
             UpperCamelCase    :  Union[str, Any]							=	uncond_input.attention_mask.to(SCREAMING_SNAKE_CASE_       )

             UpperCamelCase	,     UpperCamelCase    :  Union[str, Any]							=	self.text_encoder(
                 input_ids=SCREAMING_SNAKE_CASE_,			attention_mask=SCREAMING_SNAKE_CASE_       )

             # duplicate unconditional embeddings for each generation per prompt, using mps friendly method

             UpperCamelCase    :  str							=	negative_prompt_embeds.shape[1]
             UpperCamelCase    :  Tuple							=	negative_prompt_embeds.repeat(1,			SCREAMING_SNAKE_CASE_       )
             UpperCamelCase    :  int							=	negative_prompt_embeds.view(batch_size * num_images_per_prompt,			SCREAMING_SNAKE_CASE_       )

             UpperCamelCase    :  Dict							=	uncond_text_encoder_hidden_states.shape[1]
             UpperCamelCase    :  str							=	uncond_text_encoder_hidden_states.repeat(1,			SCREAMING_SNAKE_CASE_,			1       )
             UpperCamelCase    :  str							=	uncond_text_encoder_hidden_states.view(
                 batch_size * num_images_per_prompt,			SCREAMING_SNAKE_CASE_,			-1       )
             UpperCamelCase    :  int							=	uncond_text_mask.repeat_interleave(SCREAMING_SNAKE_CASE_,			dim=0       )

             # done duplicates

             # For classifier free guidance, we need to do two forward passes.
             # Here we concatenate the unconditional and text embeddings into a single batch
             # to avoid doing two forward passes
             UpperCamelCase    :  Dict							=	torch.cat([negative_prompt_embeds, prompt_embeds]       )
             UpperCamelCase    :  Optional[int]							=	torch.cat([uncond_text_encoder_hidden_states, text_encoder_hidden_states]       )

             UpperCamelCase    :  str							=	torch.cat([uncond_text_mask, text_mask]       )

          return prompt_embeds, text_encoder_hidden_states, text_mask
       def 					snake_case_       (			self,			SCREAMING_SNAKE_CASE_=0       )     ->		Tuple:
          if is_accelerate_available():
             from accelerate import cpu_offload
          else:
             raise ImportError('Please install accelerate via `pip install accelerate`'       )

          UpperCamelCase    :  Dict							=	torch.device(F"""cuda:{gpu_id}"""       )

          UpperCamelCase    :  Optional[int]							=	[
              self.unet,
              self.text_encoder,
              self.movq,
          ]
          for cpu_offloaded_model in models:
             if cpu_offloaded_model is not None:
                cpu_offload(SCREAMING_SNAKE_CASE_,			SCREAMING_SNAKE_CASE_       )
       def 					snake_case_       (			self,			SCREAMING_SNAKE_CASE_=0       )     ->		Dict:
          if is_accelerate_available() and is_accelerate_version('>=',			'0.17.0.dev0'       ):
             from accelerate import cpu_offload_with_hook
          else:
             raise ImportError('`enable_model_cpu_offload` requires `accelerate v0.17.0` or higher.'       )

          UpperCamelCase    :  Tuple							=	torch.device(F"""cuda:{gpu_id}"""       )

          if self.device.type != "cpu":
             self.to('cpu',			silence_dtype_warnings=SCREAMING_SNAKE_CASE_       )
             torch.cuda.empty_cache()  # otherwise we don't see the memory savings (but they probably exist)

          UpperCamelCase    :  Union[str, Any]							=	None
          for cpu_offloaded_model in [self.text_encoder, self.unet, self.movq]:
             UpperCamelCase	,     UpperCamelCase    :  List[str]							=	cpu_offload_with_hook(SCREAMING_SNAKE_CASE_,			SCREAMING_SNAKE_CASE_,			prev_module_hook=SCREAMING_SNAKE_CASE_       )

          if self.safety_checker is not None:
             UpperCamelCase	,     UpperCamelCase    :  Union[str, Any]							=	cpu_offload_with_hook(self.safety_checker,			SCREAMING_SNAKE_CASE_,			prev_module_hook=SCREAMING_SNAKE_CASE_       )

          # We'll offload the last model manually.
          UpperCamelCase    :  Dict							=	hook
       @property
       # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.StableDiffusionPipeline._execution_device
       def 					snake_case_       (			self       )     ->		Optional[Any]:
          if not hasattr(self.unet,			'_hf_hook'       ):
             return self.device
          for module in self.unet.modules():
             if (
                 hasattr(SCREAMING_SNAKE_CASE_,			'_hf_hook'       )
                 and hasattr(module._hf_hook,			'execution_device'       )
                 and module._hf_hook.execution_device is not None
             ):
                return torch.device(module._hf_hook.execution_device       )
          return self.device






       @torch.no_grad()
       @replace_example_docstring(SCREAMING_SNAKE_CASE_       )
       def __call__(			self,			SCREAMING_SNAKE_CASE_,			SCREAMING_SNAKE_CASE_,			SCREAMING_SNAKE_CASE_,			SCREAMING_SNAKE_CASE_ = None,			SCREAMING_SNAKE_CASE_ = 512,			SCREAMING_SNAKE_CASE_ = 512,			SCREAMING_SNAKE_CASE_ = 100,			SCREAMING_SNAKE_CASE_ = 4.0,			SCREAMING_SNAKE_CASE_ = 1,			SCREAMING_SNAKE_CASE_ = None,			SCREAMING_SNAKE_CASE_ = None,			SCREAMING_SNAKE_CASE_ = "pil",			SCREAMING_SNAKE_CASE_ = True,			)     ->		str:

          if isinstance(SCREAMING_SNAKE_CASE_,			SCREAMING_SNAKE_CASE_       ):
             UpperCamelCase    :  Tuple							=	1
          elif isinstance(SCREAMING_SNAKE_CASE_,			SCREAMING_SNAKE_CASE_       ):
             UpperCamelCase    :  List[Any]							=	len(SCREAMING_SNAKE_CASE_       )
          else:
             raise ValueError(F"""`prompt` has to be of type `str` or `list` but is {type(SCREAMING_SNAKE_CASE_       )}"""       )

          UpperCamelCase    :  Optional[int]							=	self._execution_device

          UpperCamelCase    :  str							=	batch_size * num_images_per_prompt
          UpperCamelCase    :  int							=	guidance_scale > 1.0

          UpperCamelCase	,     UpperCamelCase	,     UpperCamelCase    :  int							=	self._encode_prompt(
              SCREAMING_SNAKE_CASE_,			SCREAMING_SNAKE_CASE_,			SCREAMING_SNAKE_CASE_,			SCREAMING_SNAKE_CASE_,			SCREAMING_SNAKE_CASE_       )

          if isinstance(SCREAMING_SNAKE_CASE_,			SCREAMING_SNAKE_CASE_       ):
             UpperCamelCase    :  Dict							=	torch.cat(SCREAMING_SNAKE_CASE_,			dim=0       )
          if isinstance(SCREAMING_SNAKE_CASE_,			SCREAMING_SNAKE_CASE_       ):
             UpperCamelCase    :  Dict							=	torch.cat(SCREAMING_SNAKE_CASE_,			dim=0       )

          if do_classifier_free_guidance:
             UpperCamelCase    :  Tuple							=	image_embeds.repeat_interleave(SCREAMING_SNAKE_CASE_,			dim=0       )
             UpperCamelCase    :  Optional[int]							=	negative_image_embeds.repeat_interleave(SCREAMING_SNAKE_CASE_,			dim=0       )

          UpperCamelCase    :  List[str]							=	torch.cat([negative_image_embeds, image_embeds],			dim=0       ).to(
              dtype=prompt_embeds.dtype,			device=SCREAMING_SNAKE_CASE_       )

          self.scheduler.set_timesteps(SCREAMING_SNAKE_CASE_,			device=SCREAMING_SNAKE_CASE_       )
          UpperCamelCase    :  int							=	self.scheduler.timesteps

          UpperCamelCase    :  int							=	self.unet.config.in_channels

          UpperCamelCase	,     UpperCamelCase    :  Optional[int]							=	get_new_h_w(SCREAMING_SNAKE_CASE_,			SCREAMING_SNAKE_CASE_,			self.movq_scale_factor       )

          # create initial latent
          UpperCamelCase    :  Optional[int]							=	self.prepare_latents(
              (batch_size, num_channels_latents, height, width),			text_encoder_hidden_states.dtype,			SCREAMING_SNAKE_CASE_,			SCREAMING_SNAKE_CASE_,			SCREAMING_SNAKE_CASE_,			self.scheduler,			)

          for i, t in enumerate(self.progress_bar(SCREAMING_SNAKE_CASE_       )       ):
             # expand the latents if we are doing classifier free guidance
             UpperCamelCase    :  List[Any]							=	torch.cat([latents] * 2       ) if do_classifier_free_guidance else latents

             UpperCamelCase    :  int							=	{'text_embeds': prompt_embeds, 'image_embeds': image_embeds}
             UpperCamelCase    :  Optional[int]							=	self.unet(
                 sample=SCREAMING_SNAKE_CASE_,			timestep=SCREAMING_SNAKE_CASE_,			encoder_hidden_states=SCREAMING_SNAKE_CASE_,			added_cond_kwargs=SCREAMING_SNAKE_CASE_,			return_dict=SCREAMING_SNAKE_CASE_,			)[0]

             if do_classifier_free_guidance:
                UpperCamelCase	,     UpperCamelCase    :  List[str]							=	noise_pred.split(latents.shape[1],			dim=1       )
                UpperCamelCase	,     UpperCamelCase    :  Union[str, Any]							=	noise_pred.chunk(2       )
                UpperCamelCase	,     UpperCamelCase    :  Union[str, Any]							=	variance_pred.chunk(2       )
                UpperCamelCase    :  Optional[int]							=	noise_pred_uncond + guidance_scale * (noise_pred_text - noise_pred_uncond)
                UpperCamelCase    :  Any							=	torch.cat([noise_pred, variance_pred_text],			dim=1       )

             if not (
                 hasattr(self.scheduler.config,			'variance_type'       )
                 and self.scheduler.config.variance_type in ["learned", "learned_range"]
             ):
                UpperCamelCase	,     UpperCamelCase    :  List[str]							=	noise_pred.split(latents.shape[1],			dim=1       )

             # compute the previous noisy sample x_t -> x_t-1
             UpperCamelCase    :  Dict							=	self.scheduler.step(
                 SCREAMING_SNAKE_CASE_,			SCREAMING_SNAKE_CASE_,			SCREAMING_SNAKE_CASE_,			generator=SCREAMING_SNAKE_CASE_,			).prev_sample
          # post-processing
          UpperCamelCase    :  Optional[Any]							=	self.movq.decode(SCREAMING_SNAKE_CASE_,			force_not_quantize=SCREAMING_SNAKE_CASE_       )['sample']

          if output_type not in ["pt", "np", "pil"]:
             raise ValueError(F"""Only the output types `pt`, `pil` and `np` are supported not output_type={output_type}"""       )

          if output_type in ["np", "pil"]:
             UpperCamelCase    :  int							=	image * 0.5 + 0.5
             UpperCamelCase    :  Tuple							=	image.clamp(0,			1       )
             UpperCamelCase    :  Dict							=	image.cpu().permute(0,			2,			3,			1       ).float().numpy()

          if output_type == "pil":
             UpperCamelCase    :  Dict							=	self.numpy_to_pil(SCREAMING_SNAKE_CASE_       )

          if not return_dict:
             return (image,)

          return ImagePipelineOutput(images=SCREAMING_SNAKE_CASE_       )








from collections import OrderedDict
from typing import TYPE_CHECKING, Any, List, Mapping, Optional

from packaging import version


if TYPE_CHECKING:
    from ... import PreTrainedTokenizer, TensorType

from ...configuration_utils import PretrainedConfig
from ...onnx import OnnxConfigWithPast, PatchingSpec
from ...utils import is_torch_available, logging


__UpperCAmelCase          =       logging.get_logger(__name__)

__UpperCAmelCase          =       {
    '''bigscience/bloom''': '''https://huggingface.co/bigscience/bloom/resolve/main/config.json''',
    '''bigscience/bloom-560m''': '''https://huggingface.co/bigscience/bloom-560m/blob/main/config.json''',
    '''bigscience/bloom-1b1''': '''https://huggingface.co/bigscience/bloom-1b1/blob/main/config.json''',
    '''bigscience/bloom-1b7''': '''https://huggingface.co/bigscience/bloom-1b7/blob/main/config.json''',
    '''bigscience/bloom-3b''': '''https://huggingface.co/bigscience/bloom-3b/blob/main/config.json''',
    '''bigscience/bloom-7b1''': '''https://huggingface.co/bigscience/bloom-7b1/blob/main/config.json''',
}



class 				lowerCAmelCase_		(  a__							):
       UpperCAmelCase__				:					Tuple        =  "bloom"
       UpperCAmelCase__				:					List[Any]        =  ["past_key_values"]
       UpperCAmelCase__				:					str        =  {
           "num_hidden_layers": "n_layer",
           "num_attention_heads": "n_head",
       }
       def __init__(			self,			SCREAMING_SNAKE_CASE_=25_0880,			SCREAMING_SNAKE_CASE_=64,			SCREAMING_SNAKE_CASE_=2,			SCREAMING_SNAKE_CASE_=8,			SCREAMING_SNAKE_CASE_=1e-5,			SCREAMING_SNAKE_CASE_=0.02,			SCREAMING_SNAKE_CASE_=True,			SCREAMING_SNAKE_CASE_=1,			SCREAMING_SNAKE_CASE_=2,			SCREAMING_SNAKE_CASE_=False,			SCREAMING_SNAKE_CASE_=0.0,			SCREAMING_SNAKE_CASE_=0.0,			SCREAMING_SNAKE_CASE_=1,			SCREAMING_SNAKE_CASE_=False,			**SCREAMING_SNAKE_CASE_,			)     ->		Tuple:
          UpperCamelCase    :  str							=	vocab_size
          # Backward compatibility with n_embed kwarg
          UpperCamelCase    :  Optional[Any]							=	kwargs.pop('n_embed',			SCREAMING_SNAKE_CASE_       )
          UpperCamelCase    :  Union[str, Any]							=	hidden_size if n_embed is None else n_embed
          UpperCamelCase    :  Tuple							=	n_layer
          UpperCamelCase    :  Dict							=	n_head
          UpperCamelCase    :  List[Any]							=	layer_norm_epsilon
          UpperCamelCase    :  Optional[int]							=	initializer_range
          UpperCamelCase    :  int							=	use_cache
          UpperCamelCase    :  int							=	pretraining_tp
          UpperCamelCase    :  Optional[int]							=	apply_residual_connection_post_layernorm
          UpperCamelCase    :  str							=	hidden_dropout
          UpperCamelCase    :  str							=	attention_dropout

          UpperCamelCase    :  List[Any]							=	bos_token_id
          UpperCamelCase    :  Tuple							=	eos_token_id
          UpperCamelCase    :  Union[str, Any]							=	slow_but_exact

          super().__init__(bos_token_id=SCREAMING_SNAKE_CASE_,			eos_token_id=SCREAMING_SNAKE_CASE_,			**SCREAMING_SNAKE_CASE_       )






class 				lowerCAmelCase_		(  a__							):
       UpperCAmelCase__				:					Any        =  version.parse("1.12"							)
       def __init__(			self,			SCREAMING_SNAKE_CASE_,			SCREAMING_SNAKE_CASE_ = "default",			SCREAMING_SNAKE_CASE_ = None,			SCREAMING_SNAKE_CASE_ = False,			)     ->		Any:
          super().__init__(SCREAMING_SNAKE_CASE_,			task=SCREAMING_SNAKE_CASE_,			patching_specs=SCREAMING_SNAKE_CASE_,			use_past=SCREAMING_SNAKE_CASE_       )
          if not getattr(self._config,			'pad_token_id',			SCREAMING_SNAKE_CASE_       ):
             # TODO: how to do that better?
             UpperCamelCase    :  Tuple							=	0
       @property
       def 					snake_case_       (			self       )     ->		Mapping[str, Mapping[int, str]]:
          UpperCamelCase    :  str							=	OrderedDict({'input_ids': {0: 'batch', 1: 'sequence'}}       )
          if self.use_past:
             # BLOOM stores values on dynamic axis 2. For more details see: https://github.com/huggingface/transformers/pull/18344
             self.fill_with_past_key_values_(SCREAMING_SNAKE_CASE_,			direction='inputs',			inverted_values_shape=SCREAMING_SNAKE_CASE_       )
             UpperCamelCase    :  Union[str, Any]							=	{0: 'batch', 1: 'past_sequence + sequence'}
          else:
             UpperCamelCase    :  Optional[int]							=	{0: 'batch', 1: 'sequence'}

          return common_inputs
       @property
       def 					snake_case_       (			self       )     ->		int:
          return self._config.n_layer
       @property
       def 					snake_case_       (			self       )     ->		int:
          return self._config.n_head
       @property
       def 					snake_case_       (			self       )     ->		float:
          return 1e-3
       def 					snake_case_       (			self,			SCREAMING_SNAKE_CASE_,			SCREAMING_SNAKE_CASE_ = -1,			SCREAMING_SNAKE_CASE_ = -1,			SCREAMING_SNAKE_CASE_ = False,			SCREAMING_SNAKE_CASE_ = None,			)     ->		Mapping[str, Any]:
          UpperCamelCase    :  Dict							=	super(SCREAMING_SNAKE_CASE_,			self       ).generate_dummy_inputs(
              SCREAMING_SNAKE_CASE_,			batch_size=SCREAMING_SNAKE_CASE_,			seq_length=SCREAMING_SNAKE_CASE_,			is_pair=SCREAMING_SNAKE_CASE_,			framework=SCREAMING_SNAKE_CASE_       )

          # We need to order the input in the way they appears in the forward()
          UpperCamelCase    :  Any							=	OrderedDict({'input_ids': common_inputs['input_ids']}       )

          # Need to add the past_keys
          if self.use_past:
             if not is_torch_available():
                raise ValueError('Cannot generate dummy past_keys inputs without PyTorch installed.'       )
             else:
                import torch

                UpperCamelCase	,     UpperCamelCase    :  int							=	common_inputs['input_ids'].shape
                # Not using the same length for past_key_values
                UpperCamelCase    :  Any							=	seqlen + 2
                UpperCamelCase    :  Optional[int]							=	self._config.hidden_size // self.num_attention_heads
                UpperCamelCase    :  Any							=	(
                    batch * self.num_attention_heads,
                    head_dim,
                    past_key_values_length,
                )
                UpperCamelCase    :  Optional[Any]							=	(
                    batch * self.num_attention_heads,
                    past_key_values_length,
                    head_dim,
                )
                UpperCamelCase    :  List[str]							=	[
                    (torch.zeros(SCREAMING_SNAKE_CASE_       ), torch.zeros(SCREAMING_SNAKE_CASE_       )) for _ in range(self.num_layers       )
                ]

          UpperCamelCase    :  str							=	common_inputs['attention_mask']
          if self.use_past:
             UpperCamelCase    :  int							=	ordered_inputs['attention_mask'].dtype
             UpperCamelCase    :  List[Any]							=	torch.cat(
                 [ordered_inputs['attention_mask'], torch.ones(SCREAMING_SNAKE_CASE_,			SCREAMING_SNAKE_CASE_,			dtype=SCREAMING_SNAKE_CASE_       )],			dim=1       )

          return ordered_inputs






       @property
       def 					snake_case_       (			self       )     ->		int:
          return 13



