





# This model implementation is heavily inspired by https://github.com/haofanwang/ControlNet-for-Diffusers/

import gc
import random
import tempfile
import unittest

import numpy as np
import torch
from PIL import Image
from transformers import CLIPTextConfig, CLIPTextModel, CLIPTokenizer

from diffusers import (
    AutoencoderKL,
    ControlNetModel,
    DDIMScheduler,
    StableDiffusionControlNetImgaImgPipeline,
    UNetaDConditionModel,
)
from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion_controlnet import MultiControlNetModel
from diffusers.utils import floats_tensor, load_image, load_numpy, randn_tensor, slow, torch_device
from diffusers.utils.import_utils import is_xformers_available
from diffusers.utils.testing_utils import enable_full_determinism, require_torch_gpu

from ..pipeline_params import (
    IMAGE_TO_IMAGE_IMAGE_PARAMS,
    TEXT_GUIDED_IMAGE_VARIATION_BATCH_PARAMS,
    TEXT_GUIDED_IMAGE_VARIATION_PARAMS,
)
from ..test_pipelines_common import (
    PipelineKarrasSchedulerTesterMixin,
    PipelineLatentTesterMixin,
    PipelineTesterMixin,
)


enable_full_determinism()

class 		UpperCamelCase__ (      lowerCAmelCase_							,      lowerCAmelCase_							,      lowerCAmelCase_							,      unittest.TestCase   ):



     '''simple docstring'''

     __snake_case    :	Optional[Any]			=			StableDiffusionControlNetImgaImgPipeline
     __snake_case    :	List[str]			=			TEXT_GUIDED_IMAGE_VARIATION_PARAMS - {"height", "width"}
     __snake_case    :	Optional[int]			=			TEXT_GUIDED_IMAGE_VARIATION_BATCH_PARAMS
     __snake_case    :	str			=			IMAGE_TO_IMAGE_IMAGE_PARAMS.union({"control_image"}   )
     __snake_case    :	Dict			=			IMAGE_TO_IMAGE_IMAGE_PARAMS






     def 				SCREAMING_SNAKE_CASE__  (							self						:    Optional[Any]       )     ->		Union[str, Any]:




           '''simple docstring'''


           torch.manual_seed(0       )
           SCREAMING_SNAKE_CASE		=						UNetaDConditionModel(
               block_out_channels=(32, 64)						,layers_per_block=2						,sample_size=32						,in_channels=4						,out_channels=4						,down_block_types=("""DownBlock2D""", """CrossAttnDownBlock2D""")						,up_block_types=("""CrossAttnUpBlock2D""", """UpBlock2D""")						,cross_attention_dim=32						,)
           torch.manual_seed(0       )
           SCREAMING_SNAKE_CASE		=						ControlNetModel(
               block_out_channels=(32, 64)						,layers_per_block=2						,in_channels=4						,down_block_types=("""DownBlock2D""", """CrossAttnDownBlock2D""")						,cross_attention_dim=32						,conditioning_embedding_out_channels=(16, 32)						,)
           torch.manual_seed(0       )
           SCREAMING_SNAKE_CASE		=						DDIMScheduler(
               beta_start=0.00085						,beta_end=0.012						,beta_schedule="""scaled_linear"""						,clip_sample=lowerCamelCase__						,set_alpha_to_one=lowerCamelCase__						,)
           torch.manual_seed(0       )
           SCREAMING_SNAKE_CASE		=						AutoencoderKL(
               block_out_channels=[32, 64]						,in_channels=3						,out_channels=3						,down_block_types=["""DownEncoderBlock2D""", """DownEncoderBlock2D"""]						,up_block_types=["""UpDecoderBlock2D""", """UpDecoderBlock2D"""]						,latent_channels=4						,)
           torch.manual_seed(0       )
           SCREAMING_SNAKE_CASE		=						CLIPTextConfig(
               bos_token_id=0						,eos_token_id=2						,hidden_size=32						,intermediate_size=37						,layer_norm_eps=1e-0_5						,num_attention_heads=4						,num_hidden_layers=5						,pad_token_id=1						,vocab_size=1000						,)
           SCREAMING_SNAKE_CASE		=						CLIPTextModel(lowerCamelCase__       )
           SCREAMING_SNAKE_CASE		=						CLIPTokenizer.from_pretrained("""hf-internal-testing/tiny-random-clip"""       )

           SCREAMING_SNAKE_CASE		=						{
               """unet""": unet,
               """controlnet""": controlnet,
               """scheduler""": scheduler,
               """vae""": vae,
               """text_encoder""": text_encoder,
               """tokenizer""": tokenizer,
               """safety_checker""": None,
               """feature_extractor""": None,
           }
           return components






     def 				SCREAMING_SNAKE_CASE__  (							self						:    List[str]						,lowerCamelCase__						:    Optional[Any]						,lowerCamelCase__						:    str=0       )     ->		Any:




           '''simple docstring'''


           if str(lowerCamelCase__       ).startswith("""mps"""       ):
                 SCREAMING_SNAKE_CASE		=						torch.manual_seed(lowerCamelCase__       )
           else:
                 SCREAMING_SNAKE_CASE		=						torch.Generator(device=lowerCamelCase__       ).manual_seed(lowerCamelCase__       )

           SCREAMING_SNAKE_CASE		=						2
           SCREAMING_SNAKE_CASE		=						randn_tensor(
               (1, 3, 32 * controlnet_embedder_scale_factor, 32 * controlnet_embedder_scale_factor)						,generator=lowerCamelCase__						,device=torch.device(lowerCamelCase__       )						,)
           SCREAMING_SNAKE_CASE		=						floats_tensor(control_image.shape						,rng=random.Random(lowerCamelCase__       )       ).to(lowerCamelCase__       )
           SCREAMING_SNAKE_CASE		=						image.cpu().permute(0						,2						,3						,1       )[0]
           SCREAMING_SNAKE_CASE		=						Image.fromarray(np.uinta(lowerCamelCase__       )       ).convert("""RGB"""       ).resize((64, 64)       )
           SCREAMING_SNAKE_CASE		=						{
               """prompt""": """A painting of a squirrel eating a burger""",
               """generator""": generator,
               """num_inference_steps""": 2,
               """guidance_scale""": 6.0,
               """output_type""": """numpy""",
               """image""": image,
               """control_image""": control_image,
           }

           return inputs






     def 				SCREAMING_SNAKE_CASE__  (							self						:    int       )     ->		Optional[int]:




           '''simple docstring'''


           return self._test_attention_slicing_forward_pass(expected_max_diff=2e-3       )






     @unittest.skipIf(
         torch_device != """cuda""" or not is_xformers_available()						,reason="""XFormers attention is only available with CUDA and `xformers` installed"""						,)
     def 				SCREAMING_SNAKE_CASE__  (							self						:    Tuple       )     ->		Union[str, Any]:




           '''simple docstring'''


           self._test_xformers_attention_forwardGenerator_pass(expected_max_diff=2e-3       )






     def 				SCREAMING_SNAKE_CASE__  (							self						:    Dict       )     ->		Optional[Any]:




           '''simple docstring'''


           self._test_inference_batch_single_identical(expected_max_diff=2e-3       )

class 		UpperCamelCase__ (      lowerCAmelCase_							,      lowerCAmelCase_							,      unittest.TestCase   ):



     '''simple docstring'''

     __snake_case    :	Optional[Any]			=			StableDiffusionControlNetImgaImgPipeline
     __snake_case    :	int			=			TEXT_GUIDED_IMAGE_VARIATION_PARAMS - {"height", "width"}
     __snake_case    :	List[str]			=			TEXT_GUIDED_IMAGE_VARIATION_BATCH_PARAMS
     __snake_case    :	Dict			=			frozenset([]   )  # TO_DO: add image_params once refactored VaeImageProcessor.preprocess






     def 				SCREAMING_SNAKE_CASE__  (							self						:    int       )     ->		Optional[Any]:




           '''simple docstring'''


           torch.manual_seed(0       )
           SCREAMING_SNAKE_CASE		=						UNetaDConditionModel(
               block_out_channels=(32, 64)						,layers_per_block=2						,sample_size=32						,in_channels=4						,out_channels=4						,down_block_types=("""DownBlock2D""", """CrossAttnDownBlock2D""")						,up_block_types=("""CrossAttnUpBlock2D""", """UpBlock2D""")						,cross_attention_dim=32						,)
           torch.manual_seed(0       )

           def init_weights(lowerCamelCase__						:    List[str]       ):
                 if isinstance(lowerCamelCase__						,torch.nn.Convad       ):
                       torch.nn.init.normal(m.weight       )
                       m.bias.data.fill_(1.0       )

           SCREAMING_SNAKE_CASE		=						ControlNetModel(
               block_out_channels=(32, 64)						,layers_per_block=2						,in_channels=4						,down_block_types=("""DownBlock2D""", """CrossAttnDownBlock2D""")						,cross_attention_dim=32						,conditioning_embedding_out_channels=(16, 32)						,)
           controlneta.controlnet_down_blocks.apply(lowerCamelCase__       )

           torch.manual_seed(0       )
           SCREAMING_SNAKE_CASE		=						ControlNetModel(
               block_out_channels=(32, 64)						,layers_per_block=2						,in_channels=4						,down_block_types=("""DownBlock2D""", """CrossAttnDownBlock2D""")						,cross_attention_dim=32						,conditioning_embedding_out_channels=(16, 32)						,)
           controlneta.controlnet_down_blocks.apply(lowerCamelCase__       )

           torch.manual_seed(0       )
           SCREAMING_SNAKE_CASE		=						DDIMScheduler(
               beta_start=0.00085						,beta_end=0.012						,beta_schedule="""scaled_linear"""						,clip_sample=lowerCamelCase__						,set_alpha_to_one=lowerCamelCase__						,)
           torch.manual_seed(0       )
           SCREAMING_SNAKE_CASE		=						AutoencoderKL(
               block_out_channels=[32, 64]						,in_channels=3						,out_channels=3						,down_block_types=["""DownEncoderBlock2D""", """DownEncoderBlock2D"""]						,up_block_types=["""UpDecoderBlock2D""", """UpDecoderBlock2D"""]						,latent_channels=4						,)
           torch.manual_seed(0       )
           SCREAMING_SNAKE_CASE		=						CLIPTextConfig(
               bos_token_id=0						,eos_token_id=2						,hidden_size=32						,intermediate_size=37						,layer_norm_eps=1e-0_5						,num_attention_heads=4						,num_hidden_layers=5						,pad_token_id=1						,vocab_size=1000						,)
           SCREAMING_SNAKE_CASE		=						CLIPTextModel(lowerCamelCase__       )
           SCREAMING_SNAKE_CASE		=						CLIPTokenizer.from_pretrained("""hf-internal-testing/tiny-random-clip"""       )

           SCREAMING_SNAKE_CASE		=						MultiControlNetModel([controlneta, controlneta]       )

           SCREAMING_SNAKE_CASE		=						{
               """unet""": unet,
               """controlnet""": controlnet,
               """scheduler""": scheduler,
               """vae""": vae,
               """text_encoder""": text_encoder,
               """tokenizer""": tokenizer,
               """safety_checker""": None,
               """feature_extractor""": None,
           }
           return components






     def 				SCREAMING_SNAKE_CASE__  (							self						:    Tuple						,lowerCamelCase__						:    Optional[Any]						,lowerCamelCase__						:    List[Any]=0       )     ->		Union[str, Any]:




           '''simple docstring'''


           if str(lowerCamelCase__       ).startswith("""mps"""       ):
                 SCREAMING_SNAKE_CASE		=						torch.manual_seed(lowerCamelCase__       )
           else:
                 SCREAMING_SNAKE_CASE		=						torch.Generator(device=lowerCamelCase__       ).manual_seed(lowerCamelCase__       )

           SCREAMING_SNAKE_CASE		=						2

           SCREAMING_SNAKE_CASE		=						[
               randn_tensor(
                   (1, 3, 32 * controlnet_embedder_scale_factor, 32 * controlnet_embedder_scale_factor)						,generator=lowerCamelCase__						,device=torch.device(lowerCamelCase__       )						,),
               randn_tensor(
                   (1, 3, 32 * controlnet_embedder_scale_factor, 32 * controlnet_embedder_scale_factor)						,generator=lowerCamelCase__						,device=torch.device(lowerCamelCase__       )						,),
           ]

           SCREAMING_SNAKE_CASE		=						floats_tensor(control_image[0].shape						,rng=random.Random(lowerCamelCase__       )       ).to(lowerCamelCase__       )
           SCREAMING_SNAKE_CASE		=						image.cpu().permute(0						,2						,3						,1       )[0]
           SCREAMING_SNAKE_CASE		=						Image.fromarray(np.uinta(lowerCamelCase__       )       ).convert("""RGB"""       ).resize((64, 64)       )
           SCREAMING_SNAKE_CASE		=						{
               """prompt""": """A painting of a squirrel eating a burger""",
               """generator""": generator,
               """num_inference_steps""": 2,
               """guidance_scale""": 6.0,
               """output_type""": """numpy""",
               """image""": image,
               """control_image""": control_image,
           }

           return inputs






     def 				SCREAMING_SNAKE_CASE__  (							self						:    Tuple       )     ->		Optional[Any]:




           '''simple docstring'''


           SCREAMING_SNAKE_CASE		=						self.get_dummy_components()
           SCREAMING_SNAKE_CASE		=						self.pipeline_class(**lowerCamelCase__       )
           pipe.to(lowerCamelCase__       )

           SCREAMING_SNAKE_CASE		=						10.0
           SCREAMING_SNAKE_CASE		=						4

           SCREAMING_SNAKE_CASE		=						self.get_dummy_inputs(lowerCamelCase__       )
           SCREAMING_SNAKE_CASE		=						steps
           SCREAMING_SNAKE_CASE		=						scale
           SCREAMING_SNAKE_CASE		=						pipe(**lowerCamelCase__       )[0]

           SCREAMING_SNAKE_CASE		=						self.get_dummy_inputs(lowerCamelCase__       )
           SCREAMING_SNAKE_CASE		=						steps
           SCREAMING_SNAKE_CASE		=						scale
           SCREAMING_SNAKE_CASE		=						pipe(**lowerCamelCase__						,control_guidance_start=0.1						,control_guidance_end=0.2       )[0]

           SCREAMING_SNAKE_CASE		=						self.get_dummy_inputs(lowerCamelCase__       )
           SCREAMING_SNAKE_CASE		=						steps
           SCREAMING_SNAKE_CASE		=						scale
           SCREAMING_SNAKE_CASE		=						pipe(**lowerCamelCase__						,control_guidance_start=[0.1, 0.3]						,control_guidance_end=[0.2, 0.7]       )[0]

           SCREAMING_SNAKE_CASE		=						self.get_dummy_inputs(lowerCamelCase__       )
           SCREAMING_SNAKE_CASE		=						steps
           SCREAMING_SNAKE_CASE		=						scale
           SCREAMING_SNAKE_CASE		=						pipe(**lowerCamelCase__						,control_guidance_start=0.4						,control_guidance_end=[0.5, 0.8]       )[0]

           # make sure that all outputs are different
           assert np.sum(np.abs(output_a - output_a       )       ) > 1e-3
           assert np.sum(np.abs(output_a - output_a       )       ) > 1e-3
           assert np.sum(np.abs(output_a - output_a       )       ) > 1e-3






     def 				SCREAMING_SNAKE_CASE__  (							self						:    Any       )     ->		Any:




           '''simple docstring'''


           return self._test_attention_slicing_forward_pass(expected_max_diff=2e-3       )






     @unittest.skipIf(
         torch_device != """cuda""" or not is_xformers_available()						,reason="""XFormers attention is only available with CUDA and `xformers` installed"""						,)
     def 				SCREAMING_SNAKE_CASE__  (							self						:    int       )     ->		Dict:




           '''simple docstring'''


           self._test_xformers_attention_forwardGenerator_pass(expected_max_diff=2e-3       )






     def 				SCREAMING_SNAKE_CASE__  (							self						:    Union[str, Any]       )     ->		Union[str, Any]:




           '''simple docstring'''


           self._test_inference_batch_single_identical(expected_max_diff=2e-3       )






     def 				SCREAMING_SNAKE_CASE__  (							self						:    Optional[Any]       )     ->		List[str]:




           '''simple docstring'''


           SCREAMING_SNAKE_CASE		=						self.get_dummy_components()
           SCREAMING_SNAKE_CASE		=						self.pipeline_class(**lowerCamelCase__       )
           pipe.to(lowerCamelCase__       )
           pipe.set_progress_bar_config(disable=lowerCamelCase__       )
           with tempfile.TemporaryDirectory() as tmpdir:
                 try:
                       # save_pretrained is not implemented for Multi-ControlNet
                       pipe.save_pretrained(lowerCamelCase__       )
                 except NotImplementedError:
                       pass

@slow
@require_torch_gpu
class 		UpperCamelCase__ (      unittest.TestCase   ):



     '''simple docstring'''

     def 				SCREAMING_SNAKE_CASE__  (							self						:    str       )     ->		Optional[Any]:




           '''simple docstring'''


           super().tearDown()
           gc.collect()
           torch.cuda.empty_cache()






     def 				SCREAMING_SNAKE_CASE__  (							self						:    List[str]       )     ->		int:




           '''simple docstring'''


           SCREAMING_SNAKE_CASE		=						ControlNetModel.from_pretrained("""lllyasviel/sd-controlnet-canny"""       )

           SCREAMING_SNAKE_CASE		=						StableDiffusionControlNetImgaImgPipeline.from_pretrained(
               """runwayml/stable-diffusion-v1-5"""						,safety_checker=lowerCamelCase__						,controlnet=lowerCamelCase__       )
           pipe.enable_model_cpu_offload()
           pipe.set_progress_bar_config(disable=lowerCamelCase__       )

           SCREAMING_SNAKE_CASE		=						torch.Generator(device="""cpu"""       ).manual_seed(0       )
           SCREAMING_SNAKE_CASE		=						"""evil space-punk bird"""
           SCREAMING_SNAKE_CASE		=						load_image(
               """https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main/sd_controlnet/bird_canny.png"""       ).resize((512, 512)       )
           SCREAMING_SNAKE_CASE		=						load_image(
               """https://huggingface.co/lllyasviel/sd-controlnet-canny/resolve/main/images/bird.png"""       ).resize((512, 512)       )

           SCREAMING_SNAKE_CASE		=						pipe(
               lowerCamelCase__						,lowerCamelCase__						,control_image=lowerCamelCase__						,generator=lowerCamelCase__						,output_type="""np"""						,num_inference_steps=50						,strength=0.6						,)

           SCREAMING_SNAKE_CASE		=						output.images[0]

           assert image.shape == (512, 512, 3)

           SCREAMING_SNAKE_CASE		=						load_numpy(
               """https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main/sd_controlnet/img2img.npy"""       )

           assert np.abs(expected_image - image       ).max() < 9e-2









from typing import Callable, Optional

from .. import Features
from ..packaged_modules.generator.generator import Generator
from .abc import AbstractDatasetInputStream

class 		UpperCamelCase__ (      lowerCAmelCase_   ):



     '''simple docstring'''

     def __init__(							self						:    Union[str, Any]						,lowerCamelCase__						:    Callable						,lowerCamelCase__						:    Optional[Features] = None						,lowerCamelCase__						:    str = None						,lowerCamelCase__						:    bool = False						,lowerCamelCase__						:    bool = False						,lowerCamelCase__						:    Optional[dict] = None						,lowerCamelCase__						:    Optional[int] = None						,**lowerCamelCase__						:    Optional[Any]						,)     ->		List[str]:




           '''simple docstring'''


           super().__init__(
               features=lowerCamelCase__						,cache_dir=lowerCamelCase__						,keep_in_memory=lowerCamelCase__						,streaming=lowerCamelCase__						,num_proc=lowerCamelCase__						,**lowerCamelCase__						,)
           SCREAMING_SNAKE_CASE		=						Generator(
               cache_dir=lowerCamelCase__						,features=lowerCamelCase__						,generator=lowerCamelCase__						,gen_kwargs=lowerCamelCase__						,**lowerCamelCase__						,)






     def 				SCREAMING_SNAKE_CASE__  (							self						:    Union[str, Any]       )     ->		Optional[Any]:




           '''simple docstring'''


           if self.streaming:
                 SCREAMING_SNAKE_CASE		=						self.builder.as_streaming_dataset(split="""train"""       )
           # Build regular (map-style) dataset
           else:
                 SCREAMING_SNAKE_CASE		=						None
                 SCREAMING_SNAKE_CASE		=						None
                 SCREAMING_SNAKE_CASE		=						None
                 SCREAMING_SNAKE_CASE		=						None

                 self.builder.download_and_prepare(
                     download_config=lowerCamelCase__						,download_mode=lowerCamelCase__						,verification_mode=lowerCamelCase__						,base_path=lowerCamelCase__						,num_proc=self.num_proc						,)
                 SCREAMING_SNAKE_CASE		=						self.builder.as_dataset(
                     split="""train"""						,verification_mode=lowerCamelCase__						,in_memory=self.keep_in_memory       )
           return dataset


