zhiweili commited on
Commit
f0a547a
1 Parent(s): 02c0c4b

change inppaint

Browse files
app.py CHANGED
@@ -1,6 +1,6 @@
1
  import gradio as gr
2
 
3
- from app_text2img import create_demo as create_demo_haircolor
4
 
5
  with gr.Blocks(css="style.css") as demo:
6
  with gr.Tabs():
 
1
  import gradio as gr
2
 
3
+ from app_haircolor_inpaint import create_demo as create_demo_haircolor
4
 
5
  with gr.Blocks(css="style.css") as demo:
6
  with gr.Tabs():
app_haircolor_inpaint.py CHANGED
@@ -19,8 +19,8 @@ from controlnet_aux import (
19
  CannyDetector,
20
  )
21
 
22
- BASE_MODEL = "SG161222/RealVisXL_V5.0_Lightning"
23
- # BASE_MODEL = "RunDiffusion/Juggernaut-XL-v9"
24
  DEVICE = "cuda" if torch.cuda.is_available() else "cpu"
25
 
26
  DEFAULT_EDIT_PROMPT = "a woman, blue hair, high detailed"
@@ -54,7 +54,7 @@ basepipeline = DiffusionPipeline.from_pretrained(
54
  torch_dtype=torch.float16,
55
  use_safetensors=True,
56
  adapter=adapters,
57
- custom_pipeline="./pipelines/pipeline_sdxl_adapter_inpaint.py",
58
  )
59
 
60
  basepipeline = basepipeline.to(DEVICE)
@@ -76,9 +76,9 @@ def image_to_image(
76
  run_task_time = 0
77
  time_cost_str = ''
78
  run_task_time, time_cost_str = get_time_cost(run_task_time, time_cost_str)
79
- lineart_image = lineart_detector(input_image, 384, generate_size)
80
  run_task_time, time_cost_str = get_time_cost(run_task_time, time_cost_str)
81
- canny_image = canndy_detector(input_image, 384, generate_size)
82
  run_task_time, time_cost_str = get_time_cost(run_task_time, time_cost_str)
83
 
84
  cond_image = [lineart_image, canny_image]
 
19
  CannyDetector,
20
  )
21
 
22
+ BASE_MODEL = "stabilityai/sdxl-turbo"
23
+
24
  DEVICE = "cuda" if torch.cuda.is_available() else "cpu"
25
 
26
  DEFAULT_EDIT_PROMPT = "a woman, blue hair, high detailed"
 
54
  torch_dtype=torch.float16,
55
  use_safetensors=True,
56
  adapter=adapters,
57
+ custom_pipeline="./pipelines/pipeline_sdxl_adapter_inpaint_custom.py",
58
  )
59
 
60
  basepipeline = basepipeline.to(DEVICE)
 
76
  run_task_time = 0
77
  time_cost_str = ''
78
  run_task_time, time_cost_str = get_time_cost(run_task_time, time_cost_str)
79
+ lineart_image = lineart_detector(input_image, int(generate_size*0.375), generate_size)
80
  run_task_time, time_cost_str = get_time_cost(run_task_time, time_cost_str)
81
+ canny_image = canndy_detector(input_image, int(generate_size*0.375), generate_size)
82
  run_task_time, time_cost_str = get_time_cost(run_task_time, time_cost_str)
83
 
84
  cond_image = [lineart_image, canny_image]
pipelines/pipeline_sdxl_adapter_inpaint_custom.py ADDED
@@ -0,0 +1,1861 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Copyright 2024 The HuggingFace Team. All rights reserved.
2
+ #
3
+ # Licensed under the Apache License, Version 2.0 (the "License");
4
+ # you may not use this file except in compliance with the License.
5
+ # You may obtain a copy of the License at
6
+ #
7
+ # http://www.apache.org/licenses/LICENSE-2.0
8
+ #
9
+ # Unless required by applicable law or agreed to in writing, software
10
+ # distributed under the License is distributed on an "AS IS" BASIS,
11
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+ # See the License for the specific language governing permissions and
13
+ # limitations under the License.
14
+
15
+ import inspect
16
+ from typing import Any, Callable, Dict, List, Optional, Tuple, Union
17
+
18
+ import numpy as np
19
+ import PIL.Image
20
+ import torch
21
+ from transformers import (
22
+ CLIPImageProcessor,
23
+ CLIPTextModel,
24
+ CLIPTextModelWithProjection,
25
+ CLIPTokenizer,
26
+ CLIPVisionModelWithProjection,
27
+ )
28
+
29
+ from diffusers.callbacks import (
30
+ MultiPipelineCallbacks,
31
+ PipelineCallback,
32
+ )
33
+
34
+ from diffusers.image_processor import (
35
+ PipelineImageInput,
36
+ VaeImageProcessor,
37
+ )
38
+
39
+ from diffusers.loaders import (
40
+ FromSingleFileMixin,
41
+ IPAdapterMixin,
42
+ StableDiffusionXLLoraLoaderMixin,
43
+ TextualInversionLoaderMixin,
44
+ )
45
+
46
+ from diffusers.models import (
47
+ AutoencoderKL,
48
+ ImageProjection,
49
+ MultiAdapter,
50
+ T2IAdapter,
51
+ UNet2DConditionModel,
52
+ )
53
+
54
+ from diffusers.models.attention_processor import (
55
+ AttnProcessor2_0,
56
+ XFormersAttnProcessor,
57
+ )
58
+
59
+ from diffusers.models.lora import (
60
+ adjust_lora_scale_text_encoder,
61
+ )
62
+
63
+ from diffusers.schedulers import (
64
+ KarrasDiffusionSchedulers,
65
+ )
66
+
67
+ from diffusers.utils import (
68
+ PIL_INTERPOLATION,
69
+ USE_PEFT_BACKEND,
70
+ deprecate,
71
+ is_invisible_watermark_available,
72
+ is_torch_xla_available,
73
+ logging,
74
+ replace_example_docstring,
75
+ scale_lora_layers,
76
+ unscale_lora_layers,
77
+ )
78
+
79
+ from diffusers.utils.torch_utils import (
80
+ randn_tensor,
81
+ )
82
+
83
+ from diffusers.pipelines.pipeline_utils import (
84
+ DiffusionPipeline,
85
+ StableDiffusionMixin,
86
+ )
87
+
88
+ from diffusers.pipelines.stable_diffusion_xl.pipeline_output import (
89
+ StableDiffusionXLPipelineOutput,
90
+ )
91
+
92
+ if is_invisible_watermark_available():
93
+ from diffusers.pipelines.stable_diffusion_xl.watermark import (
94
+ StableDiffusionXLWatermarker,
95
+ )
96
+
97
+ if is_torch_xla_available():
98
+ import torch_xla.core.xla_model as xm
99
+
100
+ XLA_AVAILABLE = True
101
+ else:
102
+ XLA_AVAILABLE = False
103
+
104
+
105
+ logger = logging.get_logger(__name__) # pylint: disable=invalid-name
106
+
107
+
108
+ EXAMPLE_DOC_STRING = """
109
+ Examples:
110
+ ```py
111
+ >>> import torch
112
+ >>> from diffusers import StableDiffusionXLInpaintPipeline
113
+ >>> from diffusers.utils import load_image
114
+
115
+ >>> pipe = StableDiffusionXLInpaintPipeline.from_pretrained(
116
+ ... "stabilityai/stable-diffusion-xl-base-1.0",
117
+ ... torch_dtype=torch.float16,
118
+ ... variant="fp16",
119
+ ... use_safetensors=True,
120
+ ... )
121
+ >>> pipe.to("cuda")
122
+
123
+ >>> img_url = "https://raw.githubusercontent.com/CompVis/latent-diffusion/main/data/inpainting_examples/overture-creations-5sI6fQgYIuo.png"
124
+ >>> mask_url = "https://raw.githubusercontent.com/CompVis/latent-diffusion/main/data/inpainting_examples/overture-creations-5sI6fQgYIuo_mask.png"
125
+
126
+ >>> init_image = load_image(img_url).convert("RGB")
127
+ >>> mask_image = load_image(mask_url).convert("RGB")
128
+
129
+ >>> prompt = "A majestic tiger sitting on a bench"
130
+ >>> image = pipe(
131
+ ... prompt=prompt, image=init_image, mask_image=mask_image, num_inference_steps=50, strength=0.80
132
+ ... ).images[0]
133
+ ```
134
+ """
135
+
136
+ def _preprocess_adapter_image(image, height, width):
137
+ if isinstance(image, torch.Tensor):
138
+ return image
139
+ elif isinstance(image, PIL.Image.Image):
140
+ image = [image]
141
+
142
+ if isinstance(image[0], PIL.Image.Image):
143
+ image = [np.array(i.resize((width, height), resample=PIL_INTERPOLATION["lanczos"])) for i in image]
144
+ image = [
145
+ i[None, ..., None] if i.ndim == 2 else i[None, ...] for i in image
146
+ ] # expand [h, w] or [h, w, c] to [b, h, w, c]
147
+ image = np.concatenate(image, axis=0)
148
+ image = np.array(image).astype(np.float32) / 255.0
149
+ image = image.transpose(0, 3, 1, 2)
150
+ image = torch.from_numpy(image)
151
+ elif isinstance(image[0], torch.Tensor):
152
+ if image[0].ndim == 3:
153
+ image = torch.stack(image, dim=0)
154
+ elif image[0].ndim == 4:
155
+ image = torch.cat(image, dim=0)
156
+ else:
157
+ raise ValueError(
158
+ f"Invalid image tensor! Expecting image tensor with 3 or 4 dimension, but recive: {image[0].ndim}"
159
+ )
160
+ return image
161
+
162
+
163
+ # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.rescale_noise_cfg
164
+ def rescale_noise_cfg(noise_cfg, noise_pred_text, guidance_rescale=0.0):
165
+ """
166
+ Rescale `noise_cfg` according to `guidance_rescale`. Based on findings of [Common Diffusion Noise Schedules and
167
+ Sample Steps are Flawed](https://arxiv.org/pdf/2305.08891.pdf). See Section 3.4
168
+ """
169
+ std_text = noise_pred_text.std(dim=list(range(1, noise_pred_text.ndim)), keepdim=True)
170
+ std_cfg = noise_cfg.std(dim=list(range(1, noise_cfg.ndim)), keepdim=True)
171
+ # rescale the results from guidance (fixes overexposure)
172
+ noise_pred_rescaled = noise_cfg * (std_text / std_cfg)
173
+ # mix with the original results from guidance by factor guidance_rescale to avoid "plain looking" images
174
+ noise_cfg = guidance_rescale * noise_pred_rescaled + (1 - guidance_rescale) * noise_cfg
175
+ return noise_cfg
176
+
177
+
178
+ def mask_pil_to_torch(mask, height, width):
179
+ # preprocess mask
180
+ if isinstance(mask, (PIL.Image.Image, np.ndarray)):
181
+ mask = [mask]
182
+
183
+ if isinstance(mask, list) and isinstance(mask[0], PIL.Image.Image):
184
+ mask = [i.resize((width, height), resample=PIL.Image.LANCZOS) for i in mask]
185
+ mask = np.concatenate([np.array(m.convert("L"))[None, None, :] for m in mask], axis=0)
186
+ mask = mask.astype(np.float32) / 255.0
187
+ elif isinstance(mask, list) and isinstance(mask[0], np.ndarray):
188
+ mask = np.concatenate([m[None, None, :] for m in mask], axis=0)
189
+
190
+ mask = torch.from_numpy(mask)
191
+ return mask
192
+
193
+
194
+ # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion_img2img.retrieve_latents
195
+ def retrieve_latents(
196
+ encoder_output: torch.Tensor, generator: Optional[torch.Generator] = None, sample_mode: str = "sample"
197
+ ):
198
+ if hasattr(encoder_output, "latent_dist") and sample_mode == "sample":
199
+ return encoder_output.latent_dist.sample(generator)
200
+ elif hasattr(encoder_output, "latent_dist") and sample_mode == "argmax":
201
+ return encoder_output.latent_dist.mode()
202
+ elif hasattr(encoder_output, "latents"):
203
+ return encoder_output.latents
204
+ else:
205
+ raise AttributeError("Could not access latents of provided encoder_output")
206
+
207
+
208
+ # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.retrieve_timesteps
209
+ def retrieve_timesteps(
210
+ scheduler,
211
+ num_inference_steps: Optional[int] = None,
212
+ device: Optional[Union[str, torch.device]] = None,
213
+ timesteps: Optional[List[int]] = None,
214
+ sigmas: Optional[List[float]] = None,
215
+ **kwargs,
216
+ ):
217
+ """
218
+ Calls the scheduler's `set_timesteps` method and retrieves timesteps from the scheduler after the call. Handles
219
+ custom timesteps. Any kwargs will be supplied to `scheduler.set_timesteps`.
220
+
221
+ Args:
222
+ scheduler (`SchedulerMixin`):
223
+ The scheduler to get timesteps from.
224
+ num_inference_steps (`int`):
225
+ The number of diffusion steps used when generating samples with a pre-trained model. If used, `timesteps`
226
+ must be `None`.
227
+ device (`str` or `torch.device`, *optional*):
228
+ The device to which the timesteps should be moved to. If `None`, the timesteps are not moved.
229
+ timesteps (`List[int]`, *optional*):
230
+ Custom timesteps used to override the timestep spacing strategy of the scheduler. If `timesteps` is passed,
231
+ `num_inference_steps` and `sigmas` must be `None`.
232
+ sigmas (`List[float]`, *optional*):
233
+ Custom sigmas used to override the timestep spacing strategy of the scheduler. If `sigmas` is passed,
234
+ `num_inference_steps` and `timesteps` must be `None`.
235
+
236
+ Returns:
237
+ `Tuple[torch.Tensor, int]`: A tuple where the first element is the timestep schedule from the scheduler and the
238
+ second element is the number of inference steps.
239
+ """
240
+ if timesteps is not None and sigmas is not None:
241
+ raise ValueError("Only one of `timesteps` or `sigmas` can be passed. Please choose one to set custom values")
242
+ if timesteps is not None:
243
+ accepts_timesteps = "timesteps" in set(inspect.signature(scheduler.set_timesteps).parameters.keys())
244
+ if not accepts_timesteps:
245
+ raise ValueError(
246
+ f"The current scheduler class {scheduler.__class__}'s `set_timesteps` does not support custom"
247
+ f" timestep schedules. Please check whether you are using the correct scheduler."
248
+ )
249
+ scheduler.set_timesteps(timesteps=timesteps, device=device, **kwargs)
250
+ timesteps = scheduler.timesteps
251
+ num_inference_steps = len(timesteps)
252
+ elif sigmas is not None:
253
+ accept_sigmas = "sigmas" in set(inspect.signature(scheduler.set_timesteps).parameters.keys())
254
+ if not accept_sigmas:
255
+ raise ValueError(
256
+ f"The current scheduler class {scheduler.__class__}'s `set_timesteps` does not support custom"
257
+ f" sigmas schedules. Please check whether you are using the correct scheduler."
258
+ )
259
+ scheduler.set_timesteps(sigmas=sigmas, device=device, **kwargs)
260
+ timesteps = scheduler.timesteps
261
+ num_inference_steps = len(timesteps)
262
+ else:
263
+ scheduler.set_timesteps(num_inference_steps, device=device, **kwargs)
264
+ timesteps = scheduler.timesteps
265
+ return timesteps, num_inference_steps
266
+
267
+
268
+ class StableDiffusionXLInpaintPipeline(
269
+ DiffusionPipeline,
270
+ StableDiffusionMixin,
271
+ TextualInversionLoaderMixin,
272
+ StableDiffusionXLLoraLoaderMixin,
273
+ FromSingleFileMixin,
274
+ IPAdapterMixin,
275
+ ):
276
+ r"""
277
+ Pipeline for text-to-image generation using Stable Diffusion XL.
278
+
279
+ This model inherits from [`DiffusionPipeline`]. Check the superclass documentation for the generic methods the
280
+ library implements for all the pipelines (such as downloading or saving, running on a particular device, etc.)
281
+
282
+ The pipeline also inherits the following loading methods:
283
+ - [`~loaders.TextualInversionLoaderMixin.load_textual_inversion`] for loading textual inversion embeddings
284
+ - [`~loaders.FromSingleFileMixin.from_single_file`] for loading `.ckpt` files
285
+ - [`~loaders.StableDiffusionXLLoraLoaderMixin.load_lora_weights`] for loading LoRA weights
286
+ - [`~loaders.StableDiffusionXLLoraLoaderMixin.save_lora_weights`] for saving LoRA weights
287
+ - [`~loaders.IPAdapterMixin.load_ip_adapter`] for loading IP Adapters
288
+
289
+ Args:
290
+ vae ([`AutoencoderKL`]):
291
+ Variational Auto-Encoder (VAE) Model to encode and decode images to and from latent representations.
292
+ text_encoder ([`CLIPTextModel`]):
293
+ Frozen text-encoder. Stable Diffusion XL uses the text portion of
294
+ [CLIP](https://huggingface.co/docs/transformers/model_doc/clip#transformers.CLIPTextModel), specifically
295
+ the [clip-vit-large-patch14](https://huggingface.co/openai/clip-vit-large-patch14) variant.
296
+ text_encoder_2 ([` CLIPTextModelWithProjection`]):
297
+ Second frozen text-encoder. Stable Diffusion XL uses the text and pool portion of
298
+ [CLIP](https://huggingface.co/docs/transformers/model_doc/clip#transformers.CLIPTextModelWithProjection),
299
+ specifically the
300
+ [laion/CLIP-ViT-bigG-14-laion2B-39B-b160k](https://huggingface.co/laion/CLIP-ViT-bigG-14-laion2B-39B-b160k)
301
+ variant.
302
+ tokenizer (`CLIPTokenizer`):
303
+ Tokenizer of class
304
+ [CLIPTokenizer](https://huggingface.co/docs/transformers/v4.21.0/en/model_doc/clip#transformers.CLIPTokenizer).
305
+ tokenizer_2 (`CLIPTokenizer`):
306
+ Second Tokenizer of class
307
+ [CLIPTokenizer](https://huggingface.co/docs/transformers/v4.21.0/en/model_doc/clip#transformers.CLIPTokenizer).
308
+ unet ([`UNet2DConditionModel`]): Conditional U-Net architecture to denoise the encoded image latents.
309
+ scheduler ([`SchedulerMixin`]):
310
+ A scheduler to be used in combination with `unet` to denoise the encoded image latents. Can be one of
311
+ [`DDIMScheduler`], [`LMSDiscreteScheduler`], or [`PNDMScheduler`].
312
+ requires_aesthetics_score (`bool`, *optional*, defaults to `"False"`):
313
+ Whether the `unet` requires a aesthetic_score condition to be passed during inference. Also see the config
314
+ of `stabilityai/stable-diffusion-xl-refiner-1-0`.
315
+ force_zeros_for_empty_prompt (`bool`, *optional*, defaults to `"True"`):
316
+ Whether the negative prompt embeddings shall be forced to always be set to 0. Also see the config of
317
+ `stabilityai/stable-diffusion-xl-base-1-0`.
318
+ add_watermarker (`bool`, *optional*):
319
+ Whether to use the [invisible_watermark library](https://github.com/ShieldMnt/invisible-watermark/) to
320
+ watermark output images. If not defined, it will default to True if the package is installed, otherwise no
321
+ watermarker will be used.
322
+ """
323
+
324
+ model_cpu_offload_seq = "text_encoder->text_encoder_2->image_encoder->unet->vae"
325
+
326
+ _optional_components = [
327
+ "tokenizer",
328
+ "tokenizer_2",
329
+ "text_encoder",
330
+ "text_encoder_2",
331
+ "image_encoder",
332
+ "feature_extractor",
333
+ ]
334
+ _callback_tensor_inputs = [
335
+ "latents",
336
+ "prompt_embeds",
337
+ "negative_prompt_embeds",
338
+ "add_text_embeds",
339
+ "add_time_ids",
340
+ "negative_pooled_prompt_embeds",
341
+ "add_neg_time_ids",
342
+ "mask",
343
+ "masked_image_latents",
344
+ ]
345
+
346
+ def __init__(
347
+ self,
348
+ vae: AutoencoderKL,
349
+ text_encoder: CLIPTextModel,
350
+ text_encoder_2: CLIPTextModelWithProjection,
351
+ tokenizer: CLIPTokenizer,
352
+ tokenizer_2: CLIPTokenizer,
353
+ unet: UNet2DConditionModel,
354
+ adapter: Union[T2IAdapter, MultiAdapter, List[T2IAdapter]],
355
+ scheduler: KarrasDiffusionSchedulers,
356
+ image_encoder: CLIPVisionModelWithProjection = None,
357
+ feature_extractor: CLIPImageProcessor = None,
358
+ requires_aesthetics_score: bool = False,
359
+ force_zeros_for_empty_prompt: bool = True,
360
+ add_watermarker: Optional[bool] = None,
361
+ ):
362
+ super().__init__()
363
+
364
+ self.register_modules(
365
+ vae=vae,
366
+ text_encoder=text_encoder,
367
+ text_encoder_2=text_encoder_2,
368
+ tokenizer=tokenizer,
369
+ tokenizer_2=tokenizer_2,
370
+ unet=unet,
371
+ adapter=adapter,
372
+ image_encoder=image_encoder,
373
+ feature_extractor=feature_extractor,
374
+ scheduler=scheduler,
375
+ )
376
+ self.register_to_config(force_zeros_for_empty_prompt=force_zeros_for_empty_prompt)
377
+ self.register_to_config(requires_aesthetics_score=requires_aesthetics_score)
378
+ self.vae_scale_factor = 2 ** (len(self.vae.config.block_out_channels) - 1)
379
+ self.image_processor = VaeImageProcessor(vae_scale_factor=self.vae_scale_factor)
380
+ self.mask_processor = VaeImageProcessor(
381
+ vae_scale_factor=self.vae_scale_factor, do_normalize=False, do_binarize=True, do_convert_grayscale=True
382
+ )
383
+
384
+ add_watermarker = add_watermarker if add_watermarker is not None else is_invisible_watermark_available()
385
+
386
+ if add_watermarker:
387
+ self.watermark = StableDiffusionXLWatermarker()
388
+ else:
389
+ self.watermark = None
390
+
391
+ # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.StableDiffusionPipeline.encode_image
392
+ def encode_image(self, image, device, num_images_per_prompt, output_hidden_states=None):
393
+ dtype = next(self.image_encoder.parameters()).dtype
394
+
395
+ if not isinstance(image, torch.Tensor):
396
+ image = self.feature_extractor(image, return_tensors="pt").pixel_values
397
+
398
+ image = image.to(device=device, dtype=dtype)
399
+ if output_hidden_states:
400
+ image_enc_hidden_states = self.image_encoder(image, output_hidden_states=True).hidden_states[-2]
401
+ image_enc_hidden_states = image_enc_hidden_states.repeat_interleave(num_images_per_prompt, dim=0)
402
+ uncond_image_enc_hidden_states = self.image_encoder(
403
+ torch.zeros_like(image), output_hidden_states=True
404
+ ).hidden_states[-2]
405
+ uncond_image_enc_hidden_states = uncond_image_enc_hidden_states.repeat_interleave(
406
+ num_images_per_prompt, dim=0
407
+ )
408
+ return image_enc_hidden_states, uncond_image_enc_hidden_states
409
+ else:
410
+ image_embeds = self.image_encoder(image).image_embeds
411
+ image_embeds = image_embeds.repeat_interleave(num_images_per_prompt, dim=0)
412
+ uncond_image_embeds = torch.zeros_like(image_embeds)
413
+
414
+ return image_embeds, uncond_image_embeds
415
+
416
+ # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.StableDiffusionPipeline.prepare_ip_adapter_image_embeds
417
+ def prepare_ip_adapter_image_embeds(
418
+ self, ip_adapter_image, ip_adapter_image_embeds, device, num_images_per_prompt, do_classifier_free_guidance
419
+ ):
420
+ image_embeds = []
421
+ if do_classifier_free_guidance:
422
+ negative_image_embeds = []
423
+ if ip_adapter_image_embeds is None:
424
+ if not isinstance(ip_adapter_image, list):
425
+ ip_adapter_image = [ip_adapter_image]
426
+
427
+ if len(ip_adapter_image) != len(self.unet.encoder_hid_proj.image_projection_layers):
428
+ raise ValueError(
429
+ f"`ip_adapter_image` must have same length as the number of IP Adapters. Got {len(ip_adapter_image)} images and {len(self.unet.encoder_hid_proj.image_projection_layers)} IP Adapters."
430
+ )
431
+
432
+ for single_ip_adapter_image, image_proj_layer in zip(
433
+ ip_adapter_image, self.unet.encoder_hid_proj.image_projection_layers
434
+ ):
435
+ output_hidden_state = not isinstance(image_proj_layer, ImageProjection)
436
+ single_image_embeds, single_negative_image_embeds = self.encode_image(
437
+ single_ip_adapter_image, device, 1, output_hidden_state
438
+ )
439
+
440
+ image_embeds.append(single_image_embeds[None, :])
441
+ if do_classifier_free_guidance:
442
+ negative_image_embeds.append(single_negative_image_embeds[None, :])
443
+ else:
444
+ for single_image_embeds in ip_adapter_image_embeds:
445
+ if do_classifier_free_guidance:
446
+ single_negative_image_embeds, single_image_embeds = single_image_embeds.chunk(2)
447
+ negative_image_embeds.append(single_negative_image_embeds)
448
+ image_embeds.append(single_image_embeds)
449
+
450
+ ip_adapter_image_embeds = []
451
+ for i, single_image_embeds in enumerate(image_embeds):
452
+ single_image_embeds = torch.cat([single_image_embeds] * num_images_per_prompt, dim=0)
453
+ if do_classifier_free_guidance:
454
+ single_negative_image_embeds = torch.cat([negative_image_embeds[i]] * num_images_per_prompt, dim=0)
455
+ single_image_embeds = torch.cat([single_negative_image_embeds, single_image_embeds], dim=0)
456
+
457
+ single_image_embeds = single_image_embeds.to(device=device)
458
+ ip_adapter_image_embeds.append(single_image_embeds)
459
+
460
+ return ip_adapter_image_embeds
461
+
462
+ # Copied from diffusers.pipelines.stable_diffusion_xl.pipeline_stable_diffusion_xl.StableDiffusionXLPipeline.encode_prompt
463
+ def encode_prompt(
464
+ self,
465
+ prompt: str,
466
+ prompt_2: Optional[str] = None,
467
+ device: Optional[torch.device] = None,
468
+ num_images_per_prompt: int = 1,
469
+ do_classifier_free_guidance: bool = True,
470
+ negative_prompt: Optional[str] = None,
471
+ negative_prompt_2: Optional[str] = None,
472
+ prompt_embeds: Optional[torch.Tensor] = None,
473
+ negative_prompt_embeds: Optional[torch.Tensor] = None,
474
+ pooled_prompt_embeds: Optional[torch.Tensor] = None,
475
+ negative_pooled_prompt_embeds: Optional[torch.Tensor] = None,
476
+ lora_scale: Optional[float] = None,
477
+ clip_skip: Optional[int] = None,
478
+ ):
479
+ r"""
480
+ Encodes the prompt into text encoder hidden states.
481
+
482
+ Args:
483
+ prompt (`str` or `List[str]`, *optional*):
484
+ prompt to be encoded
485
+ prompt_2 (`str` or `List[str]`, *optional*):
486
+ The prompt or prompts to be sent to the `tokenizer_2` and `text_encoder_2`. If not defined, `prompt` is
487
+ used in both text-encoders
488
+ device: (`torch.device`):
489
+ torch device
490
+ num_images_per_prompt (`int`):
491
+ number of images that should be generated per prompt
492
+ do_classifier_free_guidance (`bool`):
493
+ whether to use classifier free guidance or not
494
+ negative_prompt (`str` or `List[str]`, *optional*):
495
+ The prompt or prompts not to guide the image generation. If not defined, one has to pass
496
+ `negative_prompt_embeds` instead. Ignored when not using guidance (i.e., ignored if `guidance_scale` is
497
+ less than `1`).
498
+ negative_prompt_2 (`str` or `List[str]`, *optional*):
499
+ The prompt or prompts not to guide the image generation to be sent to `tokenizer_2` and
500
+ `text_encoder_2`. If not defined, `negative_prompt` is used in both text-encoders
501
+ prompt_embeds (`torch.Tensor`, *optional*):
502
+ Pre-generated text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt weighting. If not
503
+ provided, text embeddings will be generated from `prompt` input argument.
504
+ negative_prompt_embeds (`torch.Tensor`, *optional*):
505
+ Pre-generated negative text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt
506
+ weighting. If not provided, negative_prompt_embeds will be generated from `negative_prompt` input
507
+ argument.
508
+ pooled_prompt_embeds (`torch.Tensor`, *optional*):
509
+ Pre-generated pooled text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt weighting.
510
+ If not provided, pooled text embeddings will be generated from `prompt` input argument.
511
+ negative_pooled_prompt_embeds (`torch.Tensor`, *optional*):
512
+ Pre-generated negative pooled text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt
513
+ weighting. If not provided, pooled negative_prompt_embeds will be generated from `negative_prompt`
514
+ input argument.
515
+ lora_scale (`float`, *optional*):
516
+ A lora scale that will be applied to all LoRA layers of the text encoder if LoRA layers are loaded.
517
+ clip_skip (`int`, *optional*):
518
+ Number of layers to be skipped from CLIP while computing the prompt embeddings. A value of 1 means that
519
+ the output of the pre-final layer will be used for computing the prompt embeddings.
520
+ """
521
+ device = device or self._execution_device
522
+
523
+ # set lora scale so that monkey patched LoRA
524
+ # function of text encoder can correctly access it
525
+ if lora_scale is not None and isinstance(self, StableDiffusionXLLoraLoaderMixin):
526
+ self._lora_scale = lora_scale
527
+
528
+ # dynamically adjust the LoRA scale
529
+ if self.text_encoder is not None:
530
+ if not USE_PEFT_BACKEND:
531
+ adjust_lora_scale_text_encoder(self.text_encoder, lora_scale)
532
+ else:
533
+ scale_lora_layers(self.text_encoder, lora_scale)
534
+
535
+ if self.text_encoder_2 is not None:
536
+ if not USE_PEFT_BACKEND:
537
+ adjust_lora_scale_text_encoder(self.text_encoder_2, lora_scale)
538
+ else:
539
+ scale_lora_layers(self.text_encoder_2, lora_scale)
540
+
541
+ prompt = [prompt] if isinstance(prompt, str) else prompt
542
+
543
+ if prompt is not None:
544
+ batch_size = len(prompt)
545
+ else:
546
+ batch_size = prompt_embeds.shape[0]
547
+
548
+ # Define tokenizers and text encoders
549
+ tokenizers = [self.tokenizer, self.tokenizer_2] if self.tokenizer is not None else [self.tokenizer_2]
550
+ text_encoders = (
551
+ [self.text_encoder, self.text_encoder_2] if self.text_encoder is not None else [self.text_encoder_2]
552
+ )
553
+
554
+ if prompt_embeds is None:
555
+ prompt_2 = prompt_2 or prompt
556
+ prompt_2 = [prompt_2] if isinstance(prompt_2, str) else prompt_2
557
+
558
+ # textual inversion: process multi-vector tokens if necessary
559
+ prompt_embeds_list = []
560
+ prompts = [prompt, prompt_2]
561
+ for prompt, tokenizer, text_encoder in zip(prompts, tokenizers, text_encoders):
562
+ if isinstance(self, TextualInversionLoaderMixin):
563
+ prompt = self.maybe_convert_prompt(prompt, tokenizer)
564
+
565
+ text_inputs = tokenizer(
566
+ prompt,
567
+ padding="max_length",
568
+ max_length=tokenizer.model_max_length,
569
+ truncation=True,
570
+ return_tensors="pt",
571
+ )
572
+
573
+ text_input_ids = text_inputs.input_ids
574
+ untruncated_ids = tokenizer(prompt, padding="longest", return_tensors="pt").input_ids
575
+
576
+ if untruncated_ids.shape[-1] >= text_input_ids.shape[-1] and not torch.equal(
577
+ text_input_ids, untruncated_ids
578
+ ):
579
+ removed_text = tokenizer.batch_decode(untruncated_ids[:, tokenizer.model_max_length - 1 : -1])
580
+ logger.warning(
581
+ "The following part of your input was truncated because CLIP can only handle sequences up to"
582
+ f" {tokenizer.model_max_length} tokens: {removed_text}"
583
+ )
584
+
585
+ prompt_embeds = text_encoder(text_input_ids.to(device), output_hidden_states=True)
586
+
587
+ # We are only ALWAYS interested in the pooled output of the final text encoder
588
+ pooled_prompt_embeds = prompt_embeds[0]
589
+ if clip_skip is None:
590
+ prompt_embeds = prompt_embeds.hidden_states[-2]
591
+ else:
592
+ # "2" because SDXL always indexes from the penultimate layer.
593
+ prompt_embeds = prompt_embeds.hidden_states[-(clip_skip + 2)]
594
+
595
+ prompt_embeds_list.append(prompt_embeds)
596
+
597
+ prompt_embeds = torch.concat(prompt_embeds_list, dim=-1)
598
+
599
+ # get unconditional embeddings for classifier free guidance
600
+ zero_out_negative_prompt = negative_prompt is None and self.config.force_zeros_for_empty_prompt
601
+ if do_classifier_free_guidance and negative_prompt_embeds is None and zero_out_negative_prompt:
602
+ negative_prompt_embeds = torch.zeros_like(prompt_embeds)
603
+ negative_pooled_prompt_embeds = torch.zeros_like(pooled_prompt_embeds)
604
+ elif do_classifier_free_guidance and negative_prompt_embeds is None:
605
+ negative_prompt = negative_prompt or ""
606
+ negative_prompt_2 = negative_prompt_2 or negative_prompt
607
+
608
+ # normalize str to list
609
+ negative_prompt = batch_size * [negative_prompt] if isinstance(negative_prompt, str) else negative_prompt
610
+ negative_prompt_2 = (
611
+ batch_size * [negative_prompt_2] if isinstance(negative_prompt_2, str) else negative_prompt_2
612
+ )
613
+
614
+ uncond_tokens: List[str]
615
+ if prompt is not None and type(prompt) is not type(negative_prompt):
616
+ raise TypeError(
617
+ f"`negative_prompt` should be the same type to `prompt`, but got {type(negative_prompt)} !="
618
+ f" {type(prompt)}."
619
+ )
620
+ elif batch_size != len(negative_prompt):
621
+ raise ValueError(
622
+ f"`negative_prompt`: {negative_prompt} has batch size {len(negative_prompt)}, but `prompt`:"
623
+ f" {prompt} has batch size {batch_size}. Please make sure that passed `negative_prompt` matches"
624
+ " the batch size of `prompt`."
625
+ )
626
+ else:
627
+ uncond_tokens = [negative_prompt, negative_prompt_2]
628
+
629
+ negative_prompt_embeds_list = []
630
+ for negative_prompt, tokenizer, text_encoder in zip(uncond_tokens, tokenizers, text_encoders):
631
+ if isinstance(self, TextualInversionLoaderMixin):
632
+ negative_prompt = self.maybe_convert_prompt(negative_prompt, tokenizer)
633
+
634
+ max_length = prompt_embeds.shape[1]
635
+ uncond_input = tokenizer(
636
+ negative_prompt,
637
+ padding="max_length",
638
+ max_length=max_length,
639
+ truncation=True,
640
+ return_tensors="pt",
641
+ )
642
+
643
+ negative_prompt_embeds = text_encoder(
644
+ uncond_input.input_ids.to(device),
645
+ output_hidden_states=True,
646
+ )
647
+ # We are only ALWAYS interested in the pooled output of the final text encoder
648
+ negative_pooled_prompt_embeds = negative_prompt_embeds[0]
649
+ negative_prompt_embeds = negative_prompt_embeds.hidden_states[-2]
650
+
651
+ negative_prompt_embeds_list.append(negative_prompt_embeds)
652
+
653
+ negative_prompt_embeds = torch.concat(negative_prompt_embeds_list, dim=-1)
654
+
655
+ if self.text_encoder_2 is not None:
656
+ prompt_embeds = prompt_embeds.to(dtype=self.text_encoder_2.dtype, device=device)
657
+ else:
658
+ prompt_embeds = prompt_embeds.to(dtype=self.unet.dtype, device=device)
659
+
660
+ bs_embed, seq_len, _ = prompt_embeds.shape
661
+ # duplicate text embeddings for each generation per prompt, using mps friendly method
662
+ prompt_embeds = prompt_embeds.repeat(1, num_images_per_prompt, 1)
663
+ prompt_embeds = prompt_embeds.view(bs_embed * num_images_per_prompt, seq_len, -1)
664
+
665
+ if do_classifier_free_guidance:
666
+ # duplicate unconditional embeddings for each generation per prompt, using mps friendly method
667
+ seq_len = negative_prompt_embeds.shape[1]
668
+
669
+ if self.text_encoder_2 is not None:
670
+ negative_prompt_embeds = negative_prompt_embeds.to(dtype=self.text_encoder_2.dtype, device=device)
671
+ else:
672
+ negative_prompt_embeds = negative_prompt_embeds.to(dtype=self.unet.dtype, device=device)
673
+
674
+ negative_prompt_embeds = negative_prompt_embeds.repeat(1, num_images_per_prompt, 1)
675
+ negative_prompt_embeds = negative_prompt_embeds.view(batch_size * num_images_per_prompt, seq_len, -1)
676
+
677
+ pooled_prompt_embeds = pooled_prompt_embeds.repeat(1, num_images_per_prompt).view(
678
+ bs_embed * num_images_per_prompt, -1
679
+ )
680
+ if do_classifier_free_guidance:
681
+ negative_pooled_prompt_embeds = negative_pooled_prompt_embeds.repeat(1, num_images_per_prompt).view(
682
+ bs_embed * num_images_per_prompt, -1
683
+ )
684
+
685
+ if self.text_encoder is not None:
686
+ if isinstance(self, StableDiffusionXLLoraLoaderMixin) and USE_PEFT_BACKEND:
687
+ # Retrieve the original scale by scaling back the LoRA layers
688
+ unscale_lora_layers(self.text_encoder, lora_scale)
689
+
690
+ if self.text_encoder_2 is not None:
691
+ if isinstance(self, StableDiffusionXLLoraLoaderMixin) and USE_PEFT_BACKEND:
692
+ # Retrieve the original scale by scaling back the LoRA layers
693
+ unscale_lora_layers(self.text_encoder_2, lora_scale)
694
+
695
+ return prompt_embeds, negative_prompt_embeds, pooled_prompt_embeds, negative_pooled_prompt_embeds
696
+
697
+ # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.StableDiffusionPipeline.prepare_extra_step_kwargs
698
+ def prepare_extra_step_kwargs(self, generator, eta):
699
+ # prepare extra kwargs for the scheduler step, since not all schedulers have the same signature
700
+ # eta (η) is only used with the DDIMScheduler, it will be ignored for other schedulers.
701
+ # eta corresponds to η in DDIM paper: https://arxiv.org/abs/2010.02502
702
+ # and should be between [0, 1]
703
+
704
+ accepts_eta = "eta" in set(inspect.signature(self.scheduler.step).parameters.keys())
705
+ extra_step_kwargs = {}
706
+ if accepts_eta:
707
+ extra_step_kwargs["eta"] = eta
708
+
709
+ # check if the scheduler accepts generator
710
+ accepts_generator = "generator" in set(inspect.signature(self.scheduler.step).parameters.keys())
711
+ if accepts_generator:
712
+ extra_step_kwargs["generator"] = generator
713
+ return extra_step_kwargs
714
+
715
+ def check_inputs(
716
+ self,
717
+ prompt,
718
+ prompt_2,
719
+ image,
720
+ mask_image,
721
+ height,
722
+ width,
723
+ strength,
724
+ callback_steps,
725
+ output_type,
726
+ negative_prompt=None,
727
+ negative_prompt_2=None,
728
+ prompt_embeds=None,
729
+ negative_prompt_embeds=None,
730
+ ip_adapter_image=None,
731
+ ip_adapter_image_embeds=None,
732
+ callback_on_step_end_tensor_inputs=None,
733
+ padding_mask_crop=None,
734
+ ):
735
+ if strength < 0 or strength > 1:
736
+ raise ValueError(f"The value of strength should in [0.0, 1.0] but is {strength}")
737
+
738
+ if height % 8 != 0 or width % 8 != 0:
739
+ raise ValueError(f"`height` and `width` have to be divisible by 8 but are {height} and {width}.")
740
+
741
+ if callback_steps is not None and (not isinstance(callback_steps, int) or callback_steps <= 0):
742
+ raise ValueError(
743
+ f"`callback_steps` has to be a positive integer but is {callback_steps} of type"
744
+ f" {type(callback_steps)}."
745
+ )
746
+
747
+ if callback_on_step_end_tensor_inputs is not None and not all(
748
+ k in self._callback_tensor_inputs for k in callback_on_step_end_tensor_inputs
749
+ ):
750
+ raise ValueError(
751
+ f"`callback_on_step_end_tensor_inputs` has to be in {self._callback_tensor_inputs}, but found {[k for k in callback_on_step_end_tensor_inputs if k not in self._callback_tensor_inputs]}"
752
+ )
753
+
754
+ if prompt is not None and prompt_embeds is not None:
755
+ raise ValueError(
756
+ f"Cannot forward both `prompt`: {prompt} and `prompt_embeds`: {prompt_embeds}. Please make sure to"
757
+ " only forward one of the two."
758
+ )
759
+ elif prompt_2 is not None and prompt_embeds is not None:
760
+ raise ValueError(
761
+ f"Cannot forward both `prompt_2`: {prompt_2} and `prompt_embeds`: {prompt_embeds}. Please make sure to"
762
+ " only forward one of the two."
763
+ )
764
+ elif prompt is None and prompt_embeds is None:
765
+ raise ValueError(
766
+ "Provide either `prompt` or `prompt_embeds`. Cannot leave both `prompt` and `prompt_embeds` undefined."
767
+ )
768
+ elif prompt is not None and (not isinstance(prompt, str) and not isinstance(prompt, list)):
769
+ raise ValueError(f"`prompt` has to be of type `str` or `list` but is {type(prompt)}")
770
+ elif prompt_2 is not None and (not isinstance(prompt_2, str) and not isinstance(prompt_2, list)):
771
+ raise ValueError(f"`prompt_2` has to be of type `str` or `list` but is {type(prompt_2)}")
772
+
773
+ if negative_prompt is not None and negative_prompt_embeds is not None:
774
+ raise ValueError(
775
+ f"Cannot forward both `negative_prompt`: {negative_prompt} and `negative_prompt_embeds`:"
776
+ f" {negative_prompt_embeds}. Please make sure to only forward one of the two."
777
+ )
778
+ elif negative_prompt_2 is not None and negative_prompt_embeds is not None:
779
+ raise ValueError(
780
+ f"Cannot forward both `negative_prompt_2`: {negative_prompt_2} and `negative_prompt_embeds`:"
781
+ f" {negative_prompt_embeds}. Please make sure to only forward one of the two."
782
+ )
783
+
784
+ if prompt_embeds is not None and negative_prompt_embeds is not None:
785
+ if prompt_embeds.shape != negative_prompt_embeds.shape:
786
+ raise ValueError(
787
+ "`prompt_embeds` and `negative_prompt_embeds` must have the same shape when passed directly, but"
788
+ f" got: `prompt_embeds` {prompt_embeds.shape} != `negative_prompt_embeds`"
789
+ f" {negative_prompt_embeds.shape}."
790
+ )
791
+ if padding_mask_crop is not None:
792
+ if not isinstance(image, PIL.Image.Image):
793
+ raise ValueError(
794
+ f"The image should be a PIL image when inpainting mask crop, but is of type" f" {type(image)}."
795
+ )
796
+ if not isinstance(mask_image, PIL.Image.Image):
797
+ raise ValueError(
798
+ f"The mask image should be a PIL image when inpainting mask crop, but is of type"
799
+ f" {type(mask_image)}."
800
+ )
801
+ if output_type != "pil":
802
+ raise ValueError(f"The output type should be PIL when inpainting mask crop, but is" f" {output_type}.")
803
+
804
+ if ip_adapter_image is not None and ip_adapter_image_embeds is not None:
805
+ raise ValueError(
806
+ "Provide either `ip_adapter_image` or `ip_adapter_image_embeds`. Cannot leave both `ip_adapter_image` and `ip_adapter_image_embeds` defined."
807
+ )
808
+
809
+ if ip_adapter_image_embeds is not None:
810
+ if not isinstance(ip_adapter_image_embeds, list):
811
+ raise ValueError(
812
+ f"`ip_adapter_image_embeds` has to be of type `list` but is {type(ip_adapter_image_embeds)}"
813
+ )
814
+ elif ip_adapter_image_embeds[0].ndim not in [3, 4]:
815
+ raise ValueError(
816
+ f"`ip_adapter_image_embeds` has to be a list of 3D or 4D tensors but is {ip_adapter_image_embeds[0].ndim}D"
817
+ )
818
+
819
+ def prepare_latents(
820
+ self,
821
+ batch_size,
822
+ num_channels_latents,
823
+ height,
824
+ width,
825
+ dtype,
826
+ device,
827
+ generator,
828
+ latents=None,
829
+ image=None,
830
+ timestep=None,
831
+ is_strength_max=True,
832
+ add_noise=True,
833
+ return_noise=False,
834
+ return_image_latents=False,
835
+ ):
836
+ shape = (
837
+ batch_size,
838
+ num_channels_latents,
839
+ int(height) // self.vae_scale_factor,
840
+ int(width) // self.vae_scale_factor,
841
+ )
842
+ if isinstance(generator, list) and len(generator) != batch_size:
843
+ raise ValueError(
844
+ f"You have passed a list of generators of length {len(generator)}, but requested an effective batch"
845
+ f" size of {batch_size}. Make sure the batch size matches the length of the generators."
846
+ )
847
+
848
+ if (image is None or timestep is None) and not is_strength_max:
849
+ raise ValueError(
850
+ "Since strength < 1. initial latents are to be initialised as a combination of Image + Noise."
851
+ "However, either the image or the noise timestep has not been provided."
852
+ )
853
+
854
+ if image.shape[1] == 4:
855
+ image_latents = image.to(device=device, dtype=dtype)
856
+ image_latents = image_latents.repeat(batch_size // image_latents.shape[0], 1, 1, 1)
857
+ elif return_image_latents or (latents is None and not is_strength_max):
858
+ image = image.to(device=device, dtype=dtype)
859
+ image_latents = self._encode_vae_image(image=image, generator=generator)
860
+ image_latents = image_latents.repeat(batch_size // image_latents.shape[0], 1, 1, 1)
861
+
862
+ if latents is None and add_noise:
863
+ noise = randn_tensor(shape, generator=generator, device=device, dtype=dtype)
864
+ # if strength is 1. then initialise the latents to noise, else initial to image + noise
865
+ latents = noise if is_strength_max else self.scheduler.add_noise(image_latents, noise, timestep)
866
+ # if pure noise then scale the initial latents by the Scheduler's init sigma
867
+ latents = latents * self.scheduler.init_noise_sigma if is_strength_max else latents
868
+ elif add_noise:
869
+ noise = latents.to(device)
870
+ latents = noise * self.scheduler.init_noise_sigma
871
+ else:
872
+ noise = randn_tensor(shape, generator=generator, device=device, dtype=dtype)
873
+ latents = image_latents.to(device)
874
+
875
+ outputs = (latents,)
876
+
877
+ if return_noise:
878
+ outputs += (noise,)
879
+
880
+ if return_image_latents:
881
+ outputs += (image_latents,)
882
+
883
+ return outputs
884
+
885
+ def _encode_vae_image(self, image: torch.Tensor, generator: torch.Generator):
886
+ dtype = image.dtype
887
+ if self.vae.config.force_upcast:
888
+ image = image.float()
889
+ self.vae.to(dtype=torch.float32)
890
+
891
+ if isinstance(generator, list):
892
+ image_latents = [
893
+ retrieve_latents(self.vae.encode(image[i : i + 1]), generator=generator[i])
894
+ for i in range(image.shape[0])
895
+ ]
896
+ image_latents = torch.cat(image_latents, dim=0)
897
+ else:
898
+ image_latents = retrieve_latents(self.vae.encode(image), generator=generator)
899
+
900
+ if self.vae.config.force_upcast:
901
+ self.vae.to(dtype)
902
+
903
+ image_latents = image_latents.to(dtype)
904
+ image_latents = self.vae.config.scaling_factor * image_latents
905
+
906
+ return image_latents
907
+
908
+ def prepare_mask_latents(
909
+ self, mask, masked_image, batch_size, height, width, dtype, device, generator, do_classifier_free_guidance
910
+ ):
911
+ # resize the mask to latents shape as we concatenate the mask to the latents
912
+ # we do that before converting to dtype to avoid breaking in case we're using cpu_offload
913
+ # and half precision
914
+ mask = torch.nn.functional.interpolate(
915
+ mask, size=(height // self.vae_scale_factor, width // self.vae_scale_factor)
916
+ )
917
+ mask = mask.to(device=device, dtype=dtype)
918
+
919
+ # duplicate mask and masked_image_latents for each generation per prompt, using mps friendly method
920
+ if mask.shape[0] < batch_size:
921
+ if not batch_size % mask.shape[0] == 0:
922
+ raise ValueError(
923
+ "The passed mask and the required batch size don't match. Masks are supposed to be duplicated to"
924
+ f" a total batch size of {batch_size}, but {mask.shape[0]} masks were passed. Make sure the number"
925
+ " of masks that you pass is divisible by the total requested batch size."
926
+ )
927
+ mask = mask.repeat(batch_size // mask.shape[0], 1, 1, 1)
928
+
929
+ mask = torch.cat([mask] * 2) if do_classifier_free_guidance else mask
930
+
931
+ if masked_image is not None and masked_image.shape[1] == 4:
932
+ masked_image_latents = masked_image
933
+ else:
934
+ masked_image_latents = None
935
+
936
+ if masked_image is not None:
937
+ if masked_image_latents is None:
938
+ masked_image = masked_image.to(device=device, dtype=dtype)
939
+ masked_image_latents = self._encode_vae_image(masked_image, generator=generator)
940
+
941
+ if masked_image_latents.shape[0] < batch_size:
942
+ if not batch_size % masked_image_latents.shape[0] == 0:
943
+ raise ValueError(
944
+ "The passed images and the required batch size don't match. Images are supposed to be duplicated"
945
+ f" to a total batch size of {batch_size}, but {masked_image_latents.shape[0]} images were passed."
946
+ " Make sure the number of images that you pass is divisible by the total requested batch size."
947
+ )
948
+ masked_image_latents = masked_image_latents.repeat(
949
+ batch_size // masked_image_latents.shape[0], 1, 1, 1
950
+ )
951
+
952
+ masked_image_latents = (
953
+ torch.cat([masked_image_latents] * 2) if do_classifier_free_guidance else masked_image_latents
954
+ )
955
+
956
+ # aligning device to prevent device errors when concating it with the latent model input
957
+ masked_image_latents = masked_image_latents.to(device=device, dtype=dtype)
958
+
959
+ return mask, masked_image_latents
960
+
961
+ # Copied from diffusers.pipelines.stable_diffusion_xl.pipeline_stable_diffusion_xl_img2img.StableDiffusionXLImg2ImgPipeline.get_timesteps
962
+ def get_timesteps(self, num_inference_steps, strength, device, denoising_start=None):
963
+ # get the original timestep using init_timestep
964
+ if denoising_start is None:
965
+ init_timestep = min(int(num_inference_steps * strength), num_inference_steps)
966
+ t_start = max(num_inference_steps - init_timestep, 0)
967
+
968
+ timesteps = self.scheduler.timesteps[t_start * self.scheduler.order :]
969
+ if hasattr(self.scheduler, "set_begin_index"):
970
+ self.scheduler.set_begin_index(t_start * self.scheduler.order)
971
+
972
+ return timesteps, num_inference_steps - t_start
973
+
974
+ else:
975
+ # Strength is irrelevant if we directly request a timestep to start at;
976
+ # that is, strength is determined by the denoising_start instead.
977
+ discrete_timestep_cutoff = int(
978
+ round(
979
+ self.scheduler.config.num_train_timesteps
980
+ - (denoising_start * self.scheduler.config.num_train_timesteps)
981
+ )
982
+ )
983
+
984
+ num_inference_steps = (self.scheduler.timesteps < discrete_timestep_cutoff).sum().item()
985
+ if self.scheduler.order == 2 and num_inference_steps % 2 == 0:
986
+ # if the scheduler is a 2nd order scheduler we might have to do +1
987
+ # because `num_inference_steps` might be even given that every timestep
988
+ # (except the highest one) is duplicated. If `num_inference_steps` is even it would
989
+ # mean that we cut the timesteps in the middle of the denoising step
990
+ # (between 1st and 2nd derivative) which leads to incorrect results. By adding 1
991
+ # we ensure that the denoising process always ends after the 2nd derivate step of the scheduler
992
+ num_inference_steps = num_inference_steps + 1
993
+
994
+ # because t_n+1 >= t_n, we slice the timesteps starting from the end
995
+ t_start = len(self.scheduler.timesteps) - num_inference_steps
996
+ timesteps = self.scheduler.timesteps[t_start:]
997
+ if hasattr(self.scheduler, "set_begin_index"):
998
+ self.scheduler.set_begin_index(t_start)
999
+ return timesteps, num_inference_steps
1000
+
1001
+ # Copied from diffusers.pipelines.stable_diffusion_xl.pipeline_stable_diffusion_xl_img2img.StableDiffusionXLImg2ImgPipeline._get_add_time_ids
1002
+ def _get_add_time_ids(
1003
+ self,
1004
+ original_size,
1005
+ crops_coords_top_left,
1006
+ target_size,
1007
+ aesthetic_score,
1008
+ negative_aesthetic_score,
1009
+ negative_original_size,
1010
+ negative_crops_coords_top_left,
1011
+ negative_target_size,
1012
+ dtype,
1013
+ text_encoder_projection_dim=None,
1014
+ ):
1015
+ if self.config.requires_aesthetics_score:
1016
+ add_time_ids = list(original_size + crops_coords_top_left + (aesthetic_score,))
1017
+ add_neg_time_ids = list(
1018
+ negative_original_size + negative_crops_coords_top_left + (negative_aesthetic_score,)
1019
+ )
1020
+ else:
1021
+ add_time_ids = list(original_size + crops_coords_top_left + target_size)
1022
+ add_neg_time_ids = list(negative_original_size + crops_coords_top_left + negative_target_size)
1023
+
1024
+ passed_add_embed_dim = (
1025
+ self.unet.config.addition_time_embed_dim * len(add_time_ids) + text_encoder_projection_dim
1026
+ )
1027
+ expected_add_embed_dim = self.unet.add_embedding.linear_1.in_features
1028
+
1029
+ if (
1030
+ expected_add_embed_dim > passed_add_embed_dim
1031
+ and (expected_add_embed_dim - passed_add_embed_dim) == self.unet.config.addition_time_embed_dim
1032
+ ):
1033
+ raise ValueError(
1034
+ f"Model expects an added time embedding vector of length {expected_add_embed_dim}, but a vector of {passed_add_embed_dim} was created. Please make sure to enable `requires_aesthetics_score` with `pipe.register_to_config(requires_aesthetics_score=True)` to make sure `aesthetic_score` {aesthetic_score} and `negative_aesthetic_score` {negative_aesthetic_score} is correctly used by the model."
1035
+ )
1036
+ elif (
1037
+ expected_add_embed_dim < passed_add_embed_dim
1038
+ and (passed_add_embed_dim - expected_add_embed_dim) == self.unet.config.addition_time_embed_dim
1039
+ ):
1040
+ raise ValueError(
1041
+ f"Model expects an added time embedding vector of length {expected_add_embed_dim}, but a vector of {passed_add_embed_dim} was created. Please make sure to disable `requires_aesthetics_score` with `pipe.register_to_config(requires_aesthetics_score=False)` to make sure `target_size` {target_size} is correctly used by the model."
1042
+ )
1043
+ elif expected_add_embed_dim != passed_add_embed_dim:
1044
+ raise ValueError(
1045
+ f"Model expects an added time embedding vector of length {expected_add_embed_dim}, but a vector of {passed_add_embed_dim} was created. The model has an incorrect config. Please check `unet.config.time_embedding_type` and `text_encoder_2.config.projection_dim`."
1046
+ )
1047
+
1048
+ add_time_ids = torch.tensor([add_time_ids], dtype=dtype)
1049
+ add_neg_time_ids = torch.tensor([add_neg_time_ids], dtype=dtype)
1050
+
1051
+ return add_time_ids, add_neg_time_ids
1052
+
1053
+ # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion_upscale.StableDiffusionUpscalePipeline.upcast_vae
1054
+ def upcast_vae(self):
1055
+ dtype = self.vae.dtype
1056
+ self.vae.to(dtype=torch.float32)
1057
+ use_torch_2_0_or_xformers = isinstance(
1058
+ self.vae.decoder.mid_block.attentions[0].processor,
1059
+ (
1060
+ AttnProcessor2_0,
1061
+ XFormersAttnProcessor,
1062
+ ),
1063
+ )
1064
+ # if xformers or torch_2_0 is used attention block does not need
1065
+ # to be in float32 which can save lots of memory
1066
+ if use_torch_2_0_or_xformers:
1067
+ self.vae.post_quant_conv.to(dtype)
1068
+ self.vae.decoder.conv_in.to(dtype)
1069
+ self.vae.decoder.mid_block.to(dtype)
1070
+
1071
+ # Copied from diffusers.pipelines.t2i_adapter.pipeline_stable_diffusion_adapter.StableDiffusionAdapterPipeline._default_height_width
1072
+ def _default_height_width(self, height, width, image):
1073
+ # NOTE: It is possible that a list of images have different
1074
+ # dimensions for each image, so just checking the first image
1075
+ # is not _exactly_ correct, but it is simple.
1076
+ while isinstance(image, list):
1077
+ image = image[0]
1078
+
1079
+ if height is None:
1080
+ if isinstance(image, PIL.Image.Image):
1081
+ height = image.height
1082
+ elif isinstance(image, torch.Tensor):
1083
+ height = image.shape[-2]
1084
+
1085
+ # round down to nearest multiple of `self.adapter.downscale_factor`
1086
+ height = (height // self.adapter.downscale_factor) * self.adapter.downscale_factor
1087
+
1088
+ if width is None:
1089
+ if isinstance(image, PIL.Image.Image):
1090
+ width = image.width
1091
+ elif isinstance(image, torch.Tensor):
1092
+ width = image.shape[-1]
1093
+
1094
+ # round down to nearest multiple of `self.adapter.downscale_factor`
1095
+ width = (width // self.adapter.downscale_factor) * self.adapter.downscale_factor
1096
+
1097
+ return height, width
1098
+
1099
+ # Copied from diffusers.pipelines.latent_consistency_models.pipeline_latent_consistency_text2img.LatentConsistencyModelPipeline.get_guidance_scale_embedding
1100
+ def get_guidance_scale_embedding(
1101
+ self, w: torch.Tensor, embedding_dim: int = 512, dtype: torch.dtype = torch.float32
1102
+ ) -> torch.Tensor:
1103
+ """
1104
+ See https://github.com/google-research/vdm/blob/dc27b98a554f65cdc654b800da5aa1846545d41b/model_vdm.py#L298
1105
+
1106
+ Args:
1107
+ w (`torch.Tensor`):
1108
+ Generate embedding vectors with a specified guidance scale to subsequently enrich timestep embeddings.
1109
+ embedding_dim (`int`, *optional*, defaults to 512):
1110
+ Dimension of the embeddings to generate.
1111
+ dtype (`torch.dtype`, *optional*, defaults to `torch.float32`):
1112
+ Data type of the generated embeddings.
1113
+
1114
+ Returns:
1115
+ `torch.Tensor`: Embedding vectors with shape `(len(w), embedding_dim)`.
1116
+ """
1117
+ assert len(w.shape) == 1
1118
+ w = w * 1000.0
1119
+
1120
+ half_dim = embedding_dim // 2
1121
+ emb = torch.log(torch.tensor(10000.0)) / (half_dim - 1)
1122
+ emb = torch.exp(torch.arange(half_dim, dtype=dtype) * -emb)
1123
+ emb = w.to(dtype)[:, None] * emb[None, :]
1124
+ emb = torch.cat([torch.sin(emb), torch.cos(emb)], dim=1)
1125
+ if embedding_dim % 2 == 1: # zero pad
1126
+ emb = torch.nn.functional.pad(emb, (0, 1))
1127
+ assert emb.shape == (w.shape[0], embedding_dim)
1128
+ return emb
1129
+
1130
+ @property
1131
+ def guidance_scale(self):
1132
+ return self._guidance_scale
1133
+
1134
+ @property
1135
+ def guidance_rescale(self):
1136
+ return self._guidance_rescale
1137
+
1138
+ @property
1139
+ def clip_skip(self):
1140
+ return self._clip_skip
1141
+
1142
+ # here `guidance_scale` is defined analog to the guidance weight `w` of equation (2)
1143
+ # of the Imagen paper: https://arxiv.org/pdf/2205.11487.pdf . `guidance_scale = 1`
1144
+ # corresponds to doing no classifier free guidance.
1145
+ @property
1146
+ def do_classifier_free_guidance(self):
1147
+ return self._guidance_scale > 1 and self.unet.config.time_cond_proj_dim is None
1148
+
1149
+ @property
1150
+ def cross_attention_kwargs(self):
1151
+ return self._cross_attention_kwargs
1152
+
1153
+ @property
1154
+ def denoising_end(self):
1155
+ return self._denoising_end
1156
+
1157
+ @property
1158
+ def denoising_start(self):
1159
+ return self._denoising_start
1160
+
1161
+ @property
1162
+ def num_timesteps(self):
1163
+ return self._num_timesteps
1164
+
1165
+ @property
1166
+ def interrupt(self):
1167
+ return self._interrupt
1168
+
1169
+ @torch.no_grad()
1170
+ @replace_example_docstring(EXAMPLE_DOC_STRING)
1171
+ def __call__(
1172
+ self,
1173
+ prompt: Union[str, List[str]] = None,
1174
+ prompt_2: Optional[Union[str, List[str]]] = None,
1175
+ image: PipelineImageInput = None,
1176
+ mask_image: PipelineImageInput = None,
1177
+ masked_image_latents: torch.Tensor = None,
1178
+ height: Optional[int] = None,
1179
+ adapter_image: PipelineImageInput = None,
1180
+ width: Optional[int] = None,
1181
+ padding_mask_crop: Optional[int] = None,
1182
+ strength: float = 0.9999,
1183
+ num_inference_steps: int = 50,
1184
+ timesteps: List[int] = None,
1185
+ sigmas: List[float] = None,
1186
+ denoising_start: Optional[float] = None,
1187
+ denoising_end: Optional[float] = None,
1188
+ guidance_scale: float = 7.5,
1189
+ negative_prompt: Optional[Union[str, List[str]]] = None,
1190
+ negative_prompt_2: Optional[Union[str, List[str]]] = None,
1191
+ num_images_per_prompt: Optional[int] = 1,
1192
+ eta: float = 0.0,
1193
+ generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None,
1194
+ latents: Optional[torch.Tensor] = None,
1195
+ prompt_embeds: Optional[torch.Tensor] = None,
1196
+ negative_prompt_embeds: Optional[torch.Tensor] = None,
1197
+ pooled_prompt_embeds: Optional[torch.Tensor] = None,
1198
+ negative_pooled_prompt_embeds: Optional[torch.Tensor] = None,
1199
+ ip_adapter_image: Optional[PipelineImageInput] = None,
1200
+ ip_adapter_image_embeds: Optional[List[torch.Tensor]] = None,
1201
+ output_type: Optional[str] = "pil",
1202
+ return_dict: bool = True,
1203
+ cross_attention_kwargs: Optional[Dict[str, Any]] = None,
1204
+ guidance_rescale: float = 0.0,
1205
+ original_size: Tuple[int, int] = None,
1206
+ crops_coords_top_left: Tuple[int, int] = (0, 0),
1207
+ target_size: Tuple[int, int] = None,
1208
+ negative_original_size: Optional[Tuple[int, int]] = None,
1209
+ negative_crops_coords_top_left: Tuple[int, int] = (0, 0),
1210
+ negative_target_size: Optional[Tuple[int, int]] = None,
1211
+ aesthetic_score: float = 6.0,
1212
+ negative_aesthetic_score: float = 2.5,
1213
+ adapter_conditioning_scale: Union[float, List[float]] = 1.0,
1214
+ adapter_conditioning_factor: float = 1.0,
1215
+ clip_skip: Optional[int] = None,
1216
+ callback_on_step_end: Optional[
1217
+ Union[Callable[[int, int, Dict], None], PipelineCallback, MultiPipelineCallbacks]
1218
+ ] = None,
1219
+ callback_on_step_end_tensor_inputs: List[str] = ["latents"],
1220
+ **kwargs,
1221
+ ):
1222
+ r"""
1223
+ Function invoked when calling the pipeline for generation.
1224
+
1225
+ Args:
1226
+ prompt (`str` or `List[str]`, *optional*):
1227
+ The prompt or prompts to guide the image generation. If not defined, one has to pass `prompt_embeds`.
1228
+ instead.
1229
+ prompt_2 (`str` or `List[str]`, *optional*):
1230
+ The prompt or prompts to be sent to the `tokenizer_2` and `text_encoder_2`. If not defined, `prompt` is
1231
+ used in both text-encoders
1232
+ image (`PIL.Image.Image`):
1233
+ `Image`, or tensor representing an image batch which will be inpainted, *i.e.* parts of the image will
1234
+ be masked out with `mask_image` and repainted according to `prompt`.
1235
+ mask_image (`PIL.Image.Image`):
1236
+ `Image`, or tensor representing an image batch, to mask `image`. White pixels in the mask will be
1237
+ repainted, while black pixels will be preserved. If `mask_image` is a PIL image, it will be converted
1238
+ to a single channel (luminance) before use. If it's a tensor, it should contain one color channel (L)
1239
+ instead of 3, so the expected shape would be `(B, H, W, 1)`.
1240
+ height (`int`, *optional*, defaults to self.unet.config.sample_size * self.vae_scale_factor):
1241
+ The height in pixels of the generated image. This is set to 1024 by default for the best results.
1242
+ Anything below 512 pixels won't work well for
1243
+ [stabilityai/stable-diffusion-xl-base-1.0](https://huggingface.co/stabilityai/stable-diffusion-xl-base-1.0)
1244
+ and checkpoints that are not specifically fine-tuned on low resolutions.
1245
+ width (`int`, *optional*, defaults to self.unet.config.sample_size * self.vae_scale_factor):
1246
+ The width in pixels of the generated image. This is set to 1024 by default for the best results.
1247
+ Anything below 512 pixels won't work well for
1248
+ [stabilityai/stable-diffusion-xl-base-1.0](https://huggingface.co/stabilityai/stable-diffusion-xl-base-1.0)
1249
+ and checkpoints that are not specifically fine-tuned on low resolutions.
1250
+ padding_mask_crop (`int`, *optional*, defaults to `None`):
1251
+ The size of margin in the crop to be applied to the image and masking. If `None`, no crop is applied to
1252
+ image and mask_image. If `padding_mask_crop` is not `None`, it will first find a rectangular region
1253
+ with the same aspect ration of the image and contains all masked area, and then expand that area based
1254
+ on `padding_mask_crop`. The image and mask_image will then be cropped based on the expanded area before
1255
+ resizing to the original image size for inpainting. This is useful when the masked area is small while
1256
+ the image is large and contain information irrelevant for inpainting, such as background.
1257
+ strength (`float`, *optional*, defaults to 0.9999):
1258
+ Conceptually, indicates how much to transform the masked portion of the reference `image`. Must be
1259
+ between 0 and 1. `image` will be used as a starting point, adding more noise to it the larger the
1260
+ `strength`. The number of denoising steps depends on the amount of noise initially added. When
1261
+ `strength` is 1, added noise will be maximum and the denoising process will run for the full number of
1262
+ iterations specified in `num_inference_steps`. A value of 1, therefore, essentially ignores the masked
1263
+ portion of the reference `image`. Note that in the case of `denoising_start` being declared as an
1264
+ integer, the value of `strength` will be ignored.
1265
+ num_inference_steps (`int`, *optional*, defaults to 50):
1266
+ The number of denoising steps. More denoising steps usually lead to a higher quality image at the
1267
+ expense of slower inference.
1268
+ timesteps (`List[int]`, *optional*):
1269
+ Custom timesteps to use for the denoising process with schedulers which support a `timesteps` argument
1270
+ in their `set_timesteps` method. If not defined, the default behavior when `num_inference_steps` is
1271
+ passed will be used. Must be in descending order.
1272
+ sigmas (`List[float]`, *optional*):
1273
+ Custom sigmas to use for the denoising process with schedulers which support a `sigmas` argument in
1274
+ their `set_timesteps` method. If not defined, the default behavior when `num_inference_steps` is passed
1275
+ will be used.
1276
+ denoising_start (`float`, *optional*):
1277
+ When specified, indicates the fraction (between 0.0 and 1.0) of the total denoising process to be
1278
+ bypassed before it is initiated. Consequently, the initial part of the denoising process is skipped and
1279
+ it is assumed that the passed `image` is a partly denoised image. Note that when this is specified,
1280
+ strength will be ignored. The `denoising_start` parameter is particularly beneficial when this pipeline
1281
+ is integrated into a "Mixture of Denoisers" multi-pipeline setup, as detailed in [**Refining the Image
1282
+ Output**](https://huggingface.co/docs/diffusers/api/pipelines/stable_diffusion/stable_diffusion_xl#refining-the-image-output).
1283
+ denoising_end (`float`, *optional*):
1284
+ When specified, determines the fraction (between 0.0 and 1.0) of the total denoising process to be
1285
+ completed before it is intentionally prematurely terminated. As a result, the returned sample will
1286
+ still retain a substantial amount of noise (ca. final 20% of timesteps still needed) and should be
1287
+ denoised by a successor pipeline that has `denoising_start` set to 0.8 so that it only denoises the
1288
+ final 20% of the scheduler. The denoising_end parameter should ideally be utilized when this pipeline
1289
+ forms a part of a "Mixture of Denoisers" multi-pipeline setup, as elaborated in [**Refining the Image
1290
+ Output**](https://huggingface.co/docs/diffusers/api/pipelines/stable_diffusion/stable_diffusion_xl#refining-the-image-output).
1291
+ guidance_scale (`float`, *optional*, defaults to 7.5):
1292
+ Guidance scale as defined in [Classifier-Free Diffusion Guidance](https://arxiv.org/abs/2207.12598).
1293
+ `guidance_scale` is defined as `w` of equation 2. of [Imagen
1294
+ Paper](https://arxiv.org/pdf/2205.11487.pdf). Guidance scale is enabled by setting `guidance_scale >
1295
+ 1`. Higher guidance scale encourages to generate images that are closely linked to the text `prompt`,
1296
+ usually at the expense of lower image quality.
1297
+ negative_prompt (`str` or `List[str]`, *optional*):
1298
+ The prompt or prompts not to guide the image generation. If not defined, one has to pass
1299
+ `negative_prompt_embeds` instead. Ignored when not using guidance (i.e., ignored if `guidance_scale` is
1300
+ less than `1`).
1301
+ negative_prompt_2 (`str` or `List[str]`, *optional*):
1302
+ The prompt or prompts not to guide the image generation to be sent to `tokenizer_2` and
1303
+ `text_encoder_2`. If not defined, `negative_prompt` is used in both text-encoders
1304
+ prompt_embeds (`torch.Tensor`, *optional*):
1305
+ Pre-generated text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt weighting. If not
1306
+ provided, text embeddings will be generated from `prompt` input argument.
1307
+ negative_prompt_embeds (`torch.Tensor`, *optional*):
1308
+ Pre-generated negative text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt
1309
+ weighting. If not provided, negative_prompt_embeds will be generated from `negative_prompt` input
1310
+ argument.
1311
+ pooled_prompt_embeds (`torch.Tensor`, *optional*):
1312
+ Pre-generated pooled text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt weighting.
1313
+ If not provided, pooled text embeddings will be generated from `prompt` input argument.
1314
+ negative_pooled_prompt_embeds (`torch.Tensor`, *optional*):
1315
+ Pre-generated negative pooled text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt
1316
+ weighting. If not provided, pooled negative_prompt_embeds will be generated from `negative_prompt`
1317
+ input argument.
1318
+ ip_adapter_image: (`PipelineImageInput`, *optional*): Optional image input to work with IP Adapters.
1319
+ ip_adapter_image_embeds (`List[torch.Tensor]`, *optional*):
1320
+ Pre-generated image embeddings for IP-Adapter. It should be a list of length same as number of
1321
+ IP-adapters. Each element should be a tensor of shape `(batch_size, num_images, emb_dim)`. It should
1322
+ contain the negative image embedding if `do_classifier_free_guidance` is set to `True`. If not
1323
+ provided, embeddings are computed from the `ip_adapter_image` input argument.
1324
+ num_images_per_prompt (`int`, *optional*, defaults to 1):
1325
+ The number of images to generate per prompt.
1326
+ eta (`float`, *optional*, defaults to 0.0):
1327
+ Corresponds to parameter eta (η) in the DDIM paper: https://arxiv.org/abs/2010.02502. Only applies to
1328
+ [`schedulers.DDIMScheduler`], will be ignored for others.
1329
+ generator (`torch.Generator`, *optional*):
1330
+ One or a list of [torch generator(s)](https://pytorch.org/docs/stable/generated/torch.Generator.html)
1331
+ to make generation deterministic.
1332
+ latents (`torch.Tensor`, *optional*):
1333
+ Pre-generated noisy latents, sampled from a Gaussian distribution, to be used as inputs for image
1334
+ generation. Can be used to tweak the same generation with different prompts. If not provided, a latents
1335
+ tensor will ge generated by sampling using the supplied random `generator`.
1336
+ output_type (`str`, *optional*, defaults to `"pil"`):
1337
+ The output format of the generate image. Choose between
1338
+ [PIL](https://pillow.readthedocs.io/en/stable/): `PIL.Image.Image` or `np.array`.
1339
+ return_dict (`bool`, *optional*, defaults to `True`):
1340
+ Whether or not to return a [`~pipelines.stable_diffusion.StableDiffusionPipelineOutput`] instead of a
1341
+ plain tuple.
1342
+ cross_attention_kwargs (`dict`, *optional*):
1343
+ A kwargs dictionary that if specified is passed along to the `AttentionProcessor` as defined under
1344
+ `self.processor` in
1345
+ [diffusers.models.attention_processor](https://github.com/huggingface/diffusers/blob/main/src/diffusers/models/attention_processor.py).
1346
+ original_size (`Tuple[int]`, *optional*, defaults to (1024, 1024)):
1347
+ If `original_size` is not the same as `target_size` the image will appear to be down- or upsampled.
1348
+ `original_size` defaults to `(height, width)` if not specified. Part of SDXL's micro-conditioning as
1349
+ explained in section 2.2 of
1350
+ [https://huggingface.co/papers/2307.01952](https://huggingface.co/papers/2307.01952).
1351
+ crops_coords_top_left (`Tuple[int]`, *optional*, defaults to (0, 0)):
1352
+ `crops_coords_top_left` can be used to generate an image that appears to be "cropped" from the position
1353
+ `crops_coords_top_left` downwards. Favorable, well-centered images are usually achieved by setting
1354
+ `crops_coords_top_left` to (0, 0). Part of SDXL's micro-conditioning as explained in section 2.2 of
1355
+ [https://huggingface.co/papers/2307.01952](https://huggingface.co/papers/2307.01952).
1356
+ target_size (`Tuple[int]`, *optional*, defaults to (1024, 1024)):
1357
+ For most cases, `target_size` should be set to the desired height and width of the generated image. If
1358
+ not specified it will default to `(height, width)`. Part of SDXL's micro-conditioning as explained in
1359
+ section 2.2 of [https://huggingface.co/papers/2307.01952](https://huggingface.co/papers/2307.01952).
1360
+ negative_original_size (`Tuple[int]`, *optional*, defaults to (1024, 1024)):
1361
+ To negatively condition the generation process based on a specific image resolution. Part of SDXL's
1362
+ micro-conditioning as explained in section 2.2 of
1363
+ [https://huggingface.co/papers/2307.01952](https://huggingface.co/papers/2307.01952). For more
1364
+ information, refer to this issue thread: https://github.com/huggingface/diffusers/issues/4208.
1365
+ negative_crops_coords_top_left (`Tuple[int]`, *optional*, defaults to (0, 0)):
1366
+ To negatively condition the generation process based on a specific crop coordinates. Part of SDXL's
1367
+ micro-conditioning as explained in section 2.2 of
1368
+ [https://huggingface.co/papers/2307.01952](https://huggingface.co/papers/2307.01952). For more
1369
+ information, refer to this issue thread: https://github.com/huggingface/diffusers/issues/4208.
1370
+ negative_target_size (`Tuple[int]`, *optional*, defaults to (1024, 1024)):
1371
+ To negatively condition the generation process based on a target image resolution. It should be as same
1372
+ as the `target_size` for most cases. Part of SDXL's micro-conditioning as explained in section 2.2 of
1373
+ [https://huggingface.co/papers/2307.01952](https://huggingface.co/papers/2307.01952). For more
1374
+ information, refer to this issue thread: https://github.com/huggingface/diffusers/issues/4208.
1375
+ aesthetic_score (`float`, *optional*, defaults to 6.0):
1376
+ Used to simulate an aesthetic score of the generated image by influencing the positive text condition.
1377
+ Part of SDXL's micro-conditioning as explained in section 2.2 of
1378
+ [https://huggingface.co/papers/2307.01952](https://huggingface.co/papers/2307.01952).
1379
+ negative_aesthetic_score (`float`, *optional*, defaults to 2.5):
1380
+ Part of SDXL's micro-conditioning as explained in section 2.2 of
1381
+ [https://huggingface.co/papers/2307.01952](https://huggingface.co/papers/2307.01952). Can be used to
1382
+ simulate an aesthetic score of the generated image by influencing the negative text condition.
1383
+ clip_skip (`int`, *optional*):
1384
+ Number of layers to be skipped from CLIP while computing the prompt embeddings. A value of 1 means that
1385
+ the output of the pre-final layer will be used for computing the prompt embeddings.
1386
+ callback_on_step_end (`Callable`, `PipelineCallback`, `MultiPipelineCallbacks`, *optional*):
1387
+ A function or a subclass of `PipelineCallback` or `MultiPipelineCallbacks` that is called at the end of
1388
+ each denoising step during the inference. with the following arguments: `callback_on_step_end(self:
1389
+ DiffusionPipeline, step: int, timestep: int, callback_kwargs: Dict)`. `callback_kwargs` will include a
1390
+ list of all tensors as specified by `callback_on_step_end_tensor_inputs`.
1391
+ callback_on_step_end_tensor_inputs (`List`, *optional*):
1392
+ The list of tensor inputs for the `callback_on_step_end` function. The tensors specified in the list
1393
+ will be passed as `callback_kwargs` argument. You will only be able to include variables listed in the
1394
+ `._callback_tensor_inputs` attribute of your pipeline class.
1395
+
1396
+ Examples:
1397
+
1398
+ Returns:
1399
+ [`~pipelines.stable_diffusion.StableDiffusionXLPipelineOutput`] or `tuple`:
1400
+ [`~pipelines.stable_diffusion.StableDiffusionXLPipelineOutput`] if `return_dict` is True, otherwise a
1401
+ `tuple. `tuple. When returning a tuple, the first element is a list with the generated images.
1402
+ """
1403
+ height, width = self._default_height_width(height, width, adapter_image)
1404
+ device = self._execution_device
1405
+
1406
+ if isinstance(self.adapter, MultiAdapter):
1407
+ adapter_input = []
1408
+
1409
+ for one_image in adapter_image:
1410
+ one_image = _preprocess_adapter_image(one_image, height, width)
1411
+ one_image = one_image.to(device=device, dtype=self.adapter.dtype)
1412
+ adapter_input.append(one_image)
1413
+ else:
1414
+ adapter_input = _preprocess_adapter_image(adapter_image, height, width)
1415
+ adapter_input = adapter_input.to(device=device, dtype=self.adapter.dtype)
1416
+
1417
+ callback = kwargs.pop("callback", None)
1418
+ callback_steps = kwargs.pop("callback_steps", None)
1419
+
1420
+ if callback is not None:
1421
+ deprecate(
1422
+ "callback",
1423
+ "1.0.0",
1424
+ "Passing `callback` as an input argument to `__call__` is deprecated, consider use `callback_on_step_end`",
1425
+ )
1426
+ if callback_steps is not None:
1427
+ deprecate(
1428
+ "callback_steps",
1429
+ "1.0.0",
1430
+ "Passing `callback_steps` as an input argument to `__call__` is deprecated, consider use `callback_on_step_end`",
1431
+ )
1432
+
1433
+ if isinstance(callback_on_step_end, (PipelineCallback, MultiPipelineCallbacks)):
1434
+ callback_on_step_end_tensor_inputs = callback_on_step_end.tensor_inputs
1435
+
1436
+ # 0. Default height and width to unet
1437
+ height = height or self.unet.config.sample_size * self.vae_scale_factor
1438
+ width = width or self.unet.config.sample_size * self.vae_scale_factor
1439
+
1440
+ # 1. Check inputs
1441
+ self.check_inputs(
1442
+ prompt,
1443
+ prompt_2,
1444
+ image,
1445
+ mask_image,
1446
+ height,
1447
+ width,
1448
+ strength,
1449
+ callback_steps,
1450
+ output_type,
1451
+ negative_prompt,
1452
+ negative_prompt_2,
1453
+ prompt_embeds,
1454
+ negative_prompt_embeds,
1455
+ ip_adapter_image,
1456
+ ip_adapter_image_embeds,
1457
+ callback_on_step_end_tensor_inputs,
1458
+ padding_mask_crop,
1459
+ )
1460
+
1461
+ self._guidance_scale = guidance_scale
1462
+ self._guidance_rescale = guidance_rescale
1463
+ self._clip_skip = clip_skip
1464
+ self._cross_attention_kwargs = cross_attention_kwargs
1465
+ self._denoising_end = denoising_end
1466
+ self._denoising_start = denoising_start
1467
+ self._interrupt = False
1468
+
1469
+ # 2. Define call parameters
1470
+ if prompt is not None and isinstance(prompt, str):
1471
+ batch_size = 1
1472
+ elif prompt is not None and isinstance(prompt, list):
1473
+ batch_size = len(prompt)
1474
+ else:
1475
+ batch_size = prompt_embeds.shape[0]
1476
+
1477
+ device = self._execution_device
1478
+
1479
+ # 3. Encode input prompt
1480
+ text_encoder_lora_scale = (
1481
+ self.cross_attention_kwargs.get("scale", None) if self.cross_attention_kwargs is not None else None
1482
+ )
1483
+
1484
+ (
1485
+ prompt_embeds,
1486
+ negative_prompt_embeds,
1487
+ pooled_prompt_embeds,
1488
+ negative_pooled_prompt_embeds,
1489
+ ) = self.encode_prompt(
1490
+ prompt=prompt,
1491
+ prompt_2=prompt_2,
1492
+ device=device,
1493
+ num_images_per_prompt=num_images_per_prompt,
1494
+ do_classifier_free_guidance=self.do_classifier_free_guidance,
1495
+ negative_prompt=negative_prompt,
1496
+ negative_prompt_2=negative_prompt_2,
1497
+ prompt_embeds=prompt_embeds,
1498
+ negative_prompt_embeds=negative_prompt_embeds,
1499
+ pooled_prompt_embeds=pooled_prompt_embeds,
1500
+ negative_pooled_prompt_embeds=negative_pooled_prompt_embeds,
1501
+ lora_scale=text_encoder_lora_scale,
1502
+ clip_skip=self.clip_skip,
1503
+ )
1504
+
1505
+ # 4. set timesteps
1506
+ def denoising_value_valid(dnv):
1507
+ return isinstance(dnv, float) and 0 < dnv < 1
1508
+
1509
+ timesteps, num_inference_steps = retrieve_timesteps(
1510
+ self.scheduler, num_inference_steps, device, timesteps, sigmas
1511
+ )
1512
+ timesteps, num_inference_steps = self.get_timesteps(
1513
+ num_inference_steps,
1514
+ strength,
1515
+ device,
1516
+ denoising_start=self.denoising_start if denoising_value_valid(self.denoising_start) else None,
1517
+ )
1518
+ # check that number of inference steps is not < 1 - as this doesn't make sense
1519
+ if num_inference_steps < 1:
1520
+ raise ValueError(
1521
+ f"After adjusting the num_inference_steps by strength parameter: {strength}, the number of pipeline"
1522
+ f"steps is {num_inference_steps} which is < 1 and not appropriate for this pipeline."
1523
+ )
1524
+ # at which timestep to set the initial noise (n.b. 50% if strength is 0.5)
1525
+ latent_timestep = timesteps[:1].repeat(batch_size * num_images_per_prompt)
1526
+ # create a boolean to check if the strength is set to 1. if so then initialise the latents with pure noise
1527
+ is_strength_max = strength == 1.0
1528
+
1529
+ # 5. Preprocess mask and image
1530
+ if padding_mask_crop is not None:
1531
+ crops_coords = self.mask_processor.get_crop_region(mask_image, width, height, pad=padding_mask_crop)
1532
+ resize_mode = "fill"
1533
+ else:
1534
+ crops_coords = None
1535
+ resize_mode = "default"
1536
+
1537
+ original_image = image
1538
+ init_image = self.image_processor.preprocess(
1539
+ image, height=height, width=width, crops_coords=crops_coords, resize_mode=resize_mode
1540
+ )
1541
+ init_image = init_image.to(dtype=torch.float32)
1542
+
1543
+ mask = self.mask_processor.preprocess(
1544
+ mask_image, height=height, width=width, resize_mode=resize_mode, crops_coords=crops_coords
1545
+ )
1546
+
1547
+ if masked_image_latents is not None:
1548
+ masked_image = masked_image_latents
1549
+ elif init_image.shape[1] == 4:
1550
+ # if images are in latent space, we can't mask it
1551
+ masked_image = None
1552
+ else:
1553
+ masked_image = init_image * (mask < 0.5)
1554
+
1555
+ # 6. Prepare latent variables
1556
+ num_channels_latents = self.vae.config.latent_channels
1557
+ num_channels_unet = self.unet.config.in_channels
1558
+ return_image_latents = num_channels_unet == 4
1559
+
1560
+ add_noise = True if self.denoising_start is None else False
1561
+ latents_outputs = self.prepare_latents(
1562
+ batch_size * num_images_per_prompt,
1563
+ num_channels_latents,
1564
+ height,
1565
+ width,
1566
+ prompt_embeds.dtype,
1567
+ device,
1568
+ generator,
1569
+ latents,
1570
+ image=init_image,
1571
+ timestep=latent_timestep,
1572
+ is_strength_max=is_strength_max,
1573
+ add_noise=add_noise,
1574
+ return_noise=True,
1575
+ return_image_latents=return_image_latents,
1576
+ )
1577
+
1578
+ if return_image_latents:
1579
+ latents, noise, image_latents = latents_outputs
1580
+ else:
1581
+ latents, noise = latents_outputs
1582
+
1583
+ # 7. Prepare mask latent variables
1584
+ mask, masked_image_latents = self.prepare_mask_latents(
1585
+ mask,
1586
+ masked_image,
1587
+ batch_size * num_images_per_prompt,
1588
+ height,
1589
+ width,
1590
+ prompt_embeds.dtype,
1591
+ device,
1592
+ generator,
1593
+ self.do_classifier_free_guidance,
1594
+ )
1595
+
1596
+ # 8. Check that sizes of mask, masked image and latents match
1597
+ if num_channels_unet == 9:
1598
+ # default case for runwayml/stable-diffusion-inpainting
1599
+ num_channels_mask = mask.shape[1]
1600
+ num_channels_masked_image = masked_image_latents.shape[1]
1601
+ if num_channels_latents + num_channels_mask + num_channels_masked_image != self.unet.config.in_channels:
1602
+ raise ValueError(
1603
+ f"Incorrect configuration settings! The config of `pipeline.unet`: {self.unet.config} expects"
1604
+ f" {self.unet.config.in_channels} but received `num_channels_latents`: {num_channels_latents} +"
1605
+ f" `num_channels_mask`: {num_channels_mask} + `num_channels_masked_image`: {num_channels_masked_image}"
1606
+ f" = {num_channels_latents+num_channels_masked_image+num_channels_mask}. Please verify the config of"
1607
+ " `pipeline.unet` or your `mask_image` or `image` input."
1608
+ )
1609
+ elif num_channels_unet != 4:
1610
+ raise ValueError(
1611
+ f"The unet {self.unet.__class__} should have either 4 or 9 input channels, not {self.unet.config.in_channels}."
1612
+ )
1613
+ # 8.1 Prepare extra step kwargs.
1614
+ extra_step_kwargs = self.prepare_extra_step_kwargs(generator, eta)
1615
+
1616
+ # 9. Prepare extra step kwargs. TODO: Logic should ideally just be moved out of the pipeline
1617
+ height, width = latents.shape[-2:]
1618
+ height = height * self.vae_scale_factor
1619
+ width = width * self.vae_scale_factor
1620
+
1621
+ original_size = original_size or (height, width)
1622
+ target_size = target_size or (height, width)
1623
+
1624
+ # 10. Prepare added time ids & embeddings
1625
+ if isinstance(self.adapter, MultiAdapter):
1626
+ adapter_state = self.adapter(adapter_input, adapter_conditioning_scale)
1627
+ for k, v in enumerate(adapter_state):
1628
+ adapter_state[k] = v
1629
+ else:
1630
+ adapter_state = self.adapter(adapter_input)
1631
+ for k, v in enumerate(adapter_state):
1632
+ adapter_state[k] = v * adapter_conditioning_scale
1633
+ if num_images_per_prompt > 1:
1634
+ for k, v in enumerate(adapter_state):
1635
+ adapter_state[k] = v.repeat(num_images_per_prompt, 1, 1, 1)
1636
+ if self.do_classifier_free_guidance:
1637
+ for k, v in enumerate(adapter_state):
1638
+ adapter_state[k] = torch.cat([v] * 2, dim=0)
1639
+
1640
+ if negative_original_size is None:
1641
+ negative_original_size = original_size
1642
+ if negative_target_size is None:
1643
+ negative_target_size = target_size
1644
+
1645
+ add_text_embeds = pooled_prompt_embeds
1646
+ if self.text_encoder_2 is None:
1647
+ text_encoder_projection_dim = int(pooled_prompt_embeds.shape[-1])
1648
+ else:
1649
+ text_encoder_projection_dim = self.text_encoder_2.config.projection_dim
1650
+
1651
+ add_time_ids, add_neg_time_ids = self._get_add_time_ids(
1652
+ original_size,
1653
+ crops_coords_top_left,
1654
+ target_size,
1655
+ aesthetic_score,
1656
+ negative_aesthetic_score,
1657
+ negative_original_size,
1658
+ negative_crops_coords_top_left,
1659
+ negative_target_size,
1660
+ dtype=prompt_embeds.dtype,
1661
+ text_encoder_projection_dim=text_encoder_projection_dim,
1662
+ )
1663
+ add_time_ids = add_time_ids.repeat(batch_size * num_images_per_prompt, 1)
1664
+
1665
+ if self.do_classifier_free_guidance:
1666
+ prompt_embeds = torch.cat([negative_prompt_embeds, prompt_embeds], dim=0)
1667
+ add_text_embeds = torch.cat([negative_pooled_prompt_embeds, add_text_embeds], dim=0)
1668
+ add_neg_time_ids = add_neg_time_ids.repeat(batch_size * num_images_per_prompt, 1)
1669
+ add_time_ids = torch.cat([add_neg_time_ids, add_time_ids], dim=0)
1670
+
1671
+ prompt_embeds = prompt_embeds.to(device)
1672
+ add_text_embeds = add_text_embeds.to(device)
1673
+ add_time_ids = add_time_ids.to(device)
1674
+
1675
+ if ip_adapter_image is not None or ip_adapter_image_embeds is not None:
1676
+ image_embeds = self.prepare_ip_adapter_image_embeds(
1677
+ ip_adapter_image,
1678
+ ip_adapter_image_embeds,
1679
+ device,
1680
+ batch_size * num_images_per_prompt,
1681
+ self.do_classifier_free_guidance,
1682
+ )
1683
+
1684
+ # 11. Denoising loop
1685
+ num_warmup_steps = max(len(timesteps) - num_inference_steps * self.scheduler.order, 0)
1686
+
1687
+ if (
1688
+ self.denoising_end is not None
1689
+ and self.denoising_start is not None
1690
+ and denoising_value_valid(self.denoising_end)
1691
+ and denoising_value_valid(self.denoising_start)
1692
+ and self.denoising_start >= self.denoising_end
1693
+ ):
1694
+ raise ValueError(
1695
+ f"`denoising_start`: {self.denoising_start} cannot be larger than or equal to `denoising_end`: "
1696
+ + f" {self.denoising_end} when using type float."
1697
+ )
1698
+ elif self.denoising_end is not None and denoising_value_valid(self.denoising_end):
1699
+ discrete_timestep_cutoff = int(
1700
+ round(
1701
+ self.scheduler.config.num_train_timesteps
1702
+ - (self.denoising_end * self.scheduler.config.num_train_timesteps)
1703
+ )
1704
+ )
1705
+ num_inference_steps = len(list(filter(lambda ts: ts >= discrete_timestep_cutoff, timesteps)))
1706
+ timesteps = timesteps[:num_inference_steps]
1707
+
1708
+ # 11.1 Optionally get Guidance Scale Embedding
1709
+ timestep_cond = None
1710
+ if self.unet.config.time_cond_proj_dim is not None:
1711
+ guidance_scale_tensor = torch.tensor(self.guidance_scale - 1).repeat(batch_size * num_images_per_prompt)
1712
+ timestep_cond = self.get_guidance_scale_embedding(
1713
+ guidance_scale_tensor, embedding_dim=self.unet.config.time_cond_proj_dim
1714
+ ).to(device=device, dtype=latents.dtype)
1715
+
1716
+ self._num_timesteps = len(timesteps)
1717
+ with self.progress_bar(total=num_inference_steps) as progress_bar:
1718
+ for i, t in enumerate(timesteps):
1719
+ if self.interrupt:
1720
+ continue
1721
+ # expand the latents if we are doing classifier free guidance
1722
+ latent_model_input = torch.cat([latents] * 2) if self.do_classifier_free_guidance else latents
1723
+
1724
+ # concat latents, mask, masked_image_latents in the channel dimension
1725
+ latent_model_input = self.scheduler.scale_model_input(latent_model_input, t)
1726
+
1727
+ if num_channels_unet == 9:
1728
+ latent_model_input = torch.cat([latent_model_input, mask, masked_image_latents], dim=1)
1729
+
1730
+ # predict the noise residual
1731
+ added_cond_kwargs = {"text_embeds": add_text_embeds, "time_ids": add_time_ids}
1732
+ if ip_adapter_image is not None or ip_adapter_image_embeds is not None:
1733
+ added_cond_kwargs["image_embeds"] = image_embeds
1734
+
1735
+ if i < int(num_inference_steps * adapter_conditioning_factor):
1736
+ down_intrablock_additional_residuals = [state.clone() for state in adapter_state]
1737
+ else:
1738
+ down_intrablock_additional_residuals = None
1739
+
1740
+ noise_pred = self.unet(
1741
+ latent_model_input,
1742
+ t,
1743
+ encoder_hidden_states=prompt_embeds,
1744
+ timestep_cond=timestep_cond,
1745
+ cross_attention_kwargs=self.cross_attention_kwargs,
1746
+ added_cond_kwargs=added_cond_kwargs,
1747
+ return_dict=False,
1748
+ down_intrablock_additional_residuals=down_intrablock_additional_residuals,
1749
+ )[0]
1750
+
1751
+ # perform guidance
1752
+ if self.do_classifier_free_guidance:
1753
+ noise_pred_uncond, noise_pred_text = noise_pred.chunk(2)
1754
+ noise_pred = noise_pred_uncond + self.guidance_scale * (noise_pred_text - noise_pred_uncond)
1755
+
1756
+ if self.do_classifier_free_guidance and self.guidance_rescale > 0.0:
1757
+ # Based on 3.4. in https://arxiv.org/pdf/2305.08891.pdf
1758
+ noise_pred = rescale_noise_cfg(noise_pred, noise_pred_text, guidance_rescale=self.guidance_rescale)
1759
+
1760
+ # compute the previous noisy sample x_t -> x_t-1
1761
+ latents_dtype = latents.dtype
1762
+ latents = self.scheduler.step(noise_pred, t, latents, **extra_step_kwargs, return_dict=False)[0]
1763
+ if latents.dtype != latents_dtype:
1764
+ if torch.backends.mps.is_available():
1765
+ # some platforms (eg. apple mps) misbehave due to a pytorch bug: https://github.com/pytorch/pytorch/pull/99272
1766
+ latents = latents.to(latents_dtype)
1767
+
1768
+ if num_channels_unet == 4:
1769
+ init_latents_proper = image_latents
1770
+ if self.do_classifier_free_guidance:
1771
+ init_mask, _ = mask.chunk(2)
1772
+ else:
1773
+ init_mask = mask
1774
+
1775
+ if i < len(timesteps) - 1:
1776
+ noise_timestep = timesteps[i + 1]
1777
+ init_latents_proper = self.scheduler.add_noise(
1778
+ init_latents_proper, noise, torch.tensor([noise_timestep])
1779
+ )
1780
+
1781
+ latents = (1 - init_mask) * init_latents_proper + init_mask * latents
1782
+
1783
+ if callback_on_step_end is not None:
1784
+ callback_kwargs = {}
1785
+ for k in callback_on_step_end_tensor_inputs:
1786
+ callback_kwargs[k] = locals()[k]
1787
+ callback_outputs = callback_on_step_end(self, i, t, callback_kwargs)
1788
+
1789
+ latents = callback_outputs.pop("latents", latents)
1790
+ prompt_embeds = callback_outputs.pop("prompt_embeds", prompt_embeds)
1791
+ negative_prompt_embeds = callback_outputs.pop("negative_prompt_embeds", negative_prompt_embeds)
1792
+ add_text_embeds = callback_outputs.pop("add_text_embeds", add_text_embeds)
1793
+ negative_pooled_prompt_embeds = callback_outputs.pop(
1794
+ "negative_pooled_prompt_embeds", negative_pooled_prompt_embeds
1795
+ )
1796
+ add_time_ids = callback_outputs.pop("add_time_ids", add_time_ids)
1797
+ add_neg_time_ids = callback_outputs.pop("add_neg_time_ids", add_neg_time_ids)
1798
+ mask = callback_outputs.pop("mask", mask)
1799
+ masked_image_latents = callback_outputs.pop("masked_image_latents", masked_image_latents)
1800
+
1801
+ # call the callback, if provided
1802
+ if i == len(timesteps) - 1 or ((i + 1) > num_warmup_steps and (i + 1) % self.scheduler.order == 0):
1803
+ progress_bar.update()
1804
+ if callback is not None and i % callback_steps == 0:
1805
+ step_idx = i // getattr(self.scheduler, "order", 1)
1806
+ callback(step_idx, t, latents)
1807
+
1808
+ if XLA_AVAILABLE:
1809
+ xm.mark_step()
1810
+
1811
+ if not output_type == "latent":
1812
+ # make sure the VAE is in float32 mode, as it overflows in float16
1813
+ needs_upcasting = self.vae.dtype == torch.float16 and self.vae.config.force_upcast
1814
+
1815
+ if needs_upcasting:
1816
+ self.upcast_vae()
1817
+ latents = latents.to(next(iter(self.vae.post_quant_conv.parameters())).dtype)
1818
+ elif latents.dtype != self.vae.dtype:
1819
+ if torch.backends.mps.is_available():
1820
+ # some platforms (eg. apple mps) misbehave due to a pytorch bug: https://github.com/pytorch/pytorch/pull/99272
1821
+ self.vae = self.vae.to(latents.dtype)
1822
+
1823
+ # unscale/denormalize the latents
1824
+ # denormalize with the mean and std if available and not None
1825
+ has_latents_mean = hasattr(self.vae.config, "latents_mean") and self.vae.config.latents_mean is not None
1826
+ has_latents_std = hasattr(self.vae.config, "latents_std") and self.vae.config.latents_std is not None
1827
+ if has_latents_mean and has_latents_std:
1828
+ latents_mean = (
1829
+ torch.tensor(self.vae.config.latents_mean).view(1, 4, 1, 1).to(latents.device, latents.dtype)
1830
+ )
1831
+ latents_std = (
1832
+ torch.tensor(self.vae.config.latents_std).view(1, 4, 1, 1).to(latents.device, latents.dtype)
1833
+ )
1834
+ latents = latents * latents_std / self.vae.config.scaling_factor + latents_mean
1835
+ else:
1836
+ latents = latents / self.vae.config.scaling_factor
1837
+
1838
+ image = self.vae.decode(latents, return_dict=False)[0]
1839
+
1840
+ # cast back to fp16 if needed
1841
+ if needs_upcasting:
1842
+ self.vae.to(dtype=torch.float16)
1843
+ else:
1844
+ return StableDiffusionXLPipelineOutput(images=latents)
1845
+
1846
+ # apply watermark if available
1847
+ if self.watermark is not None:
1848
+ image = self.watermark.apply_watermark(image)
1849
+
1850
+ image = self.image_processor.postprocess(image, output_type=output_type)
1851
+
1852
+ if padding_mask_crop is not None:
1853
+ image = [self.image_processor.apply_overlay(mask_image, original_image, i, crops_coords) for i in image]
1854
+
1855
+ # Offload all models
1856
+ self.maybe_free_model_hooks()
1857
+
1858
+ if not return_dict:
1859
+ return (image,)
1860
+
1861
+ return StableDiffusionXLPipelineOutput(images=image)