Spaces:
Running
on
Zero
Running
on
Zero
zhiweili
commited on
Commit
•
f0a547a
1
Parent(s):
02c0c4b
change inppaint
Browse files- app.py +1 -1
- app_haircolor_inpaint.py +5 -5
- pipelines/pipeline_sdxl_adapter_inpaint_custom.py +1861 -0
app.py
CHANGED
@@ -1,6 +1,6 @@
|
|
1 |
import gradio as gr
|
2 |
|
3 |
-
from
|
4 |
|
5 |
with gr.Blocks(css="style.css") as demo:
|
6 |
with gr.Tabs():
|
|
|
1 |
import gradio as gr
|
2 |
|
3 |
+
from app_haircolor_inpaint import create_demo as create_demo_haircolor
|
4 |
|
5 |
with gr.Blocks(css="style.css") as demo:
|
6 |
with gr.Tabs():
|
app_haircolor_inpaint.py
CHANGED
@@ -19,8 +19,8 @@ from controlnet_aux import (
|
|
19 |
CannyDetector,
|
20 |
)
|
21 |
|
22 |
-
BASE_MODEL = "
|
23 |
-
|
24 |
DEVICE = "cuda" if torch.cuda.is_available() else "cpu"
|
25 |
|
26 |
DEFAULT_EDIT_PROMPT = "a woman, blue hair, high detailed"
|
@@ -54,7 +54,7 @@ basepipeline = DiffusionPipeline.from_pretrained(
|
|
54 |
torch_dtype=torch.float16,
|
55 |
use_safetensors=True,
|
56 |
adapter=adapters,
|
57 |
-
custom_pipeline="./pipelines/
|
58 |
)
|
59 |
|
60 |
basepipeline = basepipeline.to(DEVICE)
|
@@ -76,9 +76,9 @@ def image_to_image(
|
|
76 |
run_task_time = 0
|
77 |
time_cost_str = ''
|
78 |
run_task_time, time_cost_str = get_time_cost(run_task_time, time_cost_str)
|
79 |
-
lineart_image = lineart_detector(input_image,
|
80 |
run_task_time, time_cost_str = get_time_cost(run_task_time, time_cost_str)
|
81 |
-
canny_image = canndy_detector(input_image,
|
82 |
run_task_time, time_cost_str = get_time_cost(run_task_time, time_cost_str)
|
83 |
|
84 |
cond_image = [lineart_image, canny_image]
|
|
|
19 |
CannyDetector,
|
20 |
)
|
21 |
|
22 |
+
BASE_MODEL = "stabilityai/sdxl-turbo"
|
23 |
+
|
24 |
DEVICE = "cuda" if torch.cuda.is_available() else "cpu"
|
25 |
|
26 |
DEFAULT_EDIT_PROMPT = "a woman, blue hair, high detailed"
|
|
|
54 |
torch_dtype=torch.float16,
|
55 |
use_safetensors=True,
|
56 |
adapter=adapters,
|
57 |
+
custom_pipeline="./pipelines/pipeline_sdxl_adapter_inpaint_custom.py",
|
58 |
)
|
59 |
|
60 |
basepipeline = basepipeline.to(DEVICE)
|
|
|
76 |
run_task_time = 0
|
77 |
time_cost_str = ''
|
78 |
run_task_time, time_cost_str = get_time_cost(run_task_time, time_cost_str)
|
79 |
+
lineart_image = lineart_detector(input_image, int(generate_size*0.375), generate_size)
|
80 |
run_task_time, time_cost_str = get_time_cost(run_task_time, time_cost_str)
|
81 |
+
canny_image = canndy_detector(input_image, int(generate_size*0.375), generate_size)
|
82 |
run_task_time, time_cost_str = get_time_cost(run_task_time, time_cost_str)
|
83 |
|
84 |
cond_image = [lineart_image, canny_image]
|
pipelines/pipeline_sdxl_adapter_inpaint_custom.py
ADDED
@@ -0,0 +1,1861 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
# Copyright 2024 The HuggingFace Team. All rights reserved.
|
2 |
+
#
|
3 |
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
4 |
+
# you may not use this file except in compliance with the License.
|
5 |
+
# You may obtain a copy of the License at
|
6 |
+
#
|
7 |
+
# http://www.apache.org/licenses/LICENSE-2.0
|
8 |
+
#
|
9 |
+
# Unless required by applicable law or agreed to in writing, software
|
10 |
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
11 |
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
12 |
+
# See the License for the specific language governing permissions and
|
13 |
+
# limitations under the License.
|
14 |
+
|
15 |
+
import inspect
|
16 |
+
from typing import Any, Callable, Dict, List, Optional, Tuple, Union
|
17 |
+
|
18 |
+
import numpy as np
|
19 |
+
import PIL.Image
|
20 |
+
import torch
|
21 |
+
from transformers import (
|
22 |
+
CLIPImageProcessor,
|
23 |
+
CLIPTextModel,
|
24 |
+
CLIPTextModelWithProjection,
|
25 |
+
CLIPTokenizer,
|
26 |
+
CLIPVisionModelWithProjection,
|
27 |
+
)
|
28 |
+
|
29 |
+
from diffusers.callbacks import (
|
30 |
+
MultiPipelineCallbacks,
|
31 |
+
PipelineCallback,
|
32 |
+
)
|
33 |
+
|
34 |
+
from diffusers.image_processor import (
|
35 |
+
PipelineImageInput,
|
36 |
+
VaeImageProcessor,
|
37 |
+
)
|
38 |
+
|
39 |
+
from diffusers.loaders import (
|
40 |
+
FromSingleFileMixin,
|
41 |
+
IPAdapterMixin,
|
42 |
+
StableDiffusionXLLoraLoaderMixin,
|
43 |
+
TextualInversionLoaderMixin,
|
44 |
+
)
|
45 |
+
|
46 |
+
from diffusers.models import (
|
47 |
+
AutoencoderKL,
|
48 |
+
ImageProjection,
|
49 |
+
MultiAdapter,
|
50 |
+
T2IAdapter,
|
51 |
+
UNet2DConditionModel,
|
52 |
+
)
|
53 |
+
|
54 |
+
from diffusers.models.attention_processor import (
|
55 |
+
AttnProcessor2_0,
|
56 |
+
XFormersAttnProcessor,
|
57 |
+
)
|
58 |
+
|
59 |
+
from diffusers.models.lora import (
|
60 |
+
adjust_lora_scale_text_encoder,
|
61 |
+
)
|
62 |
+
|
63 |
+
from diffusers.schedulers import (
|
64 |
+
KarrasDiffusionSchedulers,
|
65 |
+
)
|
66 |
+
|
67 |
+
from diffusers.utils import (
|
68 |
+
PIL_INTERPOLATION,
|
69 |
+
USE_PEFT_BACKEND,
|
70 |
+
deprecate,
|
71 |
+
is_invisible_watermark_available,
|
72 |
+
is_torch_xla_available,
|
73 |
+
logging,
|
74 |
+
replace_example_docstring,
|
75 |
+
scale_lora_layers,
|
76 |
+
unscale_lora_layers,
|
77 |
+
)
|
78 |
+
|
79 |
+
from diffusers.utils.torch_utils import (
|
80 |
+
randn_tensor,
|
81 |
+
)
|
82 |
+
|
83 |
+
from diffusers.pipelines.pipeline_utils import (
|
84 |
+
DiffusionPipeline,
|
85 |
+
StableDiffusionMixin,
|
86 |
+
)
|
87 |
+
|
88 |
+
from diffusers.pipelines.stable_diffusion_xl.pipeline_output import (
|
89 |
+
StableDiffusionXLPipelineOutput,
|
90 |
+
)
|
91 |
+
|
92 |
+
if is_invisible_watermark_available():
|
93 |
+
from diffusers.pipelines.stable_diffusion_xl.watermark import (
|
94 |
+
StableDiffusionXLWatermarker,
|
95 |
+
)
|
96 |
+
|
97 |
+
if is_torch_xla_available():
|
98 |
+
import torch_xla.core.xla_model as xm
|
99 |
+
|
100 |
+
XLA_AVAILABLE = True
|
101 |
+
else:
|
102 |
+
XLA_AVAILABLE = False
|
103 |
+
|
104 |
+
|
105 |
+
logger = logging.get_logger(__name__) # pylint: disable=invalid-name
|
106 |
+
|
107 |
+
|
108 |
+
EXAMPLE_DOC_STRING = """
|
109 |
+
Examples:
|
110 |
+
```py
|
111 |
+
>>> import torch
|
112 |
+
>>> from diffusers import StableDiffusionXLInpaintPipeline
|
113 |
+
>>> from diffusers.utils import load_image
|
114 |
+
|
115 |
+
>>> pipe = StableDiffusionXLInpaintPipeline.from_pretrained(
|
116 |
+
... "stabilityai/stable-diffusion-xl-base-1.0",
|
117 |
+
... torch_dtype=torch.float16,
|
118 |
+
... variant="fp16",
|
119 |
+
... use_safetensors=True,
|
120 |
+
... )
|
121 |
+
>>> pipe.to("cuda")
|
122 |
+
|
123 |
+
>>> img_url = "https://raw.githubusercontent.com/CompVis/latent-diffusion/main/data/inpainting_examples/overture-creations-5sI6fQgYIuo.png"
|
124 |
+
>>> mask_url = "https://raw.githubusercontent.com/CompVis/latent-diffusion/main/data/inpainting_examples/overture-creations-5sI6fQgYIuo_mask.png"
|
125 |
+
|
126 |
+
>>> init_image = load_image(img_url).convert("RGB")
|
127 |
+
>>> mask_image = load_image(mask_url).convert("RGB")
|
128 |
+
|
129 |
+
>>> prompt = "A majestic tiger sitting on a bench"
|
130 |
+
>>> image = pipe(
|
131 |
+
... prompt=prompt, image=init_image, mask_image=mask_image, num_inference_steps=50, strength=0.80
|
132 |
+
... ).images[0]
|
133 |
+
```
|
134 |
+
"""
|
135 |
+
|
136 |
+
def _preprocess_adapter_image(image, height, width):
|
137 |
+
if isinstance(image, torch.Tensor):
|
138 |
+
return image
|
139 |
+
elif isinstance(image, PIL.Image.Image):
|
140 |
+
image = [image]
|
141 |
+
|
142 |
+
if isinstance(image[0], PIL.Image.Image):
|
143 |
+
image = [np.array(i.resize((width, height), resample=PIL_INTERPOLATION["lanczos"])) for i in image]
|
144 |
+
image = [
|
145 |
+
i[None, ..., None] if i.ndim == 2 else i[None, ...] for i in image
|
146 |
+
] # expand [h, w] or [h, w, c] to [b, h, w, c]
|
147 |
+
image = np.concatenate(image, axis=0)
|
148 |
+
image = np.array(image).astype(np.float32) / 255.0
|
149 |
+
image = image.transpose(0, 3, 1, 2)
|
150 |
+
image = torch.from_numpy(image)
|
151 |
+
elif isinstance(image[0], torch.Tensor):
|
152 |
+
if image[0].ndim == 3:
|
153 |
+
image = torch.stack(image, dim=0)
|
154 |
+
elif image[0].ndim == 4:
|
155 |
+
image = torch.cat(image, dim=0)
|
156 |
+
else:
|
157 |
+
raise ValueError(
|
158 |
+
f"Invalid image tensor! Expecting image tensor with 3 or 4 dimension, but recive: {image[0].ndim}"
|
159 |
+
)
|
160 |
+
return image
|
161 |
+
|
162 |
+
|
163 |
+
# Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.rescale_noise_cfg
|
164 |
+
def rescale_noise_cfg(noise_cfg, noise_pred_text, guidance_rescale=0.0):
|
165 |
+
"""
|
166 |
+
Rescale `noise_cfg` according to `guidance_rescale`. Based on findings of [Common Diffusion Noise Schedules and
|
167 |
+
Sample Steps are Flawed](https://arxiv.org/pdf/2305.08891.pdf). See Section 3.4
|
168 |
+
"""
|
169 |
+
std_text = noise_pred_text.std(dim=list(range(1, noise_pred_text.ndim)), keepdim=True)
|
170 |
+
std_cfg = noise_cfg.std(dim=list(range(1, noise_cfg.ndim)), keepdim=True)
|
171 |
+
# rescale the results from guidance (fixes overexposure)
|
172 |
+
noise_pred_rescaled = noise_cfg * (std_text / std_cfg)
|
173 |
+
# mix with the original results from guidance by factor guidance_rescale to avoid "plain looking" images
|
174 |
+
noise_cfg = guidance_rescale * noise_pred_rescaled + (1 - guidance_rescale) * noise_cfg
|
175 |
+
return noise_cfg
|
176 |
+
|
177 |
+
|
178 |
+
def mask_pil_to_torch(mask, height, width):
|
179 |
+
# preprocess mask
|
180 |
+
if isinstance(mask, (PIL.Image.Image, np.ndarray)):
|
181 |
+
mask = [mask]
|
182 |
+
|
183 |
+
if isinstance(mask, list) and isinstance(mask[0], PIL.Image.Image):
|
184 |
+
mask = [i.resize((width, height), resample=PIL.Image.LANCZOS) for i in mask]
|
185 |
+
mask = np.concatenate([np.array(m.convert("L"))[None, None, :] for m in mask], axis=0)
|
186 |
+
mask = mask.astype(np.float32) / 255.0
|
187 |
+
elif isinstance(mask, list) and isinstance(mask[0], np.ndarray):
|
188 |
+
mask = np.concatenate([m[None, None, :] for m in mask], axis=0)
|
189 |
+
|
190 |
+
mask = torch.from_numpy(mask)
|
191 |
+
return mask
|
192 |
+
|
193 |
+
|
194 |
+
# Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion_img2img.retrieve_latents
|
195 |
+
def retrieve_latents(
|
196 |
+
encoder_output: torch.Tensor, generator: Optional[torch.Generator] = None, sample_mode: str = "sample"
|
197 |
+
):
|
198 |
+
if hasattr(encoder_output, "latent_dist") and sample_mode == "sample":
|
199 |
+
return encoder_output.latent_dist.sample(generator)
|
200 |
+
elif hasattr(encoder_output, "latent_dist") and sample_mode == "argmax":
|
201 |
+
return encoder_output.latent_dist.mode()
|
202 |
+
elif hasattr(encoder_output, "latents"):
|
203 |
+
return encoder_output.latents
|
204 |
+
else:
|
205 |
+
raise AttributeError("Could not access latents of provided encoder_output")
|
206 |
+
|
207 |
+
|
208 |
+
# Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.retrieve_timesteps
|
209 |
+
def retrieve_timesteps(
|
210 |
+
scheduler,
|
211 |
+
num_inference_steps: Optional[int] = None,
|
212 |
+
device: Optional[Union[str, torch.device]] = None,
|
213 |
+
timesteps: Optional[List[int]] = None,
|
214 |
+
sigmas: Optional[List[float]] = None,
|
215 |
+
**kwargs,
|
216 |
+
):
|
217 |
+
"""
|
218 |
+
Calls the scheduler's `set_timesteps` method and retrieves timesteps from the scheduler after the call. Handles
|
219 |
+
custom timesteps. Any kwargs will be supplied to `scheduler.set_timesteps`.
|
220 |
+
|
221 |
+
Args:
|
222 |
+
scheduler (`SchedulerMixin`):
|
223 |
+
The scheduler to get timesteps from.
|
224 |
+
num_inference_steps (`int`):
|
225 |
+
The number of diffusion steps used when generating samples with a pre-trained model. If used, `timesteps`
|
226 |
+
must be `None`.
|
227 |
+
device (`str` or `torch.device`, *optional*):
|
228 |
+
The device to which the timesteps should be moved to. If `None`, the timesteps are not moved.
|
229 |
+
timesteps (`List[int]`, *optional*):
|
230 |
+
Custom timesteps used to override the timestep spacing strategy of the scheduler. If `timesteps` is passed,
|
231 |
+
`num_inference_steps` and `sigmas` must be `None`.
|
232 |
+
sigmas (`List[float]`, *optional*):
|
233 |
+
Custom sigmas used to override the timestep spacing strategy of the scheduler. If `sigmas` is passed,
|
234 |
+
`num_inference_steps` and `timesteps` must be `None`.
|
235 |
+
|
236 |
+
Returns:
|
237 |
+
`Tuple[torch.Tensor, int]`: A tuple where the first element is the timestep schedule from the scheduler and the
|
238 |
+
second element is the number of inference steps.
|
239 |
+
"""
|
240 |
+
if timesteps is not None and sigmas is not None:
|
241 |
+
raise ValueError("Only one of `timesteps` or `sigmas` can be passed. Please choose one to set custom values")
|
242 |
+
if timesteps is not None:
|
243 |
+
accepts_timesteps = "timesteps" in set(inspect.signature(scheduler.set_timesteps).parameters.keys())
|
244 |
+
if not accepts_timesteps:
|
245 |
+
raise ValueError(
|
246 |
+
f"The current scheduler class {scheduler.__class__}'s `set_timesteps` does not support custom"
|
247 |
+
f" timestep schedules. Please check whether you are using the correct scheduler."
|
248 |
+
)
|
249 |
+
scheduler.set_timesteps(timesteps=timesteps, device=device, **kwargs)
|
250 |
+
timesteps = scheduler.timesteps
|
251 |
+
num_inference_steps = len(timesteps)
|
252 |
+
elif sigmas is not None:
|
253 |
+
accept_sigmas = "sigmas" in set(inspect.signature(scheduler.set_timesteps).parameters.keys())
|
254 |
+
if not accept_sigmas:
|
255 |
+
raise ValueError(
|
256 |
+
f"The current scheduler class {scheduler.__class__}'s `set_timesteps` does not support custom"
|
257 |
+
f" sigmas schedules. Please check whether you are using the correct scheduler."
|
258 |
+
)
|
259 |
+
scheduler.set_timesteps(sigmas=sigmas, device=device, **kwargs)
|
260 |
+
timesteps = scheduler.timesteps
|
261 |
+
num_inference_steps = len(timesteps)
|
262 |
+
else:
|
263 |
+
scheduler.set_timesteps(num_inference_steps, device=device, **kwargs)
|
264 |
+
timesteps = scheduler.timesteps
|
265 |
+
return timesteps, num_inference_steps
|
266 |
+
|
267 |
+
|
268 |
+
class StableDiffusionXLInpaintPipeline(
|
269 |
+
DiffusionPipeline,
|
270 |
+
StableDiffusionMixin,
|
271 |
+
TextualInversionLoaderMixin,
|
272 |
+
StableDiffusionXLLoraLoaderMixin,
|
273 |
+
FromSingleFileMixin,
|
274 |
+
IPAdapterMixin,
|
275 |
+
):
|
276 |
+
r"""
|
277 |
+
Pipeline for text-to-image generation using Stable Diffusion XL.
|
278 |
+
|
279 |
+
This model inherits from [`DiffusionPipeline`]. Check the superclass documentation for the generic methods the
|
280 |
+
library implements for all the pipelines (such as downloading or saving, running on a particular device, etc.)
|
281 |
+
|
282 |
+
The pipeline also inherits the following loading methods:
|
283 |
+
- [`~loaders.TextualInversionLoaderMixin.load_textual_inversion`] for loading textual inversion embeddings
|
284 |
+
- [`~loaders.FromSingleFileMixin.from_single_file`] for loading `.ckpt` files
|
285 |
+
- [`~loaders.StableDiffusionXLLoraLoaderMixin.load_lora_weights`] for loading LoRA weights
|
286 |
+
- [`~loaders.StableDiffusionXLLoraLoaderMixin.save_lora_weights`] for saving LoRA weights
|
287 |
+
- [`~loaders.IPAdapterMixin.load_ip_adapter`] for loading IP Adapters
|
288 |
+
|
289 |
+
Args:
|
290 |
+
vae ([`AutoencoderKL`]):
|
291 |
+
Variational Auto-Encoder (VAE) Model to encode and decode images to and from latent representations.
|
292 |
+
text_encoder ([`CLIPTextModel`]):
|
293 |
+
Frozen text-encoder. Stable Diffusion XL uses the text portion of
|
294 |
+
[CLIP](https://huggingface.co/docs/transformers/model_doc/clip#transformers.CLIPTextModel), specifically
|
295 |
+
the [clip-vit-large-patch14](https://huggingface.co/openai/clip-vit-large-patch14) variant.
|
296 |
+
text_encoder_2 ([` CLIPTextModelWithProjection`]):
|
297 |
+
Second frozen text-encoder. Stable Diffusion XL uses the text and pool portion of
|
298 |
+
[CLIP](https://huggingface.co/docs/transformers/model_doc/clip#transformers.CLIPTextModelWithProjection),
|
299 |
+
specifically the
|
300 |
+
[laion/CLIP-ViT-bigG-14-laion2B-39B-b160k](https://huggingface.co/laion/CLIP-ViT-bigG-14-laion2B-39B-b160k)
|
301 |
+
variant.
|
302 |
+
tokenizer (`CLIPTokenizer`):
|
303 |
+
Tokenizer of class
|
304 |
+
[CLIPTokenizer](https://huggingface.co/docs/transformers/v4.21.0/en/model_doc/clip#transformers.CLIPTokenizer).
|
305 |
+
tokenizer_2 (`CLIPTokenizer`):
|
306 |
+
Second Tokenizer of class
|
307 |
+
[CLIPTokenizer](https://huggingface.co/docs/transformers/v4.21.0/en/model_doc/clip#transformers.CLIPTokenizer).
|
308 |
+
unet ([`UNet2DConditionModel`]): Conditional U-Net architecture to denoise the encoded image latents.
|
309 |
+
scheduler ([`SchedulerMixin`]):
|
310 |
+
A scheduler to be used in combination with `unet` to denoise the encoded image latents. Can be one of
|
311 |
+
[`DDIMScheduler`], [`LMSDiscreteScheduler`], or [`PNDMScheduler`].
|
312 |
+
requires_aesthetics_score (`bool`, *optional*, defaults to `"False"`):
|
313 |
+
Whether the `unet` requires a aesthetic_score condition to be passed during inference. Also see the config
|
314 |
+
of `stabilityai/stable-diffusion-xl-refiner-1-0`.
|
315 |
+
force_zeros_for_empty_prompt (`bool`, *optional*, defaults to `"True"`):
|
316 |
+
Whether the negative prompt embeddings shall be forced to always be set to 0. Also see the config of
|
317 |
+
`stabilityai/stable-diffusion-xl-base-1-0`.
|
318 |
+
add_watermarker (`bool`, *optional*):
|
319 |
+
Whether to use the [invisible_watermark library](https://github.com/ShieldMnt/invisible-watermark/) to
|
320 |
+
watermark output images. If not defined, it will default to True if the package is installed, otherwise no
|
321 |
+
watermarker will be used.
|
322 |
+
"""
|
323 |
+
|
324 |
+
model_cpu_offload_seq = "text_encoder->text_encoder_2->image_encoder->unet->vae"
|
325 |
+
|
326 |
+
_optional_components = [
|
327 |
+
"tokenizer",
|
328 |
+
"tokenizer_2",
|
329 |
+
"text_encoder",
|
330 |
+
"text_encoder_2",
|
331 |
+
"image_encoder",
|
332 |
+
"feature_extractor",
|
333 |
+
]
|
334 |
+
_callback_tensor_inputs = [
|
335 |
+
"latents",
|
336 |
+
"prompt_embeds",
|
337 |
+
"negative_prompt_embeds",
|
338 |
+
"add_text_embeds",
|
339 |
+
"add_time_ids",
|
340 |
+
"negative_pooled_prompt_embeds",
|
341 |
+
"add_neg_time_ids",
|
342 |
+
"mask",
|
343 |
+
"masked_image_latents",
|
344 |
+
]
|
345 |
+
|
346 |
+
def __init__(
|
347 |
+
self,
|
348 |
+
vae: AutoencoderKL,
|
349 |
+
text_encoder: CLIPTextModel,
|
350 |
+
text_encoder_2: CLIPTextModelWithProjection,
|
351 |
+
tokenizer: CLIPTokenizer,
|
352 |
+
tokenizer_2: CLIPTokenizer,
|
353 |
+
unet: UNet2DConditionModel,
|
354 |
+
adapter: Union[T2IAdapter, MultiAdapter, List[T2IAdapter]],
|
355 |
+
scheduler: KarrasDiffusionSchedulers,
|
356 |
+
image_encoder: CLIPVisionModelWithProjection = None,
|
357 |
+
feature_extractor: CLIPImageProcessor = None,
|
358 |
+
requires_aesthetics_score: bool = False,
|
359 |
+
force_zeros_for_empty_prompt: bool = True,
|
360 |
+
add_watermarker: Optional[bool] = None,
|
361 |
+
):
|
362 |
+
super().__init__()
|
363 |
+
|
364 |
+
self.register_modules(
|
365 |
+
vae=vae,
|
366 |
+
text_encoder=text_encoder,
|
367 |
+
text_encoder_2=text_encoder_2,
|
368 |
+
tokenizer=tokenizer,
|
369 |
+
tokenizer_2=tokenizer_2,
|
370 |
+
unet=unet,
|
371 |
+
adapter=adapter,
|
372 |
+
image_encoder=image_encoder,
|
373 |
+
feature_extractor=feature_extractor,
|
374 |
+
scheduler=scheduler,
|
375 |
+
)
|
376 |
+
self.register_to_config(force_zeros_for_empty_prompt=force_zeros_for_empty_prompt)
|
377 |
+
self.register_to_config(requires_aesthetics_score=requires_aesthetics_score)
|
378 |
+
self.vae_scale_factor = 2 ** (len(self.vae.config.block_out_channels) - 1)
|
379 |
+
self.image_processor = VaeImageProcessor(vae_scale_factor=self.vae_scale_factor)
|
380 |
+
self.mask_processor = VaeImageProcessor(
|
381 |
+
vae_scale_factor=self.vae_scale_factor, do_normalize=False, do_binarize=True, do_convert_grayscale=True
|
382 |
+
)
|
383 |
+
|
384 |
+
add_watermarker = add_watermarker if add_watermarker is not None else is_invisible_watermark_available()
|
385 |
+
|
386 |
+
if add_watermarker:
|
387 |
+
self.watermark = StableDiffusionXLWatermarker()
|
388 |
+
else:
|
389 |
+
self.watermark = None
|
390 |
+
|
391 |
+
# Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.StableDiffusionPipeline.encode_image
|
392 |
+
def encode_image(self, image, device, num_images_per_prompt, output_hidden_states=None):
|
393 |
+
dtype = next(self.image_encoder.parameters()).dtype
|
394 |
+
|
395 |
+
if not isinstance(image, torch.Tensor):
|
396 |
+
image = self.feature_extractor(image, return_tensors="pt").pixel_values
|
397 |
+
|
398 |
+
image = image.to(device=device, dtype=dtype)
|
399 |
+
if output_hidden_states:
|
400 |
+
image_enc_hidden_states = self.image_encoder(image, output_hidden_states=True).hidden_states[-2]
|
401 |
+
image_enc_hidden_states = image_enc_hidden_states.repeat_interleave(num_images_per_prompt, dim=0)
|
402 |
+
uncond_image_enc_hidden_states = self.image_encoder(
|
403 |
+
torch.zeros_like(image), output_hidden_states=True
|
404 |
+
).hidden_states[-2]
|
405 |
+
uncond_image_enc_hidden_states = uncond_image_enc_hidden_states.repeat_interleave(
|
406 |
+
num_images_per_prompt, dim=0
|
407 |
+
)
|
408 |
+
return image_enc_hidden_states, uncond_image_enc_hidden_states
|
409 |
+
else:
|
410 |
+
image_embeds = self.image_encoder(image).image_embeds
|
411 |
+
image_embeds = image_embeds.repeat_interleave(num_images_per_prompt, dim=0)
|
412 |
+
uncond_image_embeds = torch.zeros_like(image_embeds)
|
413 |
+
|
414 |
+
return image_embeds, uncond_image_embeds
|
415 |
+
|
416 |
+
# Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.StableDiffusionPipeline.prepare_ip_adapter_image_embeds
|
417 |
+
def prepare_ip_adapter_image_embeds(
|
418 |
+
self, ip_adapter_image, ip_adapter_image_embeds, device, num_images_per_prompt, do_classifier_free_guidance
|
419 |
+
):
|
420 |
+
image_embeds = []
|
421 |
+
if do_classifier_free_guidance:
|
422 |
+
negative_image_embeds = []
|
423 |
+
if ip_adapter_image_embeds is None:
|
424 |
+
if not isinstance(ip_adapter_image, list):
|
425 |
+
ip_adapter_image = [ip_adapter_image]
|
426 |
+
|
427 |
+
if len(ip_adapter_image) != len(self.unet.encoder_hid_proj.image_projection_layers):
|
428 |
+
raise ValueError(
|
429 |
+
f"`ip_adapter_image` must have same length as the number of IP Adapters. Got {len(ip_adapter_image)} images and {len(self.unet.encoder_hid_proj.image_projection_layers)} IP Adapters."
|
430 |
+
)
|
431 |
+
|
432 |
+
for single_ip_adapter_image, image_proj_layer in zip(
|
433 |
+
ip_adapter_image, self.unet.encoder_hid_proj.image_projection_layers
|
434 |
+
):
|
435 |
+
output_hidden_state = not isinstance(image_proj_layer, ImageProjection)
|
436 |
+
single_image_embeds, single_negative_image_embeds = self.encode_image(
|
437 |
+
single_ip_adapter_image, device, 1, output_hidden_state
|
438 |
+
)
|
439 |
+
|
440 |
+
image_embeds.append(single_image_embeds[None, :])
|
441 |
+
if do_classifier_free_guidance:
|
442 |
+
negative_image_embeds.append(single_negative_image_embeds[None, :])
|
443 |
+
else:
|
444 |
+
for single_image_embeds in ip_adapter_image_embeds:
|
445 |
+
if do_classifier_free_guidance:
|
446 |
+
single_negative_image_embeds, single_image_embeds = single_image_embeds.chunk(2)
|
447 |
+
negative_image_embeds.append(single_negative_image_embeds)
|
448 |
+
image_embeds.append(single_image_embeds)
|
449 |
+
|
450 |
+
ip_adapter_image_embeds = []
|
451 |
+
for i, single_image_embeds in enumerate(image_embeds):
|
452 |
+
single_image_embeds = torch.cat([single_image_embeds] * num_images_per_prompt, dim=0)
|
453 |
+
if do_classifier_free_guidance:
|
454 |
+
single_negative_image_embeds = torch.cat([negative_image_embeds[i]] * num_images_per_prompt, dim=0)
|
455 |
+
single_image_embeds = torch.cat([single_negative_image_embeds, single_image_embeds], dim=0)
|
456 |
+
|
457 |
+
single_image_embeds = single_image_embeds.to(device=device)
|
458 |
+
ip_adapter_image_embeds.append(single_image_embeds)
|
459 |
+
|
460 |
+
return ip_adapter_image_embeds
|
461 |
+
|
462 |
+
# Copied from diffusers.pipelines.stable_diffusion_xl.pipeline_stable_diffusion_xl.StableDiffusionXLPipeline.encode_prompt
|
463 |
+
def encode_prompt(
|
464 |
+
self,
|
465 |
+
prompt: str,
|
466 |
+
prompt_2: Optional[str] = None,
|
467 |
+
device: Optional[torch.device] = None,
|
468 |
+
num_images_per_prompt: int = 1,
|
469 |
+
do_classifier_free_guidance: bool = True,
|
470 |
+
negative_prompt: Optional[str] = None,
|
471 |
+
negative_prompt_2: Optional[str] = None,
|
472 |
+
prompt_embeds: Optional[torch.Tensor] = None,
|
473 |
+
negative_prompt_embeds: Optional[torch.Tensor] = None,
|
474 |
+
pooled_prompt_embeds: Optional[torch.Tensor] = None,
|
475 |
+
negative_pooled_prompt_embeds: Optional[torch.Tensor] = None,
|
476 |
+
lora_scale: Optional[float] = None,
|
477 |
+
clip_skip: Optional[int] = None,
|
478 |
+
):
|
479 |
+
r"""
|
480 |
+
Encodes the prompt into text encoder hidden states.
|
481 |
+
|
482 |
+
Args:
|
483 |
+
prompt (`str` or `List[str]`, *optional*):
|
484 |
+
prompt to be encoded
|
485 |
+
prompt_2 (`str` or `List[str]`, *optional*):
|
486 |
+
The prompt or prompts to be sent to the `tokenizer_2` and `text_encoder_2`. If not defined, `prompt` is
|
487 |
+
used in both text-encoders
|
488 |
+
device: (`torch.device`):
|
489 |
+
torch device
|
490 |
+
num_images_per_prompt (`int`):
|
491 |
+
number of images that should be generated per prompt
|
492 |
+
do_classifier_free_guidance (`bool`):
|
493 |
+
whether to use classifier free guidance or not
|
494 |
+
negative_prompt (`str` or `List[str]`, *optional*):
|
495 |
+
The prompt or prompts not to guide the image generation. If not defined, one has to pass
|
496 |
+
`negative_prompt_embeds` instead. Ignored when not using guidance (i.e., ignored if `guidance_scale` is
|
497 |
+
less than `1`).
|
498 |
+
negative_prompt_2 (`str` or `List[str]`, *optional*):
|
499 |
+
The prompt or prompts not to guide the image generation to be sent to `tokenizer_2` and
|
500 |
+
`text_encoder_2`. If not defined, `negative_prompt` is used in both text-encoders
|
501 |
+
prompt_embeds (`torch.Tensor`, *optional*):
|
502 |
+
Pre-generated text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt weighting. If not
|
503 |
+
provided, text embeddings will be generated from `prompt` input argument.
|
504 |
+
negative_prompt_embeds (`torch.Tensor`, *optional*):
|
505 |
+
Pre-generated negative text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt
|
506 |
+
weighting. If not provided, negative_prompt_embeds will be generated from `negative_prompt` input
|
507 |
+
argument.
|
508 |
+
pooled_prompt_embeds (`torch.Tensor`, *optional*):
|
509 |
+
Pre-generated pooled text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt weighting.
|
510 |
+
If not provided, pooled text embeddings will be generated from `prompt` input argument.
|
511 |
+
negative_pooled_prompt_embeds (`torch.Tensor`, *optional*):
|
512 |
+
Pre-generated negative pooled text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt
|
513 |
+
weighting. If not provided, pooled negative_prompt_embeds will be generated from `negative_prompt`
|
514 |
+
input argument.
|
515 |
+
lora_scale (`float`, *optional*):
|
516 |
+
A lora scale that will be applied to all LoRA layers of the text encoder if LoRA layers are loaded.
|
517 |
+
clip_skip (`int`, *optional*):
|
518 |
+
Number of layers to be skipped from CLIP while computing the prompt embeddings. A value of 1 means that
|
519 |
+
the output of the pre-final layer will be used for computing the prompt embeddings.
|
520 |
+
"""
|
521 |
+
device = device or self._execution_device
|
522 |
+
|
523 |
+
# set lora scale so that monkey patched LoRA
|
524 |
+
# function of text encoder can correctly access it
|
525 |
+
if lora_scale is not None and isinstance(self, StableDiffusionXLLoraLoaderMixin):
|
526 |
+
self._lora_scale = lora_scale
|
527 |
+
|
528 |
+
# dynamically adjust the LoRA scale
|
529 |
+
if self.text_encoder is not None:
|
530 |
+
if not USE_PEFT_BACKEND:
|
531 |
+
adjust_lora_scale_text_encoder(self.text_encoder, lora_scale)
|
532 |
+
else:
|
533 |
+
scale_lora_layers(self.text_encoder, lora_scale)
|
534 |
+
|
535 |
+
if self.text_encoder_2 is not None:
|
536 |
+
if not USE_PEFT_BACKEND:
|
537 |
+
adjust_lora_scale_text_encoder(self.text_encoder_2, lora_scale)
|
538 |
+
else:
|
539 |
+
scale_lora_layers(self.text_encoder_2, lora_scale)
|
540 |
+
|
541 |
+
prompt = [prompt] if isinstance(prompt, str) else prompt
|
542 |
+
|
543 |
+
if prompt is not None:
|
544 |
+
batch_size = len(prompt)
|
545 |
+
else:
|
546 |
+
batch_size = prompt_embeds.shape[0]
|
547 |
+
|
548 |
+
# Define tokenizers and text encoders
|
549 |
+
tokenizers = [self.tokenizer, self.tokenizer_2] if self.tokenizer is not None else [self.tokenizer_2]
|
550 |
+
text_encoders = (
|
551 |
+
[self.text_encoder, self.text_encoder_2] if self.text_encoder is not None else [self.text_encoder_2]
|
552 |
+
)
|
553 |
+
|
554 |
+
if prompt_embeds is None:
|
555 |
+
prompt_2 = prompt_2 or prompt
|
556 |
+
prompt_2 = [prompt_2] if isinstance(prompt_2, str) else prompt_2
|
557 |
+
|
558 |
+
# textual inversion: process multi-vector tokens if necessary
|
559 |
+
prompt_embeds_list = []
|
560 |
+
prompts = [prompt, prompt_2]
|
561 |
+
for prompt, tokenizer, text_encoder in zip(prompts, tokenizers, text_encoders):
|
562 |
+
if isinstance(self, TextualInversionLoaderMixin):
|
563 |
+
prompt = self.maybe_convert_prompt(prompt, tokenizer)
|
564 |
+
|
565 |
+
text_inputs = tokenizer(
|
566 |
+
prompt,
|
567 |
+
padding="max_length",
|
568 |
+
max_length=tokenizer.model_max_length,
|
569 |
+
truncation=True,
|
570 |
+
return_tensors="pt",
|
571 |
+
)
|
572 |
+
|
573 |
+
text_input_ids = text_inputs.input_ids
|
574 |
+
untruncated_ids = tokenizer(prompt, padding="longest", return_tensors="pt").input_ids
|
575 |
+
|
576 |
+
if untruncated_ids.shape[-1] >= text_input_ids.shape[-1] and not torch.equal(
|
577 |
+
text_input_ids, untruncated_ids
|
578 |
+
):
|
579 |
+
removed_text = tokenizer.batch_decode(untruncated_ids[:, tokenizer.model_max_length - 1 : -1])
|
580 |
+
logger.warning(
|
581 |
+
"The following part of your input was truncated because CLIP can only handle sequences up to"
|
582 |
+
f" {tokenizer.model_max_length} tokens: {removed_text}"
|
583 |
+
)
|
584 |
+
|
585 |
+
prompt_embeds = text_encoder(text_input_ids.to(device), output_hidden_states=True)
|
586 |
+
|
587 |
+
# We are only ALWAYS interested in the pooled output of the final text encoder
|
588 |
+
pooled_prompt_embeds = prompt_embeds[0]
|
589 |
+
if clip_skip is None:
|
590 |
+
prompt_embeds = prompt_embeds.hidden_states[-2]
|
591 |
+
else:
|
592 |
+
# "2" because SDXL always indexes from the penultimate layer.
|
593 |
+
prompt_embeds = prompt_embeds.hidden_states[-(clip_skip + 2)]
|
594 |
+
|
595 |
+
prompt_embeds_list.append(prompt_embeds)
|
596 |
+
|
597 |
+
prompt_embeds = torch.concat(prompt_embeds_list, dim=-1)
|
598 |
+
|
599 |
+
# get unconditional embeddings for classifier free guidance
|
600 |
+
zero_out_negative_prompt = negative_prompt is None and self.config.force_zeros_for_empty_prompt
|
601 |
+
if do_classifier_free_guidance and negative_prompt_embeds is None and zero_out_negative_prompt:
|
602 |
+
negative_prompt_embeds = torch.zeros_like(prompt_embeds)
|
603 |
+
negative_pooled_prompt_embeds = torch.zeros_like(pooled_prompt_embeds)
|
604 |
+
elif do_classifier_free_guidance and negative_prompt_embeds is None:
|
605 |
+
negative_prompt = negative_prompt or ""
|
606 |
+
negative_prompt_2 = negative_prompt_2 or negative_prompt
|
607 |
+
|
608 |
+
# normalize str to list
|
609 |
+
negative_prompt = batch_size * [negative_prompt] if isinstance(negative_prompt, str) else negative_prompt
|
610 |
+
negative_prompt_2 = (
|
611 |
+
batch_size * [negative_prompt_2] if isinstance(negative_prompt_2, str) else negative_prompt_2
|
612 |
+
)
|
613 |
+
|
614 |
+
uncond_tokens: List[str]
|
615 |
+
if prompt is not None and type(prompt) is not type(negative_prompt):
|
616 |
+
raise TypeError(
|
617 |
+
f"`negative_prompt` should be the same type to `prompt`, but got {type(negative_prompt)} !="
|
618 |
+
f" {type(prompt)}."
|
619 |
+
)
|
620 |
+
elif batch_size != len(negative_prompt):
|
621 |
+
raise ValueError(
|
622 |
+
f"`negative_prompt`: {negative_prompt} has batch size {len(negative_prompt)}, but `prompt`:"
|
623 |
+
f" {prompt} has batch size {batch_size}. Please make sure that passed `negative_prompt` matches"
|
624 |
+
" the batch size of `prompt`."
|
625 |
+
)
|
626 |
+
else:
|
627 |
+
uncond_tokens = [negative_prompt, negative_prompt_2]
|
628 |
+
|
629 |
+
negative_prompt_embeds_list = []
|
630 |
+
for negative_prompt, tokenizer, text_encoder in zip(uncond_tokens, tokenizers, text_encoders):
|
631 |
+
if isinstance(self, TextualInversionLoaderMixin):
|
632 |
+
negative_prompt = self.maybe_convert_prompt(negative_prompt, tokenizer)
|
633 |
+
|
634 |
+
max_length = prompt_embeds.shape[1]
|
635 |
+
uncond_input = tokenizer(
|
636 |
+
negative_prompt,
|
637 |
+
padding="max_length",
|
638 |
+
max_length=max_length,
|
639 |
+
truncation=True,
|
640 |
+
return_tensors="pt",
|
641 |
+
)
|
642 |
+
|
643 |
+
negative_prompt_embeds = text_encoder(
|
644 |
+
uncond_input.input_ids.to(device),
|
645 |
+
output_hidden_states=True,
|
646 |
+
)
|
647 |
+
# We are only ALWAYS interested in the pooled output of the final text encoder
|
648 |
+
negative_pooled_prompt_embeds = negative_prompt_embeds[0]
|
649 |
+
negative_prompt_embeds = negative_prompt_embeds.hidden_states[-2]
|
650 |
+
|
651 |
+
negative_prompt_embeds_list.append(negative_prompt_embeds)
|
652 |
+
|
653 |
+
negative_prompt_embeds = torch.concat(negative_prompt_embeds_list, dim=-1)
|
654 |
+
|
655 |
+
if self.text_encoder_2 is not None:
|
656 |
+
prompt_embeds = prompt_embeds.to(dtype=self.text_encoder_2.dtype, device=device)
|
657 |
+
else:
|
658 |
+
prompt_embeds = prompt_embeds.to(dtype=self.unet.dtype, device=device)
|
659 |
+
|
660 |
+
bs_embed, seq_len, _ = prompt_embeds.shape
|
661 |
+
# duplicate text embeddings for each generation per prompt, using mps friendly method
|
662 |
+
prompt_embeds = prompt_embeds.repeat(1, num_images_per_prompt, 1)
|
663 |
+
prompt_embeds = prompt_embeds.view(bs_embed * num_images_per_prompt, seq_len, -1)
|
664 |
+
|
665 |
+
if do_classifier_free_guidance:
|
666 |
+
# duplicate unconditional embeddings for each generation per prompt, using mps friendly method
|
667 |
+
seq_len = negative_prompt_embeds.shape[1]
|
668 |
+
|
669 |
+
if self.text_encoder_2 is not None:
|
670 |
+
negative_prompt_embeds = negative_prompt_embeds.to(dtype=self.text_encoder_2.dtype, device=device)
|
671 |
+
else:
|
672 |
+
negative_prompt_embeds = negative_prompt_embeds.to(dtype=self.unet.dtype, device=device)
|
673 |
+
|
674 |
+
negative_prompt_embeds = negative_prompt_embeds.repeat(1, num_images_per_prompt, 1)
|
675 |
+
negative_prompt_embeds = negative_prompt_embeds.view(batch_size * num_images_per_prompt, seq_len, -1)
|
676 |
+
|
677 |
+
pooled_prompt_embeds = pooled_prompt_embeds.repeat(1, num_images_per_prompt).view(
|
678 |
+
bs_embed * num_images_per_prompt, -1
|
679 |
+
)
|
680 |
+
if do_classifier_free_guidance:
|
681 |
+
negative_pooled_prompt_embeds = negative_pooled_prompt_embeds.repeat(1, num_images_per_prompt).view(
|
682 |
+
bs_embed * num_images_per_prompt, -1
|
683 |
+
)
|
684 |
+
|
685 |
+
if self.text_encoder is not None:
|
686 |
+
if isinstance(self, StableDiffusionXLLoraLoaderMixin) and USE_PEFT_BACKEND:
|
687 |
+
# Retrieve the original scale by scaling back the LoRA layers
|
688 |
+
unscale_lora_layers(self.text_encoder, lora_scale)
|
689 |
+
|
690 |
+
if self.text_encoder_2 is not None:
|
691 |
+
if isinstance(self, StableDiffusionXLLoraLoaderMixin) and USE_PEFT_BACKEND:
|
692 |
+
# Retrieve the original scale by scaling back the LoRA layers
|
693 |
+
unscale_lora_layers(self.text_encoder_2, lora_scale)
|
694 |
+
|
695 |
+
return prompt_embeds, negative_prompt_embeds, pooled_prompt_embeds, negative_pooled_prompt_embeds
|
696 |
+
|
697 |
+
# Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.StableDiffusionPipeline.prepare_extra_step_kwargs
|
698 |
+
def prepare_extra_step_kwargs(self, generator, eta):
|
699 |
+
# prepare extra kwargs for the scheduler step, since not all schedulers have the same signature
|
700 |
+
# eta (η) is only used with the DDIMScheduler, it will be ignored for other schedulers.
|
701 |
+
# eta corresponds to η in DDIM paper: https://arxiv.org/abs/2010.02502
|
702 |
+
# and should be between [0, 1]
|
703 |
+
|
704 |
+
accepts_eta = "eta" in set(inspect.signature(self.scheduler.step).parameters.keys())
|
705 |
+
extra_step_kwargs = {}
|
706 |
+
if accepts_eta:
|
707 |
+
extra_step_kwargs["eta"] = eta
|
708 |
+
|
709 |
+
# check if the scheduler accepts generator
|
710 |
+
accepts_generator = "generator" in set(inspect.signature(self.scheduler.step).parameters.keys())
|
711 |
+
if accepts_generator:
|
712 |
+
extra_step_kwargs["generator"] = generator
|
713 |
+
return extra_step_kwargs
|
714 |
+
|
715 |
+
def check_inputs(
|
716 |
+
self,
|
717 |
+
prompt,
|
718 |
+
prompt_2,
|
719 |
+
image,
|
720 |
+
mask_image,
|
721 |
+
height,
|
722 |
+
width,
|
723 |
+
strength,
|
724 |
+
callback_steps,
|
725 |
+
output_type,
|
726 |
+
negative_prompt=None,
|
727 |
+
negative_prompt_2=None,
|
728 |
+
prompt_embeds=None,
|
729 |
+
negative_prompt_embeds=None,
|
730 |
+
ip_adapter_image=None,
|
731 |
+
ip_adapter_image_embeds=None,
|
732 |
+
callback_on_step_end_tensor_inputs=None,
|
733 |
+
padding_mask_crop=None,
|
734 |
+
):
|
735 |
+
if strength < 0 or strength > 1:
|
736 |
+
raise ValueError(f"The value of strength should in [0.0, 1.0] but is {strength}")
|
737 |
+
|
738 |
+
if height % 8 != 0 or width % 8 != 0:
|
739 |
+
raise ValueError(f"`height` and `width` have to be divisible by 8 but are {height} and {width}.")
|
740 |
+
|
741 |
+
if callback_steps is not None and (not isinstance(callback_steps, int) or callback_steps <= 0):
|
742 |
+
raise ValueError(
|
743 |
+
f"`callback_steps` has to be a positive integer but is {callback_steps} of type"
|
744 |
+
f" {type(callback_steps)}."
|
745 |
+
)
|
746 |
+
|
747 |
+
if callback_on_step_end_tensor_inputs is not None and not all(
|
748 |
+
k in self._callback_tensor_inputs for k in callback_on_step_end_tensor_inputs
|
749 |
+
):
|
750 |
+
raise ValueError(
|
751 |
+
f"`callback_on_step_end_tensor_inputs` has to be in {self._callback_tensor_inputs}, but found {[k for k in callback_on_step_end_tensor_inputs if k not in self._callback_tensor_inputs]}"
|
752 |
+
)
|
753 |
+
|
754 |
+
if prompt is not None and prompt_embeds is not None:
|
755 |
+
raise ValueError(
|
756 |
+
f"Cannot forward both `prompt`: {prompt} and `prompt_embeds`: {prompt_embeds}. Please make sure to"
|
757 |
+
" only forward one of the two."
|
758 |
+
)
|
759 |
+
elif prompt_2 is not None and prompt_embeds is not None:
|
760 |
+
raise ValueError(
|
761 |
+
f"Cannot forward both `prompt_2`: {prompt_2} and `prompt_embeds`: {prompt_embeds}. Please make sure to"
|
762 |
+
" only forward one of the two."
|
763 |
+
)
|
764 |
+
elif prompt is None and prompt_embeds is None:
|
765 |
+
raise ValueError(
|
766 |
+
"Provide either `prompt` or `prompt_embeds`. Cannot leave both `prompt` and `prompt_embeds` undefined."
|
767 |
+
)
|
768 |
+
elif prompt is not None and (not isinstance(prompt, str) and not isinstance(prompt, list)):
|
769 |
+
raise ValueError(f"`prompt` has to be of type `str` or `list` but is {type(prompt)}")
|
770 |
+
elif prompt_2 is not None and (not isinstance(prompt_2, str) and not isinstance(prompt_2, list)):
|
771 |
+
raise ValueError(f"`prompt_2` has to be of type `str` or `list` but is {type(prompt_2)}")
|
772 |
+
|
773 |
+
if negative_prompt is not None and negative_prompt_embeds is not None:
|
774 |
+
raise ValueError(
|
775 |
+
f"Cannot forward both `negative_prompt`: {negative_prompt} and `negative_prompt_embeds`:"
|
776 |
+
f" {negative_prompt_embeds}. Please make sure to only forward one of the two."
|
777 |
+
)
|
778 |
+
elif negative_prompt_2 is not None and negative_prompt_embeds is not None:
|
779 |
+
raise ValueError(
|
780 |
+
f"Cannot forward both `negative_prompt_2`: {negative_prompt_2} and `negative_prompt_embeds`:"
|
781 |
+
f" {negative_prompt_embeds}. Please make sure to only forward one of the two."
|
782 |
+
)
|
783 |
+
|
784 |
+
if prompt_embeds is not None and negative_prompt_embeds is not None:
|
785 |
+
if prompt_embeds.shape != negative_prompt_embeds.shape:
|
786 |
+
raise ValueError(
|
787 |
+
"`prompt_embeds` and `negative_prompt_embeds` must have the same shape when passed directly, but"
|
788 |
+
f" got: `prompt_embeds` {prompt_embeds.shape} != `negative_prompt_embeds`"
|
789 |
+
f" {negative_prompt_embeds.shape}."
|
790 |
+
)
|
791 |
+
if padding_mask_crop is not None:
|
792 |
+
if not isinstance(image, PIL.Image.Image):
|
793 |
+
raise ValueError(
|
794 |
+
f"The image should be a PIL image when inpainting mask crop, but is of type" f" {type(image)}."
|
795 |
+
)
|
796 |
+
if not isinstance(mask_image, PIL.Image.Image):
|
797 |
+
raise ValueError(
|
798 |
+
f"The mask image should be a PIL image when inpainting mask crop, but is of type"
|
799 |
+
f" {type(mask_image)}."
|
800 |
+
)
|
801 |
+
if output_type != "pil":
|
802 |
+
raise ValueError(f"The output type should be PIL when inpainting mask crop, but is" f" {output_type}.")
|
803 |
+
|
804 |
+
if ip_adapter_image is not None and ip_adapter_image_embeds is not None:
|
805 |
+
raise ValueError(
|
806 |
+
"Provide either `ip_adapter_image` or `ip_adapter_image_embeds`. Cannot leave both `ip_adapter_image` and `ip_adapter_image_embeds` defined."
|
807 |
+
)
|
808 |
+
|
809 |
+
if ip_adapter_image_embeds is not None:
|
810 |
+
if not isinstance(ip_adapter_image_embeds, list):
|
811 |
+
raise ValueError(
|
812 |
+
f"`ip_adapter_image_embeds` has to be of type `list` but is {type(ip_adapter_image_embeds)}"
|
813 |
+
)
|
814 |
+
elif ip_adapter_image_embeds[0].ndim not in [3, 4]:
|
815 |
+
raise ValueError(
|
816 |
+
f"`ip_adapter_image_embeds` has to be a list of 3D or 4D tensors but is {ip_adapter_image_embeds[0].ndim}D"
|
817 |
+
)
|
818 |
+
|
819 |
+
def prepare_latents(
|
820 |
+
self,
|
821 |
+
batch_size,
|
822 |
+
num_channels_latents,
|
823 |
+
height,
|
824 |
+
width,
|
825 |
+
dtype,
|
826 |
+
device,
|
827 |
+
generator,
|
828 |
+
latents=None,
|
829 |
+
image=None,
|
830 |
+
timestep=None,
|
831 |
+
is_strength_max=True,
|
832 |
+
add_noise=True,
|
833 |
+
return_noise=False,
|
834 |
+
return_image_latents=False,
|
835 |
+
):
|
836 |
+
shape = (
|
837 |
+
batch_size,
|
838 |
+
num_channels_latents,
|
839 |
+
int(height) // self.vae_scale_factor,
|
840 |
+
int(width) // self.vae_scale_factor,
|
841 |
+
)
|
842 |
+
if isinstance(generator, list) and len(generator) != batch_size:
|
843 |
+
raise ValueError(
|
844 |
+
f"You have passed a list of generators of length {len(generator)}, but requested an effective batch"
|
845 |
+
f" size of {batch_size}. Make sure the batch size matches the length of the generators."
|
846 |
+
)
|
847 |
+
|
848 |
+
if (image is None or timestep is None) and not is_strength_max:
|
849 |
+
raise ValueError(
|
850 |
+
"Since strength < 1. initial latents are to be initialised as a combination of Image + Noise."
|
851 |
+
"However, either the image or the noise timestep has not been provided."
|
852 |
+
)
|
853 |
+
|
854 |
+
if image.shape[1] == 4:
|
855 |
+
image_latents = image.to(device=device, dtype=dtype)
|
856 |
+
image_latents = image_latents.repeat(batch_size // image_latents.shape[0], 1, 1, 1)
|
857 |
+
elif return_image_latents or (latents is None and not is_strength_max):
|
858 |
+
image = image.to(device=device, dtype=dtype)
|
859 |
+
image_latents = self._encode_vae_image(image=image, generator=generator)
|
860 |
+
image_latents = image_latents.repeat(batch_size // image_latents.shape[0], 1, 1, 1)
|
861 |
+
|
862 |
+
if latents is None and add_noise:
|
863 |
+
noise = randn_tensor(shape, generator=generator, device=device, dtype=dtype)
|
864 |
+
# if strength is 1. then initialise the latents to noise, else initial to image + noise
|
865 |
+
latents = noise if is_strength_max else self.scheduler.add_noise(image_latents, noise, timestep)
|
866 |
+
# if pure noise then scale the initial latents by the Scheduler's init sigma
|
867 |
+
latents = latents * self.scheduler.init_noise_sigma if is_strength_max else latents
|
868 |
+
elif add_noise:
|
869 |
+
noise = latents.to(device)
|
870 |
+
latents = noise * self.scheduler.init_noise_sigma
|
871 |
+
else:
|
872 |
+
noise = randn_tensor(shape, generator=generator, device=device, dtype=dtype)
|
873 |
+
latents = image_latents.to(device)
|
874 |
+
|
875 |
+
outputs = (latents,)
|
876 |
+
|
877 |
+
if return_noise:
|
878 |
+
outputs += (noise,)
|
879 |
+
|
880 |
+
if return_image_latents:
|
881 |
+
outputs += (image_latents,)
|
882 |
+
|
883 |
+
return outputs
|
884 |
+
|
885 |
+
def _encode_vae_image(self, image: torch.Tensor, generator: torch.Generator):
|
886 |
+
dtype = image.dtype
|
887 |
+
if self.vae.config.force_upcast:
|
888 |
+
image = image.float()
|
889 |
+
self.vae.to(dtype=torch.float32)
|
890 |
+
|
891 |
+
if isinstance(generator, list):
|
892 |
+
image_latents = [
|
893 |
+
retrieve_latents(self.vae.encode(image[i : i + 1]), generator=generator[i])
|
894 |
+
for i in range(image.shape[0])
|
895 |
+
]
|
896 |
+
image_latents = torch.cat(image_latents, dim=0)
|
897 |
+
else:
|
898 |
+
image_latents = retrieve_latents(self.vae.encode(image), generator=generator)
|
899 |
+
|
900 |
+
if self.vae.config.force_upcast:
|
901 |
+
self.vae.to(dtype)
|
902 |
+
|
903 |
+
image_latents = image_latents.to(dtype)
|
904 |
+
image_latents = self.vae.config.scaling_factor * image_latents
|
905 |
+
|
906 |
+
return image_latents
|
907 |
+
|
908 |
+
def prepare_mask_latents(
|
909 |
+
self, mask, masked_image, batch_size, height, width, dtype, device, generator, do_classifier_free_guidance
|
910 |
+
):
|
911 |
+
# resize the mask to latents shape as we concatenate the mask to the latents
|
912 |
+
# we do that before converting to dtype to avoid breaking in case we're using cpu_offload
|
913 |
+
# and half precision
|
914 |
+
mask = torch.nn.functional.interpolate(
|
915 |
+
mask, size=(height // self.vae_scale_factor, width // self.vae_scale_factor)
|
916 |
+
)
|
917 |
+
mask = mask.to(device=device, dtype=dtype)
|
918 |
+
|
919 |
+
# duplicate mask and masked_image_latents for each generation per prompt, using mps friendly method
|
920 |
+
if mask.shape[0] < batch_size:
|
921 |
+
if not batch_size % mask.shape[0] == 0:
|
922 |
+
raise ValueError(
|
923 |
+
"The passed mask and the required batch size don't match. Masks are supposed to be duplicated to"
|
924 |
+
f" a total batch size of {batch_size}, but {mask.shape[0]} masks were passed. Make sure the number"
|
925 |
+
" of masks that you pass is divisible by the total requested batch size."
|
926 |
+
)
|
927 |
+
mask = mask.repeat(batch_size // mask.shape[0], 1, 1, 1)
|
928 |
+
|
929 |
+
mask = torch.cat([mask] * 2) if do_classifier_free_guidance else mask
|
930 |
+
|
931 |
+
if masked_image is not None and masked_image.shape[1] == 4:
|
932 |
+
masked_image_latents = masked_image
|
933 |
+
else:
|
934 |
+
masked_image_latents = None
|
935 |
+
|
936 |
+
if masked_image is not None:
|
937 |
+
if masked_image_latents is None:
|
938 |
+
masked_image = masked_image.to(device=device, dtype=dtype)
|
939 |
+
masked_image_latents = self._encode_vae_image(masked_image, generator=generator)
|
940 |
+
|
941 |
+
if masked_image_latents.shape[0] < batch_size:
|
942 |
+
if not batch_size % masked_image_latents.shape[0] == 0:
|
943 |
+
raise ValueError(
|
944 |
+
"The passed images and the required batch size don't match. Images are supposed to be duplicated"
|
945 |
+
f" to a total batch size of {batch_size}, but {masked_image_latents.shape[0]} images were passed."
|
946 |
+
" Make sure the number of images that you pass is divisible by the total requested batch size."
|
947 |
+
)
|
948 |
+
masked_image_latents = masked_image_latents.repeat(
|
949 |
+
batch_size // masked_image_latents.shape[0], 1, 1, 1
|
950 |
+
)
|
951 |
+
|
952 |
+
masked_image_latents = (
|
953 |
+
torch.cat([masked_image_latents] * 2) if do_classifier_free_guidance else masked_image_latents
|
954 |
+
)
|
955 |
+
|
956 |
+
# aligning device to prevent device errors when concating it with the latent model input
|
957 |
+
masked_image_latents = masked_image_latents.to(device=device, dtype=dtype)
|
958 |
+
|
959 |
+
return mask, masked_image_latents
|
960 |
+
|
961 |
+
# Copied from diffusers.pipelines.stable_diffusion_xl.pipeline_stable_diffusion_xl_img2img.StableDiffusionXLImg2ImgPipeline.get_timesteps
|
962 |
+
def get_timesteps(self, num_inference_steps, strength, device, denoising_start=None):
|
963 |
+
# get the original timestep using init_timestep
|
964 |
+
if denoising_start is None:
|
965 |
+
init_timestep = min(int(num_inference_steps * strength), num_inference_steps)
|
966 |
+
t_start = max(num_inference_steps - init_timestep, 0)
|
967 |
+
|
968 |
+
timesteps = self.scheduler.timesteps[t_start * self.scheduler.order :]
|
969 |
+
if hasattr(self.scheduler, "set_begin_index"):
|
970 |
+
self.scheduler.set_begin_index(t_start * self.scheduler.order)
|
971 |
+
|
972 |
+
return timesteps, num_inference_steps - t_start
|
973 |
+
|
974 |
+
else:
|
975 |
+
# Strength is irrelevant if we directly request a timestep to start at;
|
976 |
+
# that is, strength is determined by the denoising_start instead.
|
977 |
+
discrete_timestep_cutoff = int(
|
978 |
+
round(
|
979 |
+
self.scheduler.config.num_train_timesteps
|
980 |
+
- (denoising_start * self.scheduler.config.num_train_timesteps)
|
981 |
+
)
|
982 |
+
)
|
983 |
+
|
984 |
+
num_inference_steps = (self.scheduler.timesteps < discrete_timestep_cutoff).sum().item()
|
985 |
+
if self.scheduler.order == 2 and num_inference_steps % 2 == 0:
|
986 |
+
# if the scheduler is a 2nd order scheduler we might have to do +1
|
987 |
+
# because `num_inference_steps` might be even given that every timestep
|
988 |
+
# (except the highest one) is duplicated. If `num_inference_steps` is even it would
|
989 |
+
# mean that we cut the timesteps in the middle of the denoising step
|
990 |
+
# (between 1st and 2nd derivative) which leads to incorrect results. By adding 1
|
991 |
+
# we ensure that the denoising process always ends after the 2nd derivate step of the scheduler
|
992 |
+
num_inference_steps = num_inference_steps + 1
|
993 |
+
|
994 |
+
# because t_n+1 >= t_n, we slice the timesteps starting from the end
|
995 |
+
t_start = len(self.scheduler.timesteps) - num_inference_steps
|
996 |
+
timesteps = self.scheduler.timesteps[t_start:]
|
997 |
+
if hasattr(self.scheduler, "set_begin_index"):
|
998 |
+
self.scheduler.set_begin_index(t_start)
|
999 |
+
return timesteps, num_inference_steps
|
1000 |
+
|
1001 |
+
# Copied from diffusers.pipelines.stable_diffusion_xl.pipeline_stable_diffusion_xl_img2img.StableDiffusionXLImg2ImgPipeline._get_add_time_ids
|
1002 |
+
def _get_add_time_ids(
|
1003 |
+
self,
|
1004 |
+
original_size,
|
1005 |
+
crops_coords_top_left,
|
1006 |
+
target_size,
|
1007 |
+
aesthetic_score,
|
1008 |
+
negative_aesthetic_score,
|
1009 |
+
negative_original_size,
|
1010 |
+
negative_crops_coords_top_left,
|
1011 |
+
negative_target_size,
|
1012 |
+
dtype,
|
1013 |
+
text_encoder_projection_dim=None,
|
1014 |
+
):
|
1015 |
+
if self.config.requires_aesthetics_score:
|
1016 |
+
add_time_ids = list(original_size + crops_coords_top_left + (aesthetic_score,))
|
1017 |
+
add_neg_time_ids = list(
|
1018 |
+
negative_original_size + negative_crops_coords_top_left + (negative_aesthetic_score,)
|
1019 |
+
)
|
1020 |
+
else:
|
1021 |
+
add_time_ids = list(original_size + crops_coords_top_left + target_size)
|
1022 |
+
add_neg_time_ids = list(negative_original_size + crops_coords_top_left + negative_target_size)
|
1023 |
+
|
1024 |
+
passed_add_embed_dim = (
|
1025 |
+
self.unet.config.addition_time_embed_dim * len(add_time_ids) + text_encoder_projection_dim
|
1026 |
+
)
|
1027 |
+
expected_add_embed_dim = self.unet.add_embedding.linear_1.in_features
|
1028 |
+
|
1029 |
+
if (
|
1030 |
+
expected_add_embed_dim > passed_add_embed_dim
|
1031 |
+
and (expected_add_embed_dim - passed_add_embed_dim) == self.unet.config.addition_time_embed_dim
|
1032 |
+
):
|
1033 |
+
raise ValueError(
|
1034 |
+
f"Model expects an added time embedding vector of length {expected_add_embed_dim}, but a vector of {passed_add_embed_dim} was created. Please make sure to enable `requires_aesthetics_score` with `pipe.register_to_config(requires_aesthetics_score=True)` to make sure `aesthetic_score` {aesthetic_score} and `negative_aesthetic_score` {negative_aesthetic_score} is correctly used by the model."
|
1035 |
+
)
|
1036 |
+
elif (
|
1037 |
+
expected_add_embed_dim < passed_add_embed_dim
|
1038 |
+
and (passed_add_embed_dim - expected_add_embed_dim) == self.unet.config.addition_time_embed_dim
|
1039 |
+
):
|
1040 |
+
raise ValueError(
|
1041 |
+
f"Model expects an added time embedding vector of length {expected_add_embed_dim}, but a vector of {passed_add_embed_dim} was created. Please make sure to disable `requires_aesthetics_score` with `pipe.register_to_config(requires_aesthetics_score=False)` to make sure `target_size` {target_size} is correctly used by the model."
|
1042 |
+
)
|
1043 |
+
elif expected_add_embed_dim != passed_add_embed_dim:
|
1044 |
+
raise ValueError(
|
1045 |
+
f"Model expects an added time embedding vector of length {expected_add_embed_dim}, but a vector of {passed_add_embed_dim} was created. The model has an incorrect config. Please check `unet.config.time_embedding_type` and `text_encoder_2.config.projection_dim`."
|
1046 |
+
)
|
1047 |
+
|
1048 |
+
add_time_ids = torch.tensor([add_time_ids], dtype=dtype)
|
1049 |
+
add_neg_time_ids = torch.tensor([add_neg_time_ids], dtype=dtype)
|
1050 |
+
|
1051 |
+
return add_time_ids, add_neg_time_ids
|
1052 |
+
|
1053 |
+
# Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion_upscale.StableDiffusionUpscalePipeline.upcast_vae
|
1054 |
+
def upcast_vae(self):
|
1055 |
+
dtype = self.vae.dtype
|
1056 |
+
self.vae.to(dtype=torch.float32)
|
1057 |
+
use_torch_2_0_or_xformers = isinstance(
|
1058 |
+
self.vae.decoder.mid_block.attentions[0].processor,
|
1059 |
+
(
|
1060 |
+
AttnProcessor2_0,
|
1061 |
+
XFormersAttnProcessor,
|
1062 |
+
),
|
1063 |
+
)
|
1064 |
+
# if xformers or torch_2_0 is used attention block does not need
|
1065 |
+
# to be in float32 which can save lots of memory
|
1066 |
+
if use_torch_2_0_or_xformers:
|
1067 |
+
self.vae.post_quant_conv.to(dtype)
|
1068 |
+
self.vae.decoder.conv_in.to(dtype)
|
1069 |
+
self.vae.decoder.mid_block.to(dtype)
|
1070 |
+
|
1071 |
+
# Copied from diffusers.pipelines.t2i_adapter.pipeline_stable_diffusion_adapter.StableDiffusionAdapterPipeline._default_height_width
|
1072 |
+
def _default_height_width(self, height, width, image):
|
1073 |
+
# NOTE: It is possible that a list of images have different
|
1074 |
+
# dimensions for each image, so just checking the first image
|
1075 |
+
# is not _exactly_ correct, but it is simple.
|
1076 |
+
while isinstance(image, list):
|
1077 |
+
image = image[0]
|
1078 |
+
|
1079 |
+
if height is None:
|
1080 |
+
if isinstance(image, PIL.Image.Image):
|
1081 |
+
height = image.height
|
1082 |
+
elif isinstance(image, torch.Tensor):
|
1083 |
+
height = image.shape[-2]
|
1084 |
+
|
1085 |
+
# round down to nearest multiple of `self.adapter.downscale_factor`
|
1086 |
+
height = (height // self.adapter.downscale_factor) * self.adapter.downscale_factor
|
1087 |
+
|
1088 |
+
if width is None:
|
1089 |
+
if isinstance(image, PIL.Image.Image):
|
1090 |
+
width = image.width
|
1091 |
+
elif isinstance(image, torch.Tensor):
|
1092 |
+
width = image.shape[-1]
|
1093 |
+
|
1094 |
+
# round down to nearest multiple of `self.adapter.downscale_factor`
|
1095 |
+
width = (width // self.adapter.downscale_factor) * self.adapter.downscale_factor
|
1096 |
+
|
1097 |
+
return height, width
|
1098 |
+
|
1099 |
+
# Copied from diffusers.pipelines.latent_consistency_models.pipeline_latent_consistency_text2img.LatentConsistencyModelPipeline.get_guidance_scale_embedding
|
1100 |
+
def get_guidance_scale_embedding(
|
1101 |
+
self, w: torch.Tensor, embedding_dim: int = 512, dtype: torch.dtype = torch.float32
|
1102 |
+
) -> torch.Tensor:
|
1103 |
+
"""
|
1104 |
+
See https://github.com/google-research/vdm/blob/dc27b98a554f65cdc654b800da5aa1846545d41b/model_vdm.py#L298
|
1105 |
+
|
1106 |
+
Args:
|
1107 |
+
w (`torch.Tensor`):
|
1108 |
+
Generate embedding vectors with a specified guidance scale to subsequently enrich timestep embeddings.
|
1109 |
+
embedding_dim (`int`, *optional*, defaults to 512):
|
1110 |
+
Dimension of the embeddings to generate.
|
1111 |
+
dtype (`torch.dtype`, *optional*, defaults to `torch.float32`):
|
1112 |
+
Data type of the generated embeddings.
|
1113 |
+
|
1114 |
+
Returns:
|
1115 |
+
`torch.Tensor`: Embedding vectors with shape `(len(w), embedding_dim)`.
|
1116 |
+
"""
|
1117 |
+
assert len(w.shape) == 1
|
1118 |
+
w = w * 1000.0
|
1119 |
+
|
1120 |
+
half_dim = embedding_dim // 2
|
1121 |
+
emb = torch.log(torch.tensor(10000.0)) / (half_dim - 1)
|
1122 |
+
emb = torch.exp(torch.arange(half_dim, dtype=dtype) * -emb)
|
1123 |
+
emb = w.to(dtype)[:, None] * emb[None, :]
|
1124 |
+
emb = torch.cat([torch.sin(emb), torch.cos(emb)], dim=1)
|
1125 |
+
if embedding_dim % 2 == 1: # zero pad
|
1126 |
+
emb = torch.nn.functional.pad(emb, (0, 1))
|
1127 |
+
assert emb.shape == (w.shape[0], embedding_dim)
|
1128 |
+
return emb
|
1129 |
+
|
1130 |
+
@property
|
1131 |
+
def guidance_scale(self):
|
1132 |
+
return self._guidance_scale
|
1133 |
+
|
1134 |
+
@property
|
1135 |
+
def guidance_rescale(self):
|
1136 |
+
return self._guidance_rescale
|
1137 |
+
|
1138 |
+
@property
|
1139 |
+
def clip_skip(self):
|
1140 |
+
return self._clip_skip
|
1141 |
+
|
1142 |
+
# here `guidance_scale` is defined analog to the guidance weight `w` of equation (2)
|
1143 |
+
# of the Imagen paper: https://arxiv.org/pdf/2205.11487.pdf . `guidance_scale = 1`
|
1144 |
+
# corresponds to doing no classifier free guidance.
|
1145 |
+
@property
|
1146 |
+
def do_classifier_free_guidance(self):
|
1147 |
+
return self._guidance_scale > 1 and self.unet.config.time_cond_proj_dim is None
|
1148 |
+
|
1149 |
+
@property
|
1150 |
+
def cross_attention_kwargs(self):
|
1151 |
+
return self._cross_attention_kwargs
|
1152 |
+
|
1153 |
+
@property
|
1154 |
+
def denoising_end(self):
|
1155 |
+
return self._denoising_end
|
1156 |
+
|
1157 |
+
@property
|
1158 |
+
def denoising_start(self):
|
1159 |
+
return self._denoising_start
|
1160 |
+
|
1161 |
+
@property
|
1162 |
+
def num_timesteps(self):
|
1163 |
+
return self._num_timesteps
|
1164 |
+
|
1165 |
+
@property
|
1166 |
+
def interrupt(self):
|
1167 |
+
return self._interrupt
|
1168 |
+
|
1169 |
+
@torch.no_grad()
|
1170 |
+
@replace_example_docstring(EXAMPLE_DOC_STRING)
|
1171 |
+
def __call__(
|
1172 |
+
self,
|
1173 |
+
prompt: Union[str, List[str]] = None,
|
1174 |
+
prompt_2: Optional[Union[str, List[str]]] = None,
|
1175 |
+
image: PipelineImageInput = None,
|
1176 |
+
mask_image: PipelineImageInput = None,
|
1177 |
+
masked_image_latents: torch.Tensor = None,
|
1178 |
+
height: Optional[int] = None,
|
1179 |
+
adapter_image: PipelineImageInput = None,
|
1180 |
+
width: Optional[int] = None,
|
1181 |
+
padding_mask_crop: Optional[int] = None,
|
1182 |
+
strength: float = 0.9999,
|
1183 |
+
num_inference_steps: int = 50,
|
1184 |
+
timesteps: List[int] = None,
|
1185 |
+
sigmas: List[float] = None,
|
1186 |
+
denoising_start: Optional[float] = None,
|
1187 |
+
denoising_end: Optional[float] = None,
|
1188 |
+
guidance_scale: float = 7.5,
|
1189 |
+
negative_prompt: Optional[Union[str, List[str]]] = None,
|
1190 |
+
negative_prompt_2: Optional[Union[str, List[str]]] = None,
|
1191 |
+
num_images_per_prompt: Optional[int] = 1,
|
1192 |
+
eta: float = 0.0,
|
1193 |
+
generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None,
|
1194 |
+
latents: Optional[torch.Tensor] = None,
|
1195 |
+
prompt_embeds: Optional[torch.Tensor] = None,
|
1196 |
+
negative_prompt_embeds: Optional[torch.Tensor] = None,
|
1197 |
+
pooled_prompt_embeds: Optional[torch.Tensor] = None,
|
1198 |
+
negative_pooled_prompt_embeds: Optional[torch.Tensor] = None,
|
1199 |
+
ip_adapter_image: Optional[PipelineImageInput] = None,
|
1200 |
+
ip_adapter_image_embeds: Optional[List[torch.Tensor]] = None,
|
1201 |
+
output_type: Optional[str] = "pil",
|
1202 |
+
return_dict: bool = True,
|
1203 |
+
cross_attention_kwargs: Optional[Dict[str, Any]] = None,
|
1204 |
+
guidance_rescale: float = 0.0,
|
1205 |
+
original_size: Tuple[int, int] = None,
|
1206 |
+
crops_coords_top_left: Tuple[int, int] = (0, 0),
|
1207 |
+
target_size: Tuple[int, int] = None,
|
1208 |
+
negative_original_size: Optional[Tuple[int, int]] = None,
|
1209 |
+
negative_crops_coords_top_left: Tuple[int, int] = (0, 0),
|
1210 |
+
negative_target_size: Optional[Tuple[int, int]] = None,
|
1211 |
+
aesthetic_score: float = 6.0,
|
1212 |
+
negative_aesthetic_score: float = 2.5,
|
1213 |
+
adapter_conditioning_scale: Union[float, List[float]] = 1.0,
|
1214 |
+
adapter_conditioning_factor: float = 1.0,
|
1215 |
+
clip_skip: Optional[int] = None,
|
1216 |
+
callback_on_step_end: Optional[
|
1217 |
+
Union[Callable[[int, int, Dict], None], PipelineCallback, MultiPipelineCallbacks]
|
1218 |
+
] = None,
|
1219 |
+
callback_on_step_end_tensor_inputs: List[str] = ["latents"],
|
1220 |
+
**kwargs,
|
1221 |
+
):
|
1222 |
+
r"""
|
1223 |
+
Function invoked when calling the pipeline for generation.
|
1224 |
+
|
1225 |
+
Args:
|
1226 |
+
prompt (`str` or `List[str]`, *optional*):
|
1227 |
+
The prompt or prompts to guide the image generation. If not defined, one has to pass `prompt_embeds`.
|
1228 |
+
instead.
|
1229 |
+
prompt_2 (`str` or `List[str]`, *optional*):
|
1230 |
+
The prompt or prompts to be sent to the `tokenizer_2` and `text_encoder_2`. If not defined, `prompt` is
|
1231 |
+
used in both text-encoders
|
1232 |
+
image (`PIL.Image.Image`):
|
1233 |
+
`Image`, or tensor representing an image batch which will be inpainted, *i.e.* parts of the image will
|
1234 |
+
be masked out with `mask_image` and repainted according to `prompt`.
|
1235 |
+
mask_image (`PIL.Image.Image`):
|
1236 |
+
`Image`, or tensor representing an image batch, to mask `image`. White pixels in the mask will be
|
1237 |
+
repainted, while black pixels will be preserved. If `mask_image` is a PIL image, it will be converted
|
1238 |
+
to a single channel (luminance) before use. If it's a tensor, it should contain one color channel (L)
|
1239 |
+
instead of 3, so the expected shape would be `(B, H, W, 1)`.
|
1240 |
+
height (`int`, *optional*, defaults to self.unet.config.sample_size * self.vae_scale_factor):
|
1241 |
+
The height in pixels of the generated image. This is set to 1024 by default for the best results.
|
1242 |
+
Anything below 512 pixels won't work well for
|
1243 |
+
[stabilityai/stable-diffusion-xl-base-1.0](https://huggingface.co/stabilityai/stable-diffusion-xl-base-1.0)
|
1244 |
+
and checkpoints that are not specifically fine-tuned on low resolutions.
|
1245 |
+
width (`int`, *optional*, defaults to self.unet.config.sample_size * self.vae_scale_factor):
|
1246 |
+
The width in pixels of the generated image. This is set to 1024 by default for the best results.
|
1247 |
+
Anything below 512 pixels won't work well for
|
1248 |
+
[stabilityai/stable-diffusion-xl-base-1.0](https://huggingface.co/stabilityai/stable-diffusion-xl-base-1.0)
|
1249 |
+
and checkpoints that are not specifically fine-tuned on low resolutions.
|
1250 |
+
padding_mask_crop (`int`, *optional*, defaults to `None`):
|
1251 |
+
The size of margin in the crop to be applied to the image and masking. If `None`, no crop is applied to
|
1252 |
+
image and mask_image. If `padding_mask_crop` is not `None`, it will first find a rectangular region
|
1253 |
+
with the same aspect ration of the image and contains all masked area, and then expand that area based
|
1254 |
+
on `padding_mask_crop`. The image and mask_image will then be cropped based on the expanded area before
|
1255 |
+
resizing to the original image size for inpainting. This is useful when the masked area is small while
|
1256 |
+
the image is large and contain information irrelevant for inpainting, such as background.
|
1257 |
+
strength (`float`, *optional*, defaults to 0.9999):
|
1258 |
+
Conceptually, indicates how much to transform the masked portion of the reference `image`. Must be
|
1259 |
+
between 0 and 1. `image` will be used as a starting point, adding more noise to it the larger the
|
1260 |
+
`strength`. The number of denoising steps depends on the amount of noise initially added. When
|
1261 |
+
`strength` is 1, added noise will be maximum and the denoising process will run for the full number of
|
1262 |
+
iterations specified in `num_inference_steps`. A value of 1, therefore, essentially ignores the masked
|
1263 |
+
portion of the reference `image`. Note that in the case of `denoising_start` being declared as an
|
1264 |
+
integer, the value of `strength` will be ignored.
|
1265 |
+
num_inference_steps (`int`, *optional*, defaults to 50):
|
1266 |
+
The number of denoising steps. More denoising steps usually lead to a higher quality image at the
|
1267 |
+
expense of slower inference.
|
1268 |
+
timesteps (`List[int]`, *optional*):
|
1269 |
+
Custom timesteps to use for the denoising process with schedulers which support a `timesteps` argument
|
1270 |
+
in their `set_timesteps` method. If not defined, the default behavior when `num_inference_steps` is
|
1271 |
+
passed will be used. Must be in descending order.
|
1272 |
+
sigmas (`List[float]`, *optional*):
|
1273 |
+
Custom sigmas to use for the denoising process with schedulers which support a `sigmas` argument in
|
1274 |
+
their `set_timesteps` method. If not defined, the default behavior when `num_inference_steps` is passed
|
1275 |
+
will be used.
|
1276 |
+
denoising_start (`float`, *optional*):
|
1277 |
+
When specified, indicates the fraction (between 0.0 and 1.0) of the total denoising process to be
|
1278 |
+
bypassed before it is initiated. Consequently, the initial part of the denoising process is skipped and
|
1279 |
+
it is assumed that the passed `image` is a partly denoised image. Note that when this is specified,
|
1280 |
+
strength will be ignored. The `denoising_start` parameter is particularly beneficial when this pipeline
|
1281 |
+
is integrated into a "Mixture of Denoisers" multi-pipeline setup, as detailed in [**Refining the Image
|
1282 |
+
Output**](https://huggingface.co/docs/diffusers/api/pipelines/stable_diffusion/stable_diffusion_xl#refining-the-image-output).
|
1283 |
+
denoising_end (`float`, *optional*):
|
1284 |
+
When specified, determines the fraction (between 0.0 and 1.0) of the total denoising process to be
|
1285 |
+
completed before it is intentionally prematurely terminated. As a result, the returned sample will
|
1286 |
+
still retain a substantial amount of noise (ca. final 20% of timesteps still needed) and should be
|
1287 |
+
denoised by a successor pipeline that has `denoising_start` set to 0.8 so that it only denoises the
|
1288 |
+
final 20% of the scheduler. The denoising_end parameter should ideally be utilized when this pipeline
|
1289 |
+
forms a part of a "Mixture of Denoisers" multi-pipeline setup, as elaborated in [**Refining the Image
|
1290 |
+
Output**](https://huggingface.co/docs/diffusers/api/pipelines/stable_diffusion/stable_diffusion_xl#refining-the-image-output).
|
1291 |
+
guidance_scale (`float`, *optional*, defaults to 7.5):
|
1292 |
+
Guidance scale as defined in [Classifier-Free Diffusion Guidance](https://arxiv.org/abs/2207.12598).
|
1293 |
+
`guidance_scale` is defined as `w` of equation 2. of [Imagen
|
1294 |
+
Paper](https://arxiv.org/pdf/2205.11487.pdf). Guidance scale is enabled by setting `guidance_scale >
|
1295 |
+
1`. Higher guidance scale encourages to generate images that are closely linked to the text `prompt`,
|
1296 |
+
usually at the expense of lower image quality.
|
1297 |
+
negative_prompt (`str` or `List[str]`, *optional*):
|
1298 |
+
The prompt or prompts not to guide the image generation. If not defined, one has to pass
|
1299 |
+
`negative_prompt_embeds` instead. Ignored when not using guidance (i.e., ignored if `guidance_scale` is
|
1300 |
+
less than `1`).
|
1301 |
+
negative_prompt_2 (`str` or `List[str]`, *optional*):
|
1302 |
+
The prompt or prompts not to guide the image generation to be sent to `tokenizer_2` and
|
1303 |
+
`text_encoder_2`. If not defined, `negative_prompt` is used in both text-encoders
|
1304 |
+
prompt_embeds (`torch.Tensor`, *optional*):
|
1305 |
+
Pre-generated text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt weighting. If not
|
1306 |
+
provided, text embeddings will be generated from `prompt` input argument.
|
1307 |
+
negative_prompt_embeds (`torch.Tensor`, *optional*):
|
1308 |
+
Pre-generated negative text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt
|
1309 |
+
weighting. If not provided, negative_prompt_embeds will be generated from `negative_prompt` input
|
1310 |
+
argument.
|
1311 |
+
pooled_prompt_embeds (`torch.Tensor`, *optional*):
|
1312 |
+
Pre-generated pooled text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt weighting.
|
1313 |
+
If not provided, pooled text embeddings will be generated from `prompt` input argument.
|
1314 |
+
negative_pooled_prompt_embeds (`torch.Tensor`, *optional*):
|
1315 |
+
Pre-generated negative pooled text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt
|
1316 |
+
weighting. If not provided, pooled negative_prompt_embeds will be generated from `negative_prompt`
|
1317 |
+
input argument.
|
1318 |
+
ip_adapter_image: (`PipelineImageInput`, *optional*): Optional image input to work with IP Adapters.
|
1319 |
+
ip_adapter_image_embeds (`List[torch.Tensor]`, *optional*):
|
1320 |
+
Pre-generated image embeddings for IP-Adapter. It should be a list of length same as number of
|
1321 |
+
IP-adapters. Each element should be a tensor of shape `(batch_size, num_images, emb_dim)`. It should
|
1322 |
+
contain the negative image embedding if `do_classifier_free_guidance` is set to `True`. If not
|
1323 |
+
provided, embeddings are computed from the `ip_adapter_image` input argument.
|
1324 |
+
num_images_per_prompt (`int`, *optional*, defaults to 1):
|
1325 |
+
The number of images to generate per prompt.
|
1326 |
+
eta (`float`, *optional*, defaults to 0.0):
|
1327 |
+
Corresponds to parameter eta (η) in the DDIM paper: https://arxiv.org/abs/2010.02502. Only applies to
|
1328 |
+
[`schedulers.DDIMScheduler`], will be ignored for others.
|
1329 |
+
generator (`torch.Generator`, *optional*):
|
1330 |
+
One or a list of [torch generator(s)](https://pytorch.org/docs/stable/generated/torch.Generator.html)
|
1331 |
+
to make generation deterministic.
|
1332 |
+
latents (`torch.Tensor`, *optional*):
|
1333 |
+
Pre-generated noisy latents, sampled from a Gaussian distribution, to be used as inputs for image
|
1334 |
+
generation. Can be used to tweak the same generation with different prompts. If not provided, a latents
|
1335 |
+
tensor will ge generated by sampling using the supplied random `generator`.
|
1336 |
+
output_type (`str`, *optional*, defaults to `"pil"`):
|
1337 |
+
The output format of the generate image. Choose between
|
1338 |
+
[PIL](https://pillow.readthedocs.io/en/stable/): `PIL.Image.Image` or `np.array`.
|
1339 |
+
return_dict (`bool`, *optional*, defaults to `True`):
|
1340 |
+
Whether or not to return a [`~pipelines.stable_diffusion.StableDiffusionPipelineOutput`] instead of a
|
1341 |
+
plain tuple.
|
1342 |
+
cross_attention_kwargs (`dict`, *optional*):
|
1343 |
+
A kwargs dictionary that if specified is passed along to the `AttentionProcessor` as defined under
|
1344 |
+
`self.processor` in
|
1345 |
+
[diffusers.models.attention_processor](https://github.com/huggingface/diffusers/blob/main/src/diffusers/models/attention_processor.py).
|
1346 |
+
original_size (`Tuple[int]`, *optional*, defaults to (1024, 1024)):
|
1347 |
+
If `original_size` is not the same as `target_size` the image will appear to be down- or upsampled.
|
1348 |
+
`original_size` defaults to `(height, width)` if not specified. Part of SDXL's micro-conditioning as
|
1349 |
+
explained in section 2.2 of
|
1350 |
+
[https://huggingface.co/papers/2307.01952](https://huggingface.co/papers/2307.01952).
|
1351 |
+
crops_coords_top_left (`Tuple[int]`, *optional*, defaults to (0, 0)):
|
1352 |
+
`crops_coords_top_left` can be used to generate an image that appears to be "cropped" from the position
|
1353 |
+
`crops_coords_top_left` downwards. Favorable, well-centered images are usually achieved by setting
|
1354 |
+
`crops_coords_top_left` to (0, 0). Part of SDXL's micro-conditioning as explained in section 2.2 of
|
1355 |
+
[https://huggingface.co/papers/2307.01952](https://huggingface.co/papers/2307.01952).
|
1356 |
+
target_size (`Tuple[int]`, *optional*, defaults to (1024, 1024)):
|
1357 |
+
For most cases, `target_size` should be set to the desired height and width of the generated image. If
|
1358 |
+
not specified it will default to `(height, width)`. Part of SDXL's micro-conditioning as explained in
|
1359 |
+
section 2.2 of [https://huggingface.co/papers/2307.01952](https://huggingface.co/papers/2307.01952).
|
1360 |
+
negative_original_size (`Tuple[int]`, *optional*, defaults to (1024, 1024)):
|
1361 |
+
To negatively condition the generation process based on a specific image resolution. Part of SDXL's
|
1362 |
+
micro-conditioning as explained in section 2.2 of
|
1363 |
+
[https://huggingface.co/papers/2307.01952](https://huggingface.co/papers/2307.01952). For more
|
1364 |
+
information, refer to this issue thread: https://github.com/huggingface/diffusers/issues/4208.
|
1365 |
+
negative_crops_coords_top_left (`Tuple[int]`, *optional*, defaults to (0, 0)):
|
1366 |
+
To negatively condition the generation process based on a specific crop coordinates. Part of SDXL's
|
1367 |
+
micro-conditioning as explained in section 2.2 of
|
1368 |
+
[https://huggingface.co/papers/2307.01952](https://huggingface.co/papers/2307.01952). For more
|
1369 |
+
information, refer to this issue thread: https://github.com/huggingface/diffusers/issues/4208.
|
1370 |
+
negative_target_size (`Tuple[int]`, *optional*, defaults to (1024, 1024)):
|
1371 |
+
To negatively condition the generation process based on a target image resolution. It should be as same
|
1372 |
+
as the `target_size` for most cases. Part of SDXL's micro-conditioning as explained in section 2.2 of
|
1373 |
+
[https://huggingface.co/papers/2307.01952](https://huggingface.co/papers/2307.01952). For more
|
1374 |
+
information, refer to this issue thread: https://github.com/huggingface/diffusers/issues/4208.
|
1375 |
+
aesthetic_score (`float`, *optional*, defaults to 6.0):
|
1376 |
+
Used to simulate an aesthetic score of the generated image by influencing the positive text condition.
|
1377 |
+
Part of SDXL's micro-conditioning as explained in section 2.2 of
|
1378 |
+
[https://huggingface.co/papers/2307.01952](https://huggingface.co/papers/2307.01952).
|
1379 |
+
negative_aesthetic_score (`float`, *optional*, defaults to 2.5):
|
1380 |
+
Part of SDXL's micro-conditioning as explained in section 2.2 of
|
1381 |
+
[https://huggingface.co/papers/2307.01952](https://huggingface.co/papers/2307.01952). Can be used to
|
1382 |
+
simulate an aesthetic score of the generated image by influencing the negative text condition.
|
1383 |
+
clip_skip (`int`, *optional*):
|
1384 |
+
Number of layers to be skipped from CLIP while computing the prompt embeddings. A value of 1 means that
|
1385 |
+
the output of the pre-final layer will be used for computing the prompt embeddings.
|
1386 |
+
callback_on_step_end (`Callable`, `PipelineCallback`, `MultiPipelineCallbacks`, *optional*):
|
1387 |
+
A function or a subclass of `PipelineCallback` or `MultiPipelineCallbacks` that is called at the end of
|
1388 |
+
each denoising step during the inference. with the following arguments: `callback_on_step_end(self:
|
1389 |
+
DiffusionPipeline, step: int, timestep: int, callback_kwargs: Dict)`. `callback_kwargs` will include a
|
1390 |
+
list of all tensors as specified by `callback_on_step_end_tensor_inputs`.
|
1391 |
+
callback_on_step_end_tensor_inputs (`List`, *optional*):
|
1392 |
+
The list of tensor inputs for the `callback_on_step_end` function. The tensors specified in the list
|
1393 |
+
will be passed as `callback_kwargs` argument. You will only be able to include variables listed in the
|
1394 |
+
`._callback_tensor_inputs` attribute of your pipeline class.
|
1395 |
+
|
1396 |
+
Examples:
|
1397 |
+
|
1398 |
+
Returns:
|
1399 |
+
[`~pipelines.stable_diffusion.StableDiffusionXLPipelineOutput`] or `tuple`:
|
1400 |
+
[`~pipelines.stable_diffusion.StableDiffusionXLPipelineOutput`] if `return_dict` is True, otherwise a
|
1401 |
+
`tuple. `tuple. When returning a tuple, the first element is a list with the generated images.
|
1402 |
+
"""
|
1403 |
+
height, width = self._default_height_width(height, width, adapter_image)
|
1404 |
+
device = self._execution_device
|
1405 |
+
|
1406 |
+
if isinstance(self.adapter, MultiAdapter):
|
1407 |
+
adapter_input = []
|
1408 |
+
|
1409 |
+
for one_image in adapter_image:
|
1410 |
+
one_image = _preprocess_adapter_image(one_image, height, width)
|
1411 |
+
one_image = one_image.to(device=device, dtype=self.adapter.dtype)
|
1412 |
+
adapter_input.append(one_image)
|
1413 |
+
else:
|
1414 |
+
adapter_input = _preprocess_adapter_image(adapter_image, height, width)
|
1415 |
+
adapter_input = adapter_input.to(device=device, dtype=self.adapter.dtype)
|
1416 |
+
|
1417 |
+
callback = kwargs.pop("callback", None)
|
1418 |
+
callback_steps = kwargs.pop("callback_steps", None)
|
1419 |
+
|
1420 |
+
if callback is not None:
|
1421 |
+
deprecate(
|
1422 |
+
"callback",
|
1423 |
+
"1.0.0",
|
1424 |
+
"Passing `callback` as an input argument to `__call__` is deprecated, consider use `callback_on_step_end`",
|
1425 |
+
)
|
1426 |
+
if callback_steps is not None:
|
1427 |
+
deprecate(
|
1428 |
+
"callback_steps",
|
1429 |
+
"1.0.0",
|
1430 |
+
"Passing `callback_steps` as an input argument to `__call__` is deprecated, consider use `callback_on_step_end`",
|
1431 |
+
)
|
1432 |
+
|
1433 |
+
if isinstance(callback_on_step_end, (PipelineCallback, MultiPipelineCallbacks)):
|
1434 |
+
callback_on_step_end_tensor_inputs = callback_on_step_end.tensor_inputs
|
1435 |
+
|
1436 |
+
# 0. Default height and width to unet
|
1437 |
+
height = height or self.unet.config.sample_size * self.vae_scale_factor
|
1438 |
+
width = width or self.unet.config.sample_size * self.vae_scale_factor
|
1439 |
+
|
1440 |
+
# 1. Check inputs
|
1441 |
+
self.check_inputs(
|
1442 |
+
prompt,
|
1443 |
+
prompt_2,
|
1444 |
+
image,
|
1445 |
+
mask_image,
|
1446 |
+
height,
|
1447 |
+
width,
|
1448 |
+
strength,
|
1449 |
+
callback_steps,
|
1450 |
+
output_type,
|
1451 |
+
negative_prompt,
|
1452 |
+
negative_prompt_2,
|
1453 |
+
prompt_embeds,
|
1454 |
+
negative_prompt_embeds,
|
1455 |
+
ip_adapter_image,
|
1456 |
+
ip_adapter_image_embeds,
|
1457 |
+
callback_on_step_end_tensor_inputs,
|
1458 |
+
padding_mask_crop,
|
1459 |
+
)
|
1460 |
+
|
1461 |
+
self._guidance_scale = guidance_scale
|
1462 |
+
self._guidance_rescale = guidance_rescale
|
1463 |
+
self._clip_skip = clip_skip
|
1464 |
+
self._cross_attention_kwargs = cross_attention_kwargs
|
1465 |
+
self._denoising_end = denoising_end
|
1466 |
+
self._denoising_start = denoising_start
|
1467 |
+
self._interrupt = False
|
1468 |
+
|
1469 |
+
# 2. Define call parameters
|
1470 |
+
if prompt is not None and isinstance(prompt, str):
|
1471 |
+
batch_size = 1
|
1472 |
+
elif prompt is not None and isinstance(prompt, list):
|
1473 |
+
batch_size = len(prompt)
|
1474 |
+
else:
|
1475 |
+
batch_size = prompt_embeds.shape[0]
|
1476 |
+
|
1477 |
+
device = self._execution_device
|
1478 |
+
|
1479 |
+
# 3. Encode input prompt
|
1480 |
+
text_encoder_lora_scale = (
|
1481 |
+
self.cross_attention_kwargs.get("scale", None) if self.cross_attention_kwargs is not None else None
|
1482 |
+
)
|
1483 |
+
|
1484 |
+
(
|
1485 |
+
prompt_embeds,
|
1486 |
+
negative_prompt_embeds,
|
1487 |
+
pooled_prompt_embeds,
|
1488 |
+
negative_pooled_prompt_embeds,
|
1489 |
+
) = self.encode_prompt(
|
1490 |
+
prompt=prompt,
|
1491 |
+
prompt_2=prompt_2,
|
1492 |
+
device=device,
|
1493 |
+
num_images_per_prompt=num_images_per_prompt,
|
1494 |
+
do_classifier_free_guidance=self.do_classifier_free_guidance,
|
1495 |
+
negative_prompt=negative_prompt,
|
1496 |
+
negative_prompt_2=negative_prompt_2,
|
1497 |
+
prompt_embeds=prompt_embeds,
|
1498 |
+
negative_prompt_embeds=negative_prompt_embeds,
|
1499 |
+
pooled_prompt_embeds=pooled_prompt_embeds,
|
1500 |
+
negative_pooled_prompt_embeds=negative_pooled_prompt_embeds,
|
1501 |
+
lora_scale=text_encoder_lora_scale,
|
1502 |
+
clip_skip=self.clip_skip,
|
1503 |
+
)
|
1504 |
+
|
1505 |
+
# 4. set timesteps
|
1506 |
+
def denoising_value_valid(dnv):
|
1507 |
+
return isinstance(dnv, float) and 0 < dnv < 1
|
1508 |
+
|
1509 |
+
timesteps, num_inference_steps = retrieve_timesteps(
|
1510 |
+
self.scheduler, num_inference_steps, device, timesteps, sigmas
|
1511 |
+
)
|
1512 |
+
timesteps, num_inference_steps = self.get_timesteps(
|
1513 |
+
num_inference_steps,
|
1514 |
+
strength,
|
1515 |
+
device,
|
1516 |
+
denoising_start=self.denoising_start if denoising_value_valid(self.denoising_start) else None,
|
1517 |
+
)
|
1518 |
+
# check that number of inference steps is not < 1 - as this doesn't make sense
|
1519 |
+
if num_inference_steps < 1:
|
1520 |
+
raise ValueError(
|
1521 |
+
f"After adjusting the num_inference_steps by strength parameter: {strength}, the number of pipeline"
|
1522 |
+
f"steps is {num_inference_steps} which is < 1 and not appropriate for this pipeline."
|
1523 |
+
)
|
1524 |
+
# at which timestep to set the initial noise (n.b. 50% if strength is 0.5)
|
1525 |
+
latent_timestep = timesteps[:1].repeat(batch_size * num_images_per_prompt)
|
1526 |
+
# create a boolean to check if the strength is set to 1. if so then initialise the latents with pure noise
|
1527 |
+
is_strength_max = strength == 1.0
|
1528 |
+
|
1529 |
+
# 5. Preprocess mask and image
|
1530 |
+
if padding_mask_crop is not None:
|
1531 |
+
crops_coords = self.mask_processor.get_crop_region(mask_image, width, height, pad=padding_mask_crop)
|
1532 |
+
resize_mode = "fill"
|
1533 |
+
else:
|
1534 |
+
crops_coords = None
|
1535 |
+
resize_mode = "default"
|
1536 |
+
|
1537 |
+
original_image = image
|
1538 |
+
init_image = self.image_processor.preprocess(
|
1539 |
+
image, height=height, width=width, crops_coords=crops_coords, resize_mode=resize_mode
|
1540 |
+
)
|
1541 |
+
init_image = init_image.to(dtype=torch.float32)
|
1542 |
+
|
1543 |
+
mask = self.mask_processor.preprocess(
|
1544 |
+
mask_image, height=height, width=width, resize_mode=resize_mode, crops_coords=crops_coords
|
1545 |
+
)
|
1546 |
+
|
1547 |
+
if masked_image_latents is not None:
|
1548 |
+
masked_image = masked_image_latents
|
1549 |
+
elif init_image.shape[1] == 4:
|
1550 |
+
# if images are in latent space, we can't mask it
|
1551 |
+
masked_image = None
|
1552 |
+
else:
|
1553 |
+
masked_image = init_image * (mask < 0.5)
|
1554 |
+
|
1555 |
+
# 6. Prepare latent variables
|
1556 |
+
num_channels_latents = self.vae.config.latent_channels
|
1557 |
+
num_channels_unet = self.unet.config.in_channels
|
1558 |
+
return_image_latents = num_channels_unet == 4
|
1559 |
+
|
1560 |
+
add_noise = True if self.denoising_start is None else False
|
1561 |
+
latents_outputs = self.prepare_latents(
|
1562 |
+
batch_size * num_images_per_prompt,
|
1563 |
+
num_channels_latents,
|
1564 |
+
height,
|
1565 |
+
width,
|
1566 |
+
prompt_embeds.dtype,
|
1567 |
+
device,
|
1568 |
+
generator,
|
1569 |
+
latents,
|
1570 |
+
image=init_image,
|
1571 |
+
timestep=latent_timestep,
|
1572 |
+
is_strength_max=is_strength_max,
|
1573 |
+
add_noise=add_noise,
|
1574 |
+
return_noise=True,
|
1575 |
+
return_image_latents=return_image_latents,
|
1576 |
+
)
|
1577 |
+
|
1578 |
+
if return_image_latents:
|
1579 |
+
latents, noise, image_latents = latents_outputs
|
1580 |
+
else:
|
1581 |
+
latents, noise = latents_outputs
|
1582 |
+
|
1583 |
+
# 7. Prepare mask latent variables
|
1584 |
+
mask, masked_image_latents = self.prepare_mask_latents(
|
1585 |
+
mask,
|
1586 |
+
masked_image,
|
1587 |
+
batch_size * num_images_per_prompt,
|
1588 |
+
height,
|
1589 |
+
width,
|
1590 |
+
prompt_embeds.dtype,
|
1591 |
+
device,
|
1592 |
+
generator,
|
1593 |
+
self.do_classifier_free_guidance,
|
1594 |
+
)
|
1595 |
+
|
1596 |
+
# 8. Check that sizes of mask, masked image and latents match
|
1597 |
+
if num_channels_unet == 9:
|
1598 |
+
# default case for runwayml/stable-diffusion-inpainting
|
1599 |
+
num_channels_mask = mask.shape[1]
|
1600 |
+
num_channels_masked_image = masked_image_latents.shape[1]
|
1601 |
+
if num_channels_latents + num_channels_mask + num_channels_masked_image != self.unet.config.in_channels:
|
1602 |
+
raise ValueError(
|
1603 |
+
f"Incorrect configuration settings! The config of `pipeline.unet`: {self.unet.config} expects"
|
1604 |
+
f" {self.unet.config.in_channels} but received `num_channels_latents`: {num_channels_latents} +"
|
1605 |
+
f" `num_channels_mask`: {num_channels_mask} + `num_channels_masked_image`: {num_channels_masked_image}"
|
1606 |
+
f" = {num_channels_latents+num_channels_masked_image+num_channels_mask}. Please verify the config of"
|
1607 |
+
" `pipeline.unet` or your `mask_image` or `image` input."
|
1608 |
+
)
|
1609 |
+
elif num_channels_unet != 4:
|
1610 |
+
raise ValueError(
|
1611 |
+
f"The unet {self.unet.__class__} should have either 4 or 9 input channels, not {self.unet.config.in_channels}."
|
1612 |
+
)
|
1613 |
+
# 8.1 Prepare extra step kwargs.
|
1614 |
+
extra_step_kwargs = self.prepare_extra_step_kwargs(generator, eta)
|
1615 |
+
|
1616 |
+
# 9. Prepare extra step kwargs. TODO: Logic should ideally just be moved out of the pipeline
|
1617 |
+
height, width = latents.shape[-2:]
|
1618 |
+
height = height * self.vae_scale_factor
|
1619 |
+
width = width * self.vae_scale_factor
|
1620 |
+
|
1621 |
+
original_size = original_size or (height, width)
|
1622 |
+
target_size = target_size or (height, width)
|
1623 |
+
|
1624 |
+
# 10. Prepare added time ids & embeddings
|
1625 |
+
if isinstance(self.adapter, MultiAdapter):
|
1626 |
+
adapter_state = self.adapter(adapter_input, adapter_conditioning_scale)
|
1627 |
+
for k, v in enumerate(adapter_state):
|
1628 |
+
adapter_state[k] = v
|
1629 |
+
else:
|
1630 |
+
adapter_state = self.adapter(adapter_input)
|
1631 |
+
for k, v in enumerate(adapter_state):
|
1632 |
+
adapter_state[k] = v * adapter_conditioning_scale
|
1633 |
+
if num_images_per_prompt > 1:
|
1634 |
+
for k, v in enumerate(adapter_state):
|
1635 |
+
adapter_state[k] = v.repeat(num_images_per_prompt, 1, 1, 1)
|
1636 |
+
if self.do_classifier_free_guidance:
|
1637 |
+
for k, v in enumerate(adapter_state):
|
1638 |
+
adapter_state[k] = torch.cat([v] * 2, dim=0)
|
1639 |
+
|
1640 |
+
if negative_original_size is None:
|
1641 |
+
negative_original_size = original_size
|
1642 |
+
if negative_target_size is None:
|
1643 |
+
negative_target_size = target_size
|
1644 |
+
|
1645 |
+
add_text_embeds = pooled_prompt_embeds
|
1646 |
+
if self.text_encoder_2 is None:
|
1647 |
+
text_encoder_projection_dim = int(pooled_prompt_embeds.shape[-1])
|
1648 |
+
else:
|
1649 |
+
text_encoder_projection_dim = self.text_encoder_2.config.projection_dim
|
1650 |
+
|
1651 |
+
add_time_ids, add_neg_time_ids = self._get_add_time_ids(
|
1652 |
+
original_size,
|
1653 |
+
crops_coords_top_left,
|
1654 |
+
target_size,
|
1655 |
+
aesthetic_score,
|
1656 |
+
negative_aesthetic_score,
|
1657 |
+
negative_original_size,
|
1658 |
+
negative_crops_coords_top_left,
|
1659 |
+
negative_target_size,
|
1660 |
+
dtype=prompt_embeds.dtype,
|
1661 |
+
text_encoder_projection_dim=text_encoder_projection_dim,
|
1662 |
+
)
|
1663 |
+
add_time_ids = add_time_ids.repeat(batch_size * num_images_per_prompt, 1)
|
1664 |
+
|
1665 |
+
if self.do_classifier_free_guidance:
|
1666 |
+
prompt_embeds = torch.cat([negative_prompt_embeds, prompt_embeds], dim=0)
|
1667 |
+
add_text_embeds = torch.cat([negative_pooled_prompt_embeds, add_text_embeds], dim=0)
|
1668 |
+
add_neg_time_ids = add_neg_time_ids.repeat(batch_size * num_images_per_prompt, 1)
|
1669 |
+
add_time_ids = torch.cat([add_neg_time_ids, add_time_ids], dim=0)
|
1670 |
+
|
1671 |
+
prompt_embeds = prompt_embeds.to(device)
|
1672 |
+
add_text_embeds = add_text_embeds.to(device)
|
1673 |
+
add_time_ids = add_time_ids.to(device)
|
1674 |
+
|
1675 |
+
if ip_adapter_image is not None or ip_adapter_image_embeds is not None:
|
1676 |
+
image_embeds = self.prepare_ip_adapter_image_embeds(
|
1677 |
+
ip_adapter_image,
|
1678 |
+
ip_adapter_image_embeds,
|
1679 |
+
device,
|
1680 |
+
batch_size * num_images_per_prompt,
|
1681 |
+
self.do_classifier_free_guidance,
|
1682 |
+
)
|
1683 |
+
|
1684 |
+
# 11. Denoising loop
|
1685 |
+
num_warmup_steps = max(len(timesteps) - num_inference_steps * self.scheduler.order, 0)
|
1686 |
+
|
1687 |
+
if (
|
1688 |
+
self.denoising_end is not None
|
1689 |
+
and self.denoising_start is not None
|
1690 |
+
and denoising_value_valid(self.denoising_end)
|
1691 |
+
and denoising_value_valid(self.denoising_start)
|
1692 |
+
and self.denoising_start >= self.denoising_end
|
1693 |
+
):
|
1694 |
+
raise ValueError(
|
1695 |
+
f"`denoising_start`: {self.denoising_start} cannot be larger than or equal to `denoising_end`: "
|
1696 |
+
+ f" {self.denoising_end} when using type float."
|
1697 |
+
)
|
1698 |
+
elif self.denoising_end is not None and denoising_value_valid(self.denoising_end):
|
1699 |
+
discrete_timestep_cutoff = int(
|
1700 |
+
round(
|
1701 |
+
self.scheduler.config.num_train_timesteps
|
1702 |
+
- (self.denoising_end * self.scheduler.config.num_train_timesteps)
|
1703 |
+
)
|
1704 |
+
)
|
1705 |
+
num_inference_steps = len(list(filter(lambda ts: ts >= discrete_timestep_cutoff, timesteps)))
|
1706 |
+
timesteps = timesteps[:num_inference_steps]
|
1707 |
+
|
1708 |
+
# 11.1 Optionally get Guidance Scale Embedding
|
1709 |
+
timestep_cond = None
|
1710 |
+
if self.unet.config.time_cond_proj_dim is not None:
|
1711 |
+
guidance_scale_tensor = torch.tensor(self.guidance_scale - 1).repeat(batch_size * num_images_per_prompt)
|
1712 |
+
timestep_cond = self.get_guidance_scale_embedding(
|
1713 |
+
guidance_scale_tensor, embedding_dim=self.unet.config.time_cond_proj_dim
|
1714 |
+
).to(device=device, dtype=latents.dtype)
|
1715 |
+
|
1716 |
+
self._num_timesteps = len(timesteps)
|
1717 |
+
with self.progress_bar(total=num_inference_steps) as progress_bar:
|
1718 |
+
for i, t in enumerate(timesteps):
|
1719 |
+
if self.interrupt:
|
1720 |
+
continue
|
1721 |
+
# expand the latents if we are doing classifier free guidance
|
1722 |
+
latent_model_input = torch.cat([latents] * 2) if self.do_classifier_free_guidance else latents
|
1723 |
+
|
1724 |
+
# concat latents, mask, masked_image_latents in the channel dimension
|
1725 |
+
latent_model_input = self.scheduler.scale_model_input(latent_model_input, t)
|
1726 |
+
|
1727 |
+
if num_channels_unet == 9:
|
1728 |
+
latent_model_input = torch.cat([latent_model_input, mask, masked_image_latents], dim=1)
|
1729 |
+
|
1730 |
+
# predict the noise residual
|
1731 |
+
added_cond_kwargs = {"text_embeds": add_text_embeds, "time_ids": add_time_ids}
|
1732 |
+
if ip_adapter_image is not None or ip_adapter_image_embeds is not None:
|
1733 |
+
added_cond_kwargs["image_embeds"] = image_embeds
|
1734 |
+
|
1735 |
+
if i < int(num_inference_steps * adapter_conditioning_factor):
|
1736 |
+
down_intrablock_additional_residuals = [state.clone() for state in adapter_state]
|
1737 |
+
else:
|
1738 |
+
down_intrablock_additional_residuals = None
|
1739 |
+
|
1740 |
+
noise_pred = self.unet(
|
1741 |
+
latent_model_input,
|
1742 |
+
t,
|
1743 |
+
encoder_hidden_states=prompt_embeds,
|
1744 |
+
timestep_cond=timestep_cond,
|
1745 |
+
cross_attention_kwargs=self.cross_attention_kwargs,
|
1746 |
+
added_cond_kwargs=added_cond_kwargs,
|
1747 |
+
return_dict=False,
|
1748 |
+
down_intrablock_additional_residuals=down_intrablock_additional_residuals,
|
1749 |
+
)[0]
|
1750 |
+
|
1751 |
+
# perform guidance
|
1752 |
+
if self.do_classifier_free_guidance:
|
1753 |
+
noise_pred_uncond, noise_pred_text = noise_pred.chunk(2)
|
1754 |
+
noise_pred = noise_pred_uncond + self.guidance_scale * (noise_pred_text - noise_pred_uncond)
|
1755 |
+
|
1756 |
+
if self.do_classifier_free_guidance and self.guidance_rescale > 0.0:
|
1757 |
+
# Based on 3.4. in https://arxiv.org/pdf/2305.08891.pdf
|
1758 |
+
noise_pred = rescale_noise_cfg(noise_pred, noise_pred_text, guidance_rescale=self.guidance_rescale)
|
1759 |
+
|
1760 |
+
# compute the previous noisy sample x_t -> x_t-1
|
1761 |
+
latents_dtype = latents.dtype
|
1762 |
+
latents = self.scheduler.step(noise_pred, t, latents, **extra_step_kwargs, return_dict=False)[0]
|
1763 |
+
if latents.dtype != latents_dtype:
|
1764 |
+
if torch.backends.mps.is_available():
|
1765 |
+
# some platforms (eg. apple mps) misbehave due to a pytorch bug: https://github.com/pytorch/pytorch/pull/99272
|
1766 |
+
latents = latents.to(latents_dtype)
|
1767 |
+
|
1768 |
+
if num_channels_unet == 4:
|
1769 |
+
init_latents_proper = image_latents
|
1770 |
+
if self.do_classifier_free_guidance:
|
1771 |
+
init_mask, _ = mask.chunk(2)
|
1772 |
+
else:
|
1773 |
+
init_mask = mask
|
1774 |
+
|
1775 |
+
if i < len(timesteps) - 1:
|
1776 |
+
noise_timestep = timesteps[i + 1]
|
1777 |
+
init_latents_proper = self.scheduler.add_noise(
|
1778 |
+
init_latents_proper, noise, torch.tensor([noise_timestep])
|
1779 |
+
)
|
1780 |
+
|
1781 |
+
latents = (1 - init_mask) * init_latents_proper + init_mask * latents
|
1782 |
+
|
1783 |
+
if callback_on_step_end is not None:
|
1784 |
+
callback_kwargs = {}
|
1785 |
+
for k in callback_on_step_end_tensor_inputs:
|
1786 |
+
callback_kwargs[k] = locals()[k]
|
1787 |
+
callback_outputs = callback_on_step_end(self, i, t, callback_kwargs)
|
1788 |
+
|
1789 |
+
latents = callback_outputs.pop("latents", latents)
|
1790 |
+
prompt_embeds = callback_outputs.pop("prompt_embeds", prompt_embeds)
|
1791 |
+
negative_prompt_embeds = callback_outputs.pop("negative_prompt_embeds", negative_prompt_embeds)
|
1792 |
+
add_text_embeds = callback_outputs.pop("add_text_embeds", add_text_embeds)
|
1793 |
+
negative_pooled_prompt_embeds = callback_outputs.pop(
|
1794 |
+
"negative_pooled_prompt_embeds", negative_pooled_prompt_embeds
|
1795 |
+
)
|
1796 |
+
add_time_ids = callback_outputs.pop("add_time_ids", add_time_ids)
|
1797 |
+
add_neg_time_ids = callback_outputs.pop("add_neg_time_ids", add_neg_time_ids)
|
1798 |
+
mask = callback_outputs.pop("mask", mask)
|
1799 |
+
masked_image_latents = callback_outputs.pop("masked_image_latents", masked_image_latents)
|
1800 |
+
|
1801 |
+
# call the callback, if provided
|
1802 |
+
if i == len(timesteps) - 1 or ((i + 1) > num_warmup_steps and (i + 1) % self.scheduler.order == 0):
|
1803 |
+
progress_bar.update()
|
1804 |
+
if callback is not None and i % callback_steps == 0:
|
1805 |
+
step_idx = i // getattr(self.scheduler, "order", 1)
|
1806 |
+
callback(step_idx, t, latents)
|
1807 |
+
|
1808 |
+
if XLA_AVAILABLE:
|
1809 |
+
xm.mark_step()
|
1810 |
+
|
1811 |
+
if not output_type == "latent":
|
1812 |
+
# make sure the VAE is in float32 mode, as it overflows in float16
|
1813 |
+
needs_upcasting = self.vae.dtype == torch.float16 and self.vae.config.force_upcast
|
1814 |
+
|
1815 |
+
if needs_upcasting:
|
1816 |
+
self.upcast_vae()
|
1817 |
+
latents = latents.to(next(iter(self.vae.post_quant_conv.parameters())).dtype)
|
1818 |
+
elif latents.dtype != self.vae.dtype:
|
1819 |
+
if torch.backends.mps.is_available():
|
1820 |
+
# some platforms (eg. apple mps) misbehave due to a pytorch bug: https://github.com/pytorch/pytorch/pull/99272
|
1821 |
+
self.vae = self.vae.to(latents.dtype)
|
1822 |
+
|
1823 |
+
# unscale/denormalize the latents
|
1824 |
+
# denormalize with the mean and std if available and not None
|
1825 |
+
has_latents_mean = hasattr(self.vae.config, "latents_mean") and self.vae.config.latents_mean is not None
|
1826 |
+
has_latents_std = hasattr(self.vae.config, "latents_std") and self.vae.config.latents_std is not None
|
1827 |
+
if has_latents_mean and has_latents_std:
|
1828 |
+
latents_mean = (
|
1829 |
+
torch.tensor(self.vae.config.latents_mean).view(1, 4, 1, 1).to(latents.device, latents.dtype)
|
1830 |
+
)
|
1831 |
+
latents_std = (
|
1832 |
+
torch.tensor(self.vae.config.latents_std).view(1, 4, 1, 1).to(latents.device, latents.dtype)
|
1833 |
+
)
|
1834 |
+
latents = latents * latents_std / self.vae.config.scaling_factor + latents_mean
|
1835 |
+
else:
|
1836 |
+
latents = latents / self.vae.config.scaling_factor
|
1837 |
+
|
1838 |
+
image = self.vae.decode(latents, return_dict=False)[0]
|
1839 |
+
|
1840 |
+
# cast back to fp16 if needed
|
1841 |
+
if needs_upcasting:
|
1842 |
+
self.vae.to(dtype=torch.float16)
|
1843 |
+
else:
|
1844 |
+
return StableDiffusionXLPipelineOutput(images=latents)
|
1845 |
+
|
1846 |
+
# apply watermark if available
|
1847 |
+
if self.watermark is not None:
|
1848 |
+
image = self.watermark.apply_watermark(image)
|
1849 |
+
|
1850 |
+
image = self.image_processor.postprocess(image, output_type=output_type)
|
1851 |
+
|
1852 |
+
if padding_mask_crop is not None:
|
1853 |
+
image = [self.image_processor.apply_overlay(mask_image, original_image, i, crops_coords) for i in image]
|
1854 |
+
|
1855 |
+
# Offload all models
|
1856 |
+
self.maybe_free_model_hooks()
|
1857 |
+
|
1858 |
+
if not return_dict:
|
1859 |
+
return (image,)
|
1860 |
+
|
1861 |
+
return StableDiffusionXLPipelineOutput(images=image)
|