# https://github.com/showlab/X-Adapter
import torch
import diffusers
import gradio as gr
import huggingface_hub as hf
from modules import errors, shared, devices, scripts, processing, sd_models, sd_samplers
adapter = None
class Script(scripts.Script):
def title(self):
return 'X-Adapter'
def show(self, is_img2img):
return False
# return True if shared.backend == shared.Backend.DIFFUSERS else False
def ui(self, _is_img2img):
with gr.Row():
gr.HTML('  X-Adapter
')
with gr.Row():
model = gr.Dropdown(label='Adapter model', choices=['None'] + sd_models.checkpoint_tiles(), value='None')
sampler = gr.Dropdown(label='Adapter sampler', choices=[s.name for s in sd_samplers.samplers], value='Default')
with gr.Row():
width = gr.Slider(label='Adapter width', minimum=64, maximum=2048, step=8, value=512)
height = gr.Slider(label='Adapter height', minimum=64, maximum=2048, step=8, value=512)
with gr.Row():
start = gr.Slider(label='Adapter start', minimum=0.0, maximum=1.0, step=0.01, value=0.5)
scale = gr.Slider(label='Adapter scale', minimum=0.0, maximum=1.0, step=0.01, value=1.0)
with gr.Row():
lora = gr.Textbox('', label='Adapter LoRA', default='')
return model, sampler, width, height, start, scale, lora
def run(self, p: processing.StableDiffusionProcessing, model, sampler, width, height, start, scale, lora): # pylint: disable=arguments-differ
from modules.xadapter.xadapter_hijacks import PositionNet
diffusers.models.embeddings.PositionNet = PositionNet # patch diffusers==0.26 from diffusers==0.20
from modules.xadapter.adapter import Adapter_XL
from modules.xadapter.pipeline_sd_xl_adapter import StableDiffusionXLAdapterPipeline
from modules.xadapter.unet_adapter import UNet2DConditionModel as UNet2DConditionModelAdapter
global adapter # pylint: disable=global-statement
if model == 'None':
return
else:
shared.opts.sd_model_refiner = model
if shared.sd_model_type != 'sdxl':
shared.log.error(f'X-Adapter: incorrect base model: {shared.sd_model.__class__.__name__}')
return
if adapter is None:
shared.log.debug('X-Adapter: adapter loading')
adapter = Adapter_XL()
adapter_path = hf.hf_hub_download(repo_id='Lingmin-Ran/X-Adapter', filename='X_Adapter_v1.bin')
adapter_dict = torch.load(adapter_path)
adapter.load_state_dict(adapter_dict)
try:
if adapter is not None:
sd_models.move_model(adapter, devices.device)
except Exception:
pass
if adapter is None:
shared.log.error('X-Adapter: adapter loading failed')
return
sd_models.unload_model_weights(op='model')
sd_models.unload_model_weights(op='refiner')
orig_unetcondmodel = diffusers.models.unets.unet_2d_condition.UNet2DConditionModel
diffusers.models.UNet2DConditionModel = UNet2DConditionModelAdapter # patch diffusers with x-adapter
diffusers.models.unets.unet_2d_condition.UNet2DConditionModel = UNet2DConditionModelAdapter # patch diffusers with x-adapter
sd_models.reload_model_weights(op='model')
sd_models.reload_model_weights(op='refiner')
diffusers.models.unets.unet_2d_condition.UNet2DConditionModel = orig_unetcondmodel # unpatch diffusers
diffusers.models.UNet2DConditionModel = orig_unetcondmodel # unpatch diffusers
if shared.sd_refiner_type != 'sd':
shared.log.error(f'X-Adapter: incorrect adapter model: {shared.sd_model.__class__.__name__}')
return
# backup pipeline and params
orig_pipeline = shared.sd_model
orig_prompt_attention = shared.opts.prompt_attention
pipe = None
try:
shared.log.debug('X-Adapter: creating pipeline')
pipe = StableDiffusionXLAdapterPipeline(
vae=shared.sd_model.vae,
text_encoder=shared.sd_model.text_encoder,
text_encoder_2=shared.sd_model.text_encoder_2,
tokenizer=shared.sd_model.tokenizer,
tokenizer_2=shared.sd_model.tokenizer_2,
unet=shared.sd_model.unet,
scheduler=shared.sd_model.scheduler,
vae_sd1_5=shared.sd_refiner.vae,
text_encoder_sd1_5=shared.sd_refiner.text_encoder,
tokenizer_sd1_5=shared.sd_refiner.tokenizer,
unet_sd1_5=shared.sd_refiner.unet,
scheduler_sd1_5=shared.sd_refiner.scheduler,
adapter=adapter,
)
sd_models.copy_diffuser_options(pipe, shared.sd_model)
sd_models.set_diffuser_options(pipe)
try:
pipe.to(device=devices.device, dtype=devices.dtype)
except Exception:
pass
shared.opts.data['prompt_attention'] = 'Fixed attention'
prompt = shared.prompt_styles.apply_styles_to_prompt(p.prompt, p.styles)
negative = shared.prompt_styles.apply_negative_styles_to_prompt(p.negative_prompt, p.styles)
p.task_args['prompt'] = prompt
p.task_args['negative_prompt'] = negative
p.task_args['prompt_sd1_5'] = prompt
p.task_args['width_sd1_5'] = width
p.task_args['height_sd1_5'] = height
p.task_args['adapter_guidance_start'] = start
p.task_args['adapter_condition_scale'] = scale
p.task_args['fusion_guidance_scale'] = 1.0 # ???
if sampler != 'Default':
pipe.scheduler_sd1_5 = sd_samplers.create_sampler(sampler, shared.sd_refiner)
else:
pipe.scheduler = diffusers.DPMSolverMultistepScheduler.from_config(pipe.scheduler.config)
pipe.scheduler_sd1_5 = diffusers.DPMSolverMultistepScheduler.from_config(pipe.scheduler_sd1_5.config)
pipe.scheduler_sd1_5.config.timestep_spacing = "leading"
shared.log.debug(f'X-Adapter: pipeline={pipe.__class__.__name__} args={p.task_args}')
shared.sd_model = pipe
except Exception as e:
shared.log.error(f'X-Adapter: pipeline creation failed: {e}')
errors.display(e, 'X-Adapter: pipeline creation failed')
shared.sd_model = orig_pipeline
# run pipeline
processed: processing.Processed = processing.process_images(p) # runs processing using main loop
# restore pipeline and params
try:
if adapter is not None:
adapter.to(devices.cpu)
except Exception:
pass
pipe = None
shared.opts.data['prompt_attention'] = orig_prompt_attention
shared.sd_model = orig_pipeline
devices.torch_gc()
return processed