import torch import time from PIL import Image from lyrasd_model import LyraSdControlnetTxt2ImgPipeline # 存放模型文件的路径,应该包含一下结构: # 1. clip 模型 # 2. 转换好的优化后的 unet 模型 # 3. 转换好的优化后的 controlnet 模型 # 4. vae 模型 # 5. scheduler 配置 # LyraSD 的 C++ 编译动态链接库,其中包含 C++ CUDA 计算的细节 lib_path = "./lyrasd_model/lyrasd_lib/libth_lyrasd_cu12_sm86.so" model_path = "./models/lyrasd_rev_animated" canny_controlnet_path = "./models/lyrasd_canny" # 构建 Txt2Img 的 Pipeline pipe = LyraSdControlnetTxt2ImgPipeline(model_path, lib_path) # load Controlnet 模型,最多load 3个 start = time.perf_counter() pipe.load_controlnet_model("canny", canny_controlnet_path, "fp32") print(f"controlnet load cost: {time.perf_counter() - start}") # 可以通过 get_loaded_controlnet 方法获取目前已经load 好的Controlnet list print(pipe.get_loaded_controlnet()) # 可以通过unload_controlnet_model 方法unload Controlnet # pipe.unload_controlnet_model("canny") control_img = Image.open("control_bird_canny.png") # 准备应用的输入和超参数 prompt = "a blue bird" negative_prompt = "NSFW" height, width = 512, 512 steps = 20 guidance_scale = 7.5 generator = torch.Generator().manual_seed(123) num_images = 1 guess_mode = False # 可以一次性load 3 个 Controlnets,达到multi Controlnet的效果,这里的参数的长度需要对其 # Controlnet 所输入的img list 长度应该和 controlnet scale 与 Controlnet name 一致,而内部的list长度需要和batch size一致 # 对应的index 可以对其 controlnet_images = [[control_img]] controlnet_scale = [0.5] controlnet_names = ['canny'] # 推理生成,返回结果都是生成好的 PIL.Image start = time.perf_counter() images = pipe(prompt, height, width, steps, guidance_scale, negative_prompt, num_images, generator=generator, controlnet_images=controlnet_images, controlnet_scale=controlnet_scale, controlnet_names=controlnet_names, guess_mode=guess_mode ) print("cur cost: ",time.perf_counter() - start) # 存储生成的图片 for i, image in enumerate(images): image.save(f"./outputs/res_controlnet_txt2img_{i}.png")