from diffusers.schedulers import UniPCMultistepScheduler from diffusers import AutoencoderKL from diffusion_module.unet import UNetModel import torch from diffusion_module.utils.LSDMPipeline_expandDataset import SDMLDMPipeline from accelerate import Accelerator from evolution import random_walk import cv2 import numpy as np def mask2onehot(data, num_classes): # move to GPU and change data types data = data.to(dtype=torch.int64) # create one-hot label map label_map = data bs, _, h, w = label_map.size() input_label = torch.FloatTensor(bs, num_classes, h, w).zero_().to(data.device) input_semantics = input_label.scatter_(1, label_map, 1.0) return input_semantics def generate(img, pretrain_weight,seed=None): noise_scheduler = UniPCMultistepScheduler() vae = AutoencoderKL.from_pretrained("runwayml/stable-diffusion-v1-5", subfolder="vae") latent_size = (64, 64) unet = UNetModel( image_size = latent_size, in_channels=vae.config.latent_channels, model_channels=256, out_channels=vae.config.latent_channels, num_res_blocks=2, attention_resolutions=(2, 4, 8), dropout=0, channel_mult=(1, 2, 3, 4), num_heads=8, num_head_channels=-1, num_heads_upsample=-1, use_scale_shift_norm=True, resblock_updown=True, use_new_attention_order=False, num_classes=151, mask_emb="resize", use_checkpoint=True, SPADE_type="spade", ) unet = unet.from_pretrained(pretrain_weight) device = 'cpu' if device != 'cpu': mixed_precision = "fp16" else: mixed_precision = "no" accelerator = Accelerator( mixed_precision=mixed_precision, cpu= True if device is 'cpu' else False ) weight_dtype = torch.float32 if accelerator.mixed_precision == "fp16": weight_dtype = torch.float16 unet,vae = accelerator.prepare(unet, vae) vae.to(device=accelerator.device, dtype=weight_dtype) pipeline = SDMLDMPipeline( vae=accelerator.unwrap_model(vae), unet=accelerator.unwrap_model(unet), scheduler=noise_scheduler, torch_dtype=weight_dtype, resolution_type="crack" ) """ if accelerator.device != 'cpu': pipeline.enable_xformers_memory_efficient_attention() """ pipeline = pipeline.to(accelerator.device) pipeline.set_progress_bar_config(disable=False) if seed is None: generator = None else: generator = torch.Generator(device=accelerator.device).manual_seed(seed) resized_s = cv2.resize(img, (64, 64), interpolation=cv2.INTER_AREA) # 灰度图放大到255 _, binary_s = cv2.threshold(resized_s, 1, 255, cv2.THRESH_BINARY) # 转换为0,1 tensor_s = torch.from_numpy(binary_s / 255) # h,w -> 1,1,h,w tensor_s = tensor_s.unsqueeze(0).unsqueeze(0) onehot_skeletons=[] onehot_s = mask2onehot(tensor_s, 151) onehot_skeletons.append(onehot_s) onehot_skeletons = torch.stack(onehot_skeletons, dim=1).squeeze(0) onehot_skeletons = onehot_skeletons.to(dtype=weight_dtype,device=accelerator.device) images = pipeline(onehot_skeletons, generator=generator,batch_size = 1, num_inference_steps=20, s=1.5, num_evolution_per_mask=1).images return images