app.py

import os
# os.environ['CUDA_VISIBLE_DEVICES'] = "6"

# uncomment the next line to use huggingface model in China
# os.environ['HF_ENDPOINT'] = 'https://hf-mirror.com'

import cv2
import io
import gc
import yaml
import argparse
import torch
import torchvision
import diffusers
from diffusers import StableDiffusionPipeline, AutoencoderKL, DDPMScheduler, ControlNetModel
import gradio as gr
from enum import Enum
import imageio.v2 as imageio

from src.utils import *
from src.keyframe_selection import get_keyframe_ind
from src.diffusion_hacked import apply_FRESCO_attn, apply_FRESCO_opt, disable_FRESCO_opt
from src.diffusion_hacked import get_flow_and_interframe_paras, get_intraframe_paras
from src.pipe_FRESCO import inference
from src.free_lunch_utils import apply_freeu

import sys
sys.path.append("./src/ebsynth/deps/gmflow/")
sys.path.append("./src/EGNet/")
sys.path.append("./src/ControlNet/")

from gmflow.gmflow import GMFlow
from model import build_model
from annotator.hed import HEDdetector
from annotator.canny import CannyDetector
from annotator.midas import MidasDetector

import huggingface_hub
import shutil
import os
import spaces

huggingface_hub.hf_hub_download('SingleZombie/FRESCO',
                                'boxer-punching-towards-camera.mp4',
                                local_dir='data')
huggingface_hub.hf_hub_download('SingleZombie/FRESCO',
                                'car-turn.mp4',
                                local_dir='data')
huggingface_hub.hf_hub_download('SingleZombie/FRESCO',
                                'dog.mp4',
                                local_dir='data')
huggingface_hub.hf_hub_download('SingleZombie/FRESCO',
                                'music.mp4',
                                local_dir='data')

huggingface_hub.hf_hub_download('PKUWilliamYang/Rerender',
                                'models/gmflow_sintel-0c07dcb3.pth',
                                local_dir='./')

huggingface_hub.hf_hub_download('PKUWilliamYang/Rerender',
                                'models/epoch_resnet.pth',
                                local_dir='./')

huggingface_hub.hf_hub_download('PKUWilliamYang/Rerender',
                                'models/ebsynth',
                                local_dir='./')
                                
os.symlink('models', 'model')
os.symlink(os.path.abspath(os.readlink('models/ebsynth')), 'src/ebsynth/deps/ebsynth/bin/ebsynth')

device = 'cuda' if torch.cuda.is_available() else 'cpu'

def get_models(config):
    # optical flow
    flow_model = GMFlow(feature_channels=128,
                        num_scales=1,
                        upsample_factor=8,
                        num_head=1,
                        attention_type='swin',
                        ffn_dim_expansion=4,
                        num_transformer_layers=6,
                        ).to(device)

    checkpoint = torch.load(
        config['gmflow_path'], map_location=lambda storage, loc: storage)
    weights = checkpoint['model'] if 'model' in checkpoint else checkpoint
    flow_model.load_state_dict(weights, strict=False)
    flow_model.eval()

    # saliency detection
    sod_model = build_model('resnet')
    sod_model.load_state_dict(torch.load(config['sod_path']))
    sod_model.to(device).eval()

    # controlnet
    if config['controlnet_type'] not in ['hed', 'depth', 'canny']:
        config['controlnet_type'] = 'hed'
    controlnet = ControlNetModel.from_pretrained("lllyasviel/sd-controlnet-"+config['controlnet_type'],
                                                 torch_dtype=torch.float16)
    controlnet.to(device)
    if config['controlnet_type'] == 'depth':
        detector = MidasDetector()
    elif config['controlnet_type'] == 'canny':
        detector = CannyDetector()
    else:
        detector = HEDdetector()

    # diffusion model
    vae = AutoencoderKL.from_pretrained(
        "stabilityai/sd-vae-ft-mse", torch_dtype=torch.float16)
    pipe = StableDiffusionPipeline.from_pretrained(
        config['sd_path'], vae=vae, torch_dtype=torch.float16)
    pipe.scheduler = DDPMScheduler.from_config(pipe.scheduler.config)
    pipe.to(device)
    pipe.scheduler.set_timesteps(
        config['num_inference_steps'], device=pipe._execution_device)

    frescoProc = apply_FRESCO_attn(pipe)
    frescoProc.controller.disable_controller()
    apply_FRESCO_opt(pipe)

    for param in flow_model.parameters():
        param.requires_grad = False
    for param in sod_model.parameters():
        param.requires_grad = False
    for param in controlnet.parameters():
        param.requires_grad = False
    for param in pipe.unet.parameters():
        param.requires_grad = False

    return pipe, frescoProc, controlnet, detector, flow_model, sod_model


def apply_control(x, detector, control_type):
    if control_type == 'depth':
        detected_map, _ = detector(x)
    elif control_type == 'canny':
        detected_map = detector(x, 50, 100)
    else:
        detected_map = detector(x)
    return detected_map


class ProcessingState(Enum):
    NULL = 0
    KEY_IMGS = 1


def cfg_to_input(filename):

    with open(filename, "r") as f:
        cfg = yaml.safe_load(f)
    use_constraints = [
        'spatial-guided attention',
        'cross-frame attention',
        'temporal-guided attention',
        'spatial-guided optimization',
        'temporal-guided optimization',
    ]

    if 'realistic' in cfg['sd_path'].lower():
        a_prompt = 'RAW photo, subject, (high detailed skin:1.2), 8k uhd, dslr, soft lighting, high quality, film grain, Fujifilm XT3'
        n_prompt = '(deformed iris, deformed pupils, semi-realistic, cgi, 3d, render, sketch, cartoon, drawing, anime, mutated hands and fingers:1.4), (deformed, distorted, disfigured:1.3), poorly drawn, bad anatomy, wrong anatomy, extra limb, missing limb, floating limbs, disconnected limbs, mutation, mutated, ugly, disgusting, amputation'
    else:
        a_prompt = 'best quality, extremely detailed'
        n_prompt = 'longbody, lowres, bad anatomy, bad hands, missing fingers, extra digit, fewer digits, cropped, worst quality, low quality'

    frame_count = get_frame_count(cfg['file_path'])
    args = [
        cfg['file_path'], cfg['prompt'], cfg['sd_path'], cfg['seed'], 512, cfg['cond_scale'],
        1.0, cfg['controlnet_type'], 50, 100,
        cfg['num_inference_steps'], 7.5, a_prompt, n_prompt,
        frame_count, cfg['batch_size'], cfg['mininterv'], cfg['maxinterv'],
        use_constraints, True,
        1, 1, 1, 1
    ]
    return args


class GlobalState:
    def __init__(self):
        config_path = 'config/config_dog.yaml'
        with open(config_path, "r") as f:
            config = yaml.safe_load(f)

        self.sd_model = config['sd_path']
        self.control_type = config['controlnet_type']
        self.processing_state = ProcessingState.NULL
        pipe, frescoProc, controlnet, detector, flow_model, sod_model = get_models(
            config)
        self.pipe = pipe
        self.frescoProc = frescoProc
        self.controlnet = controlnet
        self.detector = detector
        self.flow_model = flow_model
        self.sod_model = sod_model
        self.keys = []

    def update_controlnet_model(self, control_type):
        if self.control_type == control_type:
            return
        self.control_type = control_type
        self.controlnet = ControlNetModel.from_pretrained("lllyasviel/sd-controlnet-"+control_type,
                                                          torch_dtype=torch.float16)
        self.controlnet.to(device)
        if control_type == 'depth':
            self.detector = MidasDetector()
        elif control_type == 'canny':
            self.detector = CannyDetector()
        else:
            self.detector = HEDdetector()
        if device == 'cuda':
            torch.cuda.empty_cache()
        for param in self.controlnet.parameters():
            param.requires_grad = False

    def update_sd_model(self, sd_model):
        if self.sd_model == sd_model:
            return
        self.sd_model = sd_model
        vae = AutoencoderKL.from_pretrained(
            "stabilityai/sd-vae-ft-mse", torch_dtype=torch.float16)
        self.pipe = StableDiffusionPipeline.from_pretrained(
            sd_model, vae=vae, torch_dtype=torch.float16)
        self.pipe.scheduler = DDPMScheduler.from_config(
            self.pipe.scheduler.config)
        self.pipe.to(device)
        self.frescoProc = apply_FRESCO_attn(self.pipe)
        self.frescoProc.controller.disable_controller()
        if device == 'cuda':
            torch.cuda.empty_cache()
        for param in self.pipe.unet.parameters():
            param.requires_grad = False




@torch.no_grad()
@spaces.GPU(duration=240)
def process1(input_path, prompt, sd_model, seed, image_resolution, control_strength,
             x0_strength, control_type, low_threshold, high_threshold,
             ddpm_steps, scale, a_prompt, n_prompt,
             frame_count, batch_size, mininterv, maxinterv,
             use_constraints, bg_smooth, use_poisson, max_process,
             b1, b2, s1, s2):
    global_state = GlobalState()
    global_state.update_controlnet_model(control_type)
    global_state.update_sd_model(sd_model)
    apply_freeu(global_state.pipe, b1=b1, b2=b2, s1=s1, s2=s2)

    filename = os.path.splitext(os.path.basename(input_path))[0]
    save_path = os.path.join('output', filename)
    device = global_state.pipe._execution_device
    guidance_scale = scale
    do_classifier_free_guidance = True
    global_state.pipe.scheduler.set_timesteps(ddpm_steps, device=device)
    timesteps = global_state.pipe.scheduler.timesteps
    cond_scale = [control_strength] * ddpm_steps
    dilate = Dilate(device=device)

    base_prompt = prompt
    video_cap = cv2.VideoCapture(input_path)
    frame_num = min(frame_count, int(video_cap.get(cv2.CAP_PROP_FRAME_COUNT)))
    fps = int(video_cap.get(cv2.CAP_PROP_FPS))

    if mininterv > maxinterv:
        mininterv = maxinterv

    keys = get_keyframe_ind(input_path, frame_num, mininterv, maxinterv)
    if len(keys) < 3:
        raise gr.Error('Too few (%d) keyframes detected!' % (len(keys)))
    global_state.keys = keys
    fps = max(int(fps * len(keys) / frame_num), 1)
    os.makedirs(save_path, exist_ok=True)
    os.makedirs(os.path.join(save_path, 'keys'), exist_ok=True)
    os.makedirs(os.path.join(save_path, 'video'), exist_ok=True)

    sublists = [keys[i:i+batch_size-2]
                for i in range(2, len(keys), batch_size-2)]
    sublists[0].insert(0, keys[0])
    sublists[0].insert(1, keys[1])
    if len(sublists) > 1 and len(sublists[-1]) < 3:
        add_num = 3 - len(sublists[-1])
        sublists[-1] = sublists[-2][-add_num:] + sublists[-1]
        sublists[-2] = sublists[-2][:-add_num]

    batch_ind = 0
    propagation_mode = batch_ind > 0
    imgs = []
    record_latents = []
    video_cap = cv2.VideoCapture(input_path)

    for i in range(frame_num):
        success, frame = video_cap.read()
        frame = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)
        img = resize_image(frame, image_resolution)
        H, W, C = img.shape
        Image.fromarray(img).save(os.path.join(
            save_path, 'video/%04d.png' % (i)))
        if i not in sublists[batch_ind]:
            continue

        imgs += [img]
        if i != sublists[batch_ind][-1]:
            continue

        # prepare input
        batch_size = len(imgs)
        n_prompts = [n_prompt] * len(imgs)
        prompts = [base_prompt + a_prompt] * len(sublists[batch_ind])
        if propagation_mode:
            prompts = ref_prompt + prompts

        prompt_embeds = global_state.pipe._encode_prompt(
            prompts,
            device,
            1,
            do_classifier_free_guidance,
            n_prompts,
        )

        imgs_torch = torch.cat([numpy2tensor(img) for img in imgs], dim=0)

        edges = torch.cat([numpy2tensor(apply_control(img,
                                                      global_state.detector, control_type)[:, :, None]) for img in imgs], dim=0)
        edges = edges.repeat(1, 3, 1, 1).to(device) * 0.5 + 0.5
        edges = torch.cat([edges.to(global_state.pipe.unet.dtype)] * 2)

        if bg_smooth:
            saliency = get_saliency(imgs, global_state.sod_model, dilate)
        else:
            saliency = None

        # prepare parameters for inter-frame and intra-frame consistency
        flows, occs, attn_mask, interattn_paras = get_flow_and_interframe_paras(
            global_state.flow_model, imgs)
        correlation_matrix = get_intraframe_paras(global_state.pipe, imgs_torch, global_state.frescoProc,
                                                  prompt_embeds, seed=seed)

        global_state.frescoProc.controller.disable_controller()
        if 'spatial-guided attention' in use_constraints:
            global_state.frescoProc.controller.enable_intraattn()
        if 'temporal-guided attention' in use_constraints:
            global_state.frescoProc.controller.enable_interattn(
                interattn_paras)
        if 'cross-frame attention' in use_constraints:
            global_state.frescoProc.controller.enable_cfattn(attn_mask)

        global_state.frescoProc.controller.enable_controller(
            interattn_paras=interattn_paras, attn_mask=attn_mask)
        optimize_temporal = True
        if 'temporal-guided optimization' not in use_constraints:
            correlation_matrix = []
        if 'spatial-guided optimization' not in use_constraints:
            optimize_temporal = False
        apply_FRESCO_opt(global_state.pipe, steps=timesteps[:int(ddpm_steps*0.75)],
                         flows=flows, occs=occs, correlation_matrix=correlation_matrix,
                         saliency=saliency, optimize_temporal=optimize_temporal)

        gc.collect()
        if device == 'cuda':
            torch.cuda.empty_cache()

        # run!
        latents = inference(global_state.pipe, global_state.controlnet, global_state.frescoProc,
                            imgs_torch, prompt_embeds, edges, timesteps,
                            cond_scale, ddpm_steps, int(
                                ddpm_steps*(1-x0_strength)),
                            True, seed, guidance_scale, True,
                            record_latents, propagation_mode,
                            flows=flows, occs=occs, saliency=saliency, repeat_noise=True)

        with torch.no_grad():
            image = global_state.pipe.vae.decode(
                latents / global_state.pipe.vae.config.scaling_factor, return_dict=False)[0]
            image = torch.clamp(image, -1, 1)
            save_imgs = tensor2numpy(image)
            bias = 2 if propagation_mode else 0
            for ind, num in enumerate(sublists[batch_ind]):
                Image.fromarray(
                    save_imgs[ind+bias]).save(os.path.join(save_path, 'keys/%04d.png' % (num)))

        batch_ind += 1
        # current batch uses the last frame of the previous batch as ref
        ref_prompt = [prompts[0], prompts[-1]]
        imgs = [imgs[0], imgs[-1]]
        propagation_mode = batch_ind > 0
        if batch_ind == len(sublists):
            gc.collect()
            if device == 'cuda':
                torch.cuda.empty_cache()
            break  

    writer = imageio.get_writer(os.path.join(save_path, 'key.mp4'), fps=fps)
    file_list = sorted(os.listdir(os.path.join(save_path, 'keys')))
    for file_name in file_list:
        if not (file_name.endswith('jpg') or file_name.endswith('png')):
            continue
        fn = os.path.join(os.path.join(save_path, 'keys'), file_name)
        curImg = imageio.imread(fn)
        writer.append_data(curImg)
    writer.close()

    return os.path.join(save_path, 'key.mp4')


config_dir = 'config'
filenames = os.listdir(config_dir)
config_list = []
for filename in filenames:
    if filename.endswith('yaml'):
        config_list.append(f'{config_dir}/{filename}')

block = gr.Blocks().queue()
with block:
    with gr.Row():
        gr.Markdown('## FRESCO Video-to-Video Translation')
    with gr.Row():
        with gr.Column():
            input_path = gr.Video(label='Input Video',
                                  sources='upload',
                                  format='mp4',
                                  visible=True)
            prompt = gr.Textbox(label='Prompt')
            sd_model = gr.Dropdown(['SG161222/Realistic_Vision_V2.0',
                                    'runwayml/stable-diffusion-v1-5',
                                    'stablediffusionapi/rev-animated',
                                    'stablediffusionapi/flat-2d-animerge'],
                                   label='Base model',
                                   value='SG161222/Realistic_Vision_V2.0')
            seed = gr.Slider(label='Seed',
                             minimum=0,
                             maximum=2147483647,
                             step=1,
                             value=0,
                             randomize=True)
            run_button1 = gr.Button(value='Run Key Frames')
            with gr.Accordion('Advanced options for single frame processing',
                              open=False):
                image_resolution = gr.Slider(label='Frame resolution',
                                             minimum=256,
                                             maximum=512,
                                             value=512,
                                             step=64)
                control_strength = gr.Slider(label='ControlNet strength',
                                             minimum=0.0,
                                             maximum=2.0,
                                             value=1.0,
                                             step=0.01)
                x0_strength = gr.Slider(
                    label='Denoising strength',
                    minimum=0.00,
                    maximum=1.05,
                    value=0.75,
                    step=0.05,
                    info=('0: fully recover the input.'
                          '1.05: fully redraw the input.'))
                with gr.Row():
                    control_type = gr.Dropdown(['hed', 'canny', 'depth'],
                                               label='Control type',
                                               value='hed')
                    low_threshold = gr.Slider(label='Canny low threshold',
                                              minimum=1,
                                              maximum=255,
                                              value=50,
                                              step=1)
                    high_threshold = gr.Slider(label='Canny high threshold',
                                               minimum=1,
                                               maximum=255,
                                               value=100,
                                               step=1)
                ddpm_steps = gr.Slider(label='Steps',
                                       minimum=20,
                                       maximum=100,
                                       value=20,
                                       step=20)
                scale = gr.Slider(label='CFG scale',
                                  minimum=1.1,
                                  maximum=30.0,
                                  value=7.5,
                                  step=0.1)
                a_prompt = gr.Textbox(label='Added prompt',
                                      value='best quality, extremely detailed')
                n_prompt = gr.Textbox(
                    label='Negative prompt',
                    value=('longbody, lowres, bad anatomy, bad hands, '
                           'missing fingers, extra digit, fewer digits, '
                           'cropped, worst quality, low quality'))
                with gr.Row():
                    b1 = gr.Slider(label='FreeU first-stage backbone factor',
                                   minimum=1,
                                   maximum=1.6,
                                   value=1,
                                   step=0.01,
                                   info='FreeU to enhance texture and color')
                    b2 = gr.Slider(label='FreeU second-stage backbone factor',
                                   minimum=1,
                                   maximum=1.6,
                                   value=1,
                                   step=0.01)
                with gr.Row():
                    s1 = gr.Slider(label='FreeU first-stage skip factor',
                                   minimum=0,
                                   maximum=1,
                                   value=1,
                                   step=0.01)
                    s2 = gr.Slider(label='FreeU second-stage skip factor',
                                   minimum=0,
                                   maximum=1,
                                   value=1,
                                   step=0.01)
            with gr.Accordion('Advanced options for FRESCO constraints',
                              open=False):
                frame_count = gr.Slider(
                    label='Number of frames',
                    minimum=8,
                    maximum=300,
                    value=100,
                    step=1)
                batch_size = gr.Slider(
                    label='Number of frames in a batch',
                    minimum=3,
                    maximum=8,
                    value=8,
                    step=1)
                mininterv = gr.Slider(label='Min keyframe interval',
                                      minimum=1,
                                      maximum=20,
                                      value=5,
                                      step=1)
                maxinterv = gr.Slider(label='Max keyframe interval',
                                      minimum=1,
                                      maximum=50,
                                      value=20,
                                      step=1)
                use_constraints = gr.CheckboxGroup(
                    [
                        'spatial-guided attention',
                        'cross-frame attention',
                        'temporal-guided attention',
                        'spatial-guided optimization',
                        'temporal-guided optimization',
                    ],
                    label='Select the FRESCO contraints to be used',
                    value=[
                        'spatial-guided attention',
                        'cross-frame attention',
                        'temporal-guided attention',
                        'spatial-guided optimization',
                        'temporal-guided optimization',
                    ]),
                bg_smooth = gr.Checkbox(
                    label='Background smoothing',
                    value=True,
                    info='Select to smooth background')

            with gr.Accordion('Example configs', open=True):

                example_list = [cfg_to_input(x) for x in config_list]

                ips = [
                    input_path, prompt, sd_model, seed, image_resolution, control_strength,
                    x0_strength, control_type, low_threshold, high_threshold,
                    ddpm_steps, scale, a_prompt, n_prompt,
                    frame_count, batch_size, mininterv, maxinterv,
                    use_constraints[0], bg_smooth,
                    b1, b2, s1, s2
                ]

                gr.Examples(
                    examples=example_list,
                    inputs=[*ips],
                )

        with gr.Column():
            result_keyframe = gr.Video(label='Output key frame video',
                                       format='mp4',
                                       interactive=False)

    def input_changed(path):
        if path is None:
            return (gr.Slider.update(), gr.Slider.update(), gr.Slider.update())
        frame_count = get_frame_count(path)
        if frame_count == 0:
            return (gr.Slider.update(), gr.Slider.update(), gr.Slider.update())
        if frame_count <= 8:
            raise gr.Error('The input video is too short!'
                           'Please input another video.')
        min_interv_l = 1
        max_interv_l = 1
        min_interv_c = min(5, frame_count)
        max_interv_c = min(20, frame_count)
        min_interv_r = frame_count
        max_interv_r = frame_count
        return (gr.Slider.change(minimum=min_interv_l,
                                 value=min_interv_c,
                                 maximum=min_interv_r),
                gr.Slider.change(minimum=max_interv_l,
                                 value=max_interv_c,
                                 maximum=max_interv_r),
                gr.Slider.change(minimum=8,
                                 value=frame_count,
                                 maximum=frame_count),
                )

    input_path.change(input_changed, input_path, [
                      mininterv, maxinterv, frame_count])
    input_path.upload(input_changed, input_path, [
                      mininterv, maxinterv, frame_count])

    run_button1.click(fn=process1, inputs=ips, outputs=[result_keyframe])

block.launch()