Update app.py

app.py

@@ -1,28 +1,27 @@
-import subprocess
-command = 'pip install git+https://github.com/snekkenull/diffusers.git'
-subprocess.run(command, shell=True)
-
 import os
 import gradio as gr
 import torch
 import numpy as np
-import random
-from diffusers import StableDiffusion3Pipeline, AutoencoderKL, SD3Transformer2DModel, StableDiffusion3Img2ImgPipeline, FlowMatchEulerDiscreteScheduler
 import spaces
-from diffusers.utils import load_image
+import random
 from PIL import Image
-import requests
-import transformers
-from transformers import AutoTokenizer, T5EncoderModel
-from translatepy import Translator
 
+from glob import glob
+from pathlib import Path
+from typing import Optional
+
+from diffusers import StableVideoDiffusionPipeline
+from diffusers.utils import load_image, export_to_video
 
-os.environ["HF_HUB_ENABLE_HF_TRANSFER"] = "1"
-translator = Translator()
-HF_TOKEN = os.environ.get("HF_TOKEN", None)
+import uuid
+# from huggingface_hub import hf_hub_download
+
+
+# os.environ["HF_HUB_ENABLE_HF_TRANSFER"] = "1"
+# HF_TOKEN = os.environ.get("HF_TOKEN", None)
 # Constants
-model = "stabilityai/stable-diffusion-3-medium"
-repo= "stabilityai/stable-diffusion-3-medium-diffusers"
+model = "ECNU-CILab/ExVideo-SVD-128f-v1"
+
 MAX_SEED = np.iinfo(np.int32).max
 
 CSS = """
@@ -39,200 +38,130 @@ JS = """function () {
 }"""
 
 
-vae = AutoencoderKL.from_pretrained(
-    repo,
-    subfolder="vae",
-    torch_dtype=torch.float16,
- )
-
-transformer = SD3Transformer2DModel.from_pretrained(
-    repo, 
-    subfolder="transformer",
-    torch_dtype=torch.float16,
-)
-
-
-# text_encoder_3 = T5EncoderModel.from_pretrained(
-#     repo, 
-#     subfolder="text_encoder_3", 
-# )
-
-# tokenizer_3 = AutoTokenizer.from_pretrained(
-#     repo, 
-#     subfolder="tokenizer_3",
-#     torch_dtype=torch.float16,
-# )
-
-
 # Ensure model and scheduler are initialized in GPU-enabled function
 if torch.cuda.is_available():
-    pipe = StableDiffusion3Pipeline.from_pretrained(
-        repo,
-        vae=vae,
-        transformer=transformer,
-        torch_dtype=torch.float16).to("cuda")
-    pipe2 = StableDiffusion3Img2ImgPipeline.from_pretrained(
-        repo,
-        vae=vae,
-        transformer=transformer,
-        torch_dtype=torch.float16).to("cuda")
-
-
-
-pipe.scheduler = FlowMatchEulerDiscreteScheduler.from_config(pipe.scheduler.config)
-pipe2.scheduler = FlowMatchEulerDiscreteScheduler.from_config(pipe2.scheduler.config)
-
-print(pipe.tokenizer_max_length)
-
-# Function 
-@spaces.GPU()
-def generate_image(
-    prompt,
-    negative="low quality",
-    width=1024,
-    height=1024,
-    scales=5,
-    steps=30,
-    strength=0.7,
-    seed: int =-1,
-    nums=1,
+    pipe = StableVideoDiffusionPipeline.from_pretrained(
+    model, 
+    torch_dtype=torch.float16, 
+    variant="fp16").to("cuda")
+
+# function source codes modified from multimodalart/stable-video-diffusion
+@spaces.GPU(duration=120)
+def generate(
+    image: Image,
+    seed: Optional[int] = -1,
+    motion_bucket_id: int = 127,
+    fps_id: int = 6,
+    version: str = "svd_xt",
+    cond_aug: float = 0.02,
+    decoding_t: int = 1,
+    device: str = "cuda",
+    output_folder: str = "outputs",
     progress=gr.Progress(track_tqdm=True)):
 
     if seed == -1:
         seed = random.randint(0, MAX_SEED)
-    seed = int(seed)
-    print(f'prompt:{prompt}')
+        
+    if image.mode == "RGBA":
+        image = image.convert("RGB")
+        
+    generator = torch.manual_seed(seed)
     
-    text = str(translator.translate(prompt['text'], 'English'))
+    os.makedirs(output_folder, exist_ok=True)
+    base_count = len(glob(os.path.join(output_folder, "*.mp4")))
+    video_path = os.path.join(output_folder, f"{base_count:06d}.mp4")
 
+    frames = pipe(image, decode_chunk_size=decoding_t, generator=generator, motion_bucket_id=motion_bucket_id, noise_aug_strength=0.1, num_frames=25).frames[0]
+    export_to_video(frames, video_path, fps=fps_id)
+    torch.manual_seed(seed)
     
-    if prompt['files']:
-        #images = Image.open(prompt['files'][-1]).convert('RGB')
-        init_image = load_image(prompt['files'][-1]).resize((height, width))
+    return video_path, seed
+
+def resize_image(image, output_size=(1024, 576)):
+    # Calculate aspect ratios
+    target_aspect = output_size[0] / output_size[1]  # Aspect ratio of the desired size
+    image_aspect = image.width / image.height  # Aspect ratio of the original image
+
+    # Resize then crop if the original image is larger
+    if image_aspect > target_aspect:
+        # Resize the image to match the target height, maintaining aspect ratio
+        new_height = output_size[1]
+        new_width = int(new_height * image_aspect)
+        resized_image = image.resize((new_width, new_height), Image.LANCZOS)
+        # Calculate coordinates for cropping
+        left = (new_width - output_size[0]) / 2
+        top = 0
+        right = (new_width + output_size[0]) / 2
+        bottom = output_size[1]
     else:
-        init_image = None
-    generator = torch.Generator().manual_seed(seed)
-    
+        # Resize the image to match the target width, maintaining aspect ratio
+        new_width = output_size[0]
+        new_height = int(new_width / image_aspect)
+        resized_image = image.resize((new_width, new_height), Image.LANCZOS)
+        # Calculate coordinates for cropping
+        left = 0
+        top = (new_height - output_size[1]) / 2
+        right = output_size[0]
+        bottom = (new_height + output_size[1]) / 2
 
-    if init_image:
-        image = pipe2(
-            prompt=text,
-            image=init_image,
-            negative_prompt=negative, 
-            guidance_scale=scales,
-            num_inference_steps=steps,
-            strength=strength,
-            generator = generator,
-            num_images_per_prompt = nums,
-        ).images
-    else:
-        image = pipe(
-            prompt=text,
-            negative_prompt=negative, 
-            width=width,
-            height=height,
-            guidance_scale=scales,
-            num_inference_steps=steps,
-            generator = generator,
-            num_images_per_prompt = nums,
-        ).images
-
-    print(image)
-    print(seed)
-    return image, seed
+    # Crop the image
+    cropped_image = resized_image.crop((left, top, right, bottom))
+    return cropped_image
 
 
 examples = [
-        [{"text": "a female character with long, flowing hair that appears to be made of ethereal, swirling patterns resembling the Northern Lights or Aurora Borealis. The background is dominated by deep blues and purples, creating a mysterious and dramatic atmosphere. The character's face is serene, with pale skin and striking features. She wears a dark-colored outfit with subtle patterns. The overall style of the artwork is reminiscent of fantasy or supernatural genres", "files": []}],
-        [{"text": "Digital art, portrait of an anthropomorphic roaring Tiger warrior with full armor, close up in the middle of a battle, behind him there is a banner with the text \"Open Source\".", "files": []}],
-        [{"text": "photo of a dog and a cat both standing on a red box, with a blue ball in the middle with a parrot standing on top of the ball. The box has the text \"SD3\"", "files": []}],
-		[{"text": "selfie photo of a wizard with long beard and purple robes, he is apparently in the middle of Tokyo. Probably taken from a phone.", "files": []}],
-		[{"text": "A vibrant street wall covered in colorful graffiti, the centerpiece spells \"SD3 MEDIUM\", in a storm of colors", "files": []}],
-		[{"text":  "photo of a young woman with long, wavy brown hair tied in a bun and glasses. She has a fair complexion and is wearing subtle makeup, emphasizing her eyes and lips. She is dressed in a black top. The background appears to be an urban setting with a building facade, and the sunlight casts a warm glow on her face.", "files": []}],
-		[{"text": "anime art of a steampunk inventor in their workshop, surrounded by gears, gadgets, and steam. He is holding a blue potion and a red potion, one in each hand", "files": []}],
-		[{"text": "photo of picturesque scene of a road surrounded by lush green trees and shrubs. The road is wide and smooth, leading into the distance. On the right side of the road, there's a blue sports car parked with the license plate spelling \"SD32B\". The sky above is partly cloudy, suggesting a pleasant day. The trees have a mix of green and brown foliage. There are no people visible in the image. The overall composition is balanced, with the car serving as a focal point.", "files": []}],
-		[{"text": "photo of young man in a black suit, white shirt, and black tie. He has a neatly styled haircut and is looking directly at the camera with a neutral expression. The background consists of a textured wall with horizontal lines. The photograph is in black and white, emphasizing contrasts and shadows. The man appears to be in his late twenties or early thirties, with fair skin and short, dark hair.", "files": []}],
-		[{"text": "photo of a woman on the beach, shot from above. She is facing the sea, while wearing a white dress. She has long blonde hair", "files": []}],
-]
+        "./train.jpg",
+        "./girl.webp",
+        "./robo.jpg",
+    ]
 
 
 
 # Gradio Interface
 
 with gr.Blocks(css=CSS, js=JS, theme="soft") as demo:
-    gr.HTML("<h1><center>SD3M🐉T5</center></h1>")
-    gr.HTML("<p><center><a href='https://huggingface.co/stabilityai/stable-diffusion-3-medium'>sd3m</a> text/image-to-image generation<br><b>Update</b>: fix diffuser to support 512 token</center></p>")
+    gr.HTML("<h1><center>Exvideo📽️</center></h1>")
+    gr.HTML("<p><center><a href='https://huggingface.co/ECNU-CILab/ExVideo-SVD-128f-v1'>ExVideo</a> image-to-video generation<br><b>Update</b>: first version</center></p>")
     with gr.Row():
-        with gr.Column(scale=4):
-            img = gr.Gallery(label='SD3M Generated Image', columns = 1, preview=True, height=600)
-            prompt = gr.MultimodalTextbox(label='Enter Your Prompt (Multi-Languages)', interactive=True, placeholder="Enter prompt, add one image.", file_types=['image'])
+        image = gr.Image(label='Upload Image', height=600, scale=2)
+        video = gr.Video(label="Generated Video", height=600, scale=2)
         with gr.Accordion("Advanced Options", open=True):
             with gr.Column(scale=1):
-                negative = gr.Textbox(label="Negative prompt", value="low quality, ugly, blurry, poor face, bad anatomy")
-                width = gr.Slider(
-                    label="Width",
-                    minimum=512,
-                    maximum=1280,
-                    step=8,
-                    value=1024,
-                )
-                height = gr.Slider(
-                    label="Height",
-                    minimum=512,
-                    maximum=1280,
-                    step=8,
-                    value=1024,
-                )
-                scales = gr.Slider(
-                    label="Guidance",
-                    minimum=3.5,
-                    maximum=7,
-                    step=0.1,
-                    value=5,
-                )
-                steps = gr.Slider(
-                    label="Steps",
-                    minimum=1,
-                    maximum=50,
-                    step=1,
-                    value=30,
-                )
-                strength = gr.Slider(
-                    label="Strength",
-                    minimum=0.0,
-                    maximum=1.0,
-                    step=0.1,
-                    value=0.7,
-                )
                 seed = gr.Slider(
                     label="Seed (-1 Random)",
                     minimum=-1,
                     maximum=MAX_SEED,
                     step=1,
                     value=-1,
-                    scale=2,
                 )
-                nums = gr.Slider(
-                    label="Image Numbers",
-                    minimum=1,
-                    maximum=4,
-                    step=1,
-                    value=1,
-                    scale=1,
-                )  
+                motion_bucket_id = gr.Slider(
+                    label="Motion bucket id", 
+                    info="Controls how much motion to add/remove from the image", 
+                    value=127, 
+                    minimum=1, 
+                    maximum=255
+                )
+                fps_id = gr.Slider(
+                    label="Frames per second", 
+                    info="The length of your video in seconds will be 25/fps", 
+                    value=6, 
+                    minimum=5, 
+                    maximum=30
+                )
+
+    submit_btn = gr.Button("Generate")
+    clear_btn = gr.ClearButton("Clear")
     gr.Examples(
         examples=examples,
-        inputs=prompt,
-        outputs=[img, seed],
-        fn=generate_image,
+        inputs=image,
+        outputs=[video, seed],
+        fn=generate,
         cache_examples="lazy",
         examples_per_page=4,
     )
 
-    prompt.submit(fn=generate_image,
-                 inputs=[prompt, negative, width, height, scales, steps, strength, seed, nums],
-                 outputs=[img, seed],
-                 )
+    image.upload(fn=resize_image, inputs=image, outputs=image, queue=False)
 
+    generate_btn.click(fn=generate, inputs=[image, seed, motion_bucket_id, fps_id], outputs=[video, seed], api_name="video")
     
 demo.queue().launch()