mostly merged

.gitignore

@@ -3,7 +3,8 @@ flagged/
 .ipynb_checkpoints
 temp/
 */.ipynb_checkpoints/*
-
+frame-interpolation
+frame_interpolation
 
 # Byte-compiled / optimized / DLL files
 __pycache__/

animate.py

@@ -10,6 +10,7 @@ from image_tools.sizes import resize_and_crop
 from moviepy.editor import CompositeVideoClip
 from moviepy.editor import VideoFileClip as vfc
 from PIL import Image
+from pathlib import Path
 
 
 # get key positions at which frame needs to be generated
@@ -23,9 +24,8 @@ def list_of_positions(num_contours, num_frames=100):
 def contourfinder(image1, image2, text=None, num_frames=100):
     # Create two blank pages to write into
     # I just hardcoded 1024*1024 as the size, ideally this should be np.shape(image1)
-    blank = np.zeros((2048, 2048, 3), dtype="uint8")
-    blank2 = np.zeros((2048, 2048, 3), dtype="uint8")
-
+    blank=np.zeros(np.shape(image1), dtype="uint8")
+    blank2=np.zeros(np.shape(image2), dtype="uint8") 
     # Threshold and contours for image 1 and 2
     threshold = cv2.Canny(image=image1, threshold1=100, threshold2=200)
     contours, hierarchies = cv2.findContours(
@@ -60,14 +60,12 @@ def contourfinder(image1, image2, text=None, num_frames=100):
 
         if i in positions:
             frames.append(blank)
-
             # Complile to video
             vid1.write(blank)
-
+            
     vid1.release()
-    clip1 = vfc("/home/user/app/vid1.mp4")
-    # print(f"Type clip 1{type(clip1)}")
-
+    full_dir_vid_1 = Path("vid1.mp4").resolve().as_posix()
+    clip1 = vfc(full_dir_vid_1)
     positions = list_of_positions((len(contours2)))
 
     for i in range(0, len(contours2)):
@@ -80,7 +78,8 @@ def contourfinder(image1, image2, text=None, num_frames=100):
             vid2.write(blank2)
 
     vid2.release()
-    clip3 = vfc("/home/user/app/vid2.mp4")
+    full_dir_vid_2 = Path("vid2.mp4").resolve().as_posix()
+    clip3 = vfc(full_dir_vid_2)
 
     # Next is the text vid
 
@@ -160,24 +159,27 @@ def resize(width, img):
     return img
 
 
+
+
+
 def resize_img(img1, img2):
-    img_target_size = Image.open(cv2_images[0])
+    img_target_size = Image.open(img1)
     img_to_resize = resize_and_crop(
-        cv2_images[1],
+        img2,
         (
             img_target_size.size[0],
             img_target_size.size[1],
         ),  # set width and height to match cv2_images[0]
         crop_origin="middle",
     )
-    img_to_resize.save("resized_cv2_images[1].png")
+    img_to_resize.save("resized_img2.png")
 
 
 def get_video_frames(images, times_to_interpolate=6, model_name_index=0):
     frame1 = images[0]
     frame2 = images[1]
 
-    model = models[model_name_index]
+    model = models[model_names[model_name_index]]
     cv2_images = [cv2.imread(frame1), cv2.imread(frame2)]
 
     frame1 = resize(256, frame1)
@@ -187,7 +189,7 @@ def get_video_frames(images, times_to_interpolate=6, model_name_index=0):
     frame2.save("test2.png")
 
     resize_img("test1.png", "test2.png")
-    input_frames = ["test1.png", "resized_cv2_images[1].png"]
+    input_frames = ["test1.png", "resized_img2.png"]
 
     frames = list(
         util.interpolate_recursively_from_files(
@@ -210,7 +212,7 @@ def create_mp4_with_audio(frames, cv2_images, duration, audio, output_path):
     clip3 = clip3.set_start((clip1.duration - 0.5) + (clip2.duration)).crossfadein(2)
 
     new_clip = CompositeVideoClip([clip1, clip2, clip3])
-    new_clip.set_audio(audio)
+    new_clip.audio = audio
     new_clip.set_duration(duration)
     new_clip.write_videofile(output_path, audio_codec="aac")
     return output_path

app.py

@@ -1,4 +1,5 @@
 import os
+import sys
 import shutil
 import subprocess
 import uuid
@@ -6,17 +7,9 @@ from pathlib import Path
 import gradio as gr
 from moviepy.editor import AudioFileClip
 
-from animate import create_mp4_with_audio, get_video_frames
-
-# My installs
 output_dir = Path("temp/").absolute()
 output_dir.mkdir(exist_ok=True, parents=True)
-subprocess.call(["git", "submodule", "update", "--init", "--recursive"]) # install frame_interplation
-
 
-os.chdir(
-    output_dir
-)  # change working directory to output_dir because the hf spaces model has no option to specify output directory ¯\_(ツ)_/¯
 
 
 class SpotifyApi:
@@ -78,12 +71,14 @@ def process_inputs(
     if spotify_url:
         spotify = SpotifyApi()
         audio_input.path = spotify.download_episode(spotify_url)
-    image = get_stable_diffusion_image(prompt)
-    video = add_static_image_to_audio(image, audio_input)
+    images = get_stable_diffusion_images(prompt)
+    video = animate_images(images, audio_input)
     return video
 
 
-def animate_images(image_paths: list[str], audio_input: str) -> str:
+def animate_images(image_paths: list[str], audio_input: AudioInput) -> str:
+    from animate import create_mp4_with_audio, get_video_frames # Only import after git clone and when necessary takes loooong
+
     # Generate a random folder name and change directories to there
     foldername = str(uuid.uuid4())[:8]
     vid_output_dir = Path(output_dir / foldername)
@@ -93,13 +88,13 @@ def animate_images(image_paths: list[str], audio_input: str) -> str:
         audio_input.start_time, audio_input.start_time + audio_input.run_for
     )
     video_frames, cv2_images = get_video_frames(image_paths)
-    path = Path(vid_output_dir / "output.mp4").as_posix()
+    path = Path(vid_output_dir / "output_final.mp4").as_posix()
     return create_mp4_with_audio(
         video_frames, cv2_images, audio_clip.duration, audio_clip, path
     )
 
 
-def get_stable_diffusion_image(prompt) -> str:
+def get_stable_diffusion_images(prompt) -> str:
     stable_diffusion = gr.Blocks.load(name="spaces/stabilityai/stable-diffusion")
     gallery_dir = stable_diffusion(prompt, fn_index=2)
     return [os.path.join(gallery_dir, img) for img in os.listdir(gallery_dir)][:2]
@@ -117,5 +112,12 @@ iface = gr.Interface(
     outputs="video",
 )
 
+if __name__ == '__main__':
+    subprocess.call(["git", "clone", "https://github.com/google-research/frame-interpolation", "frame_interpolation"]) # install frame_interplation I guess
+    sys.path.append("frame_interpolation")
 
-iface.launch()
+    # My installs
+    os.chdir(
+        output_dir
+    )  # change working directory to output_dir because the hf spaces model has no option to specify output directory ¯\_(ツ)_/¯
+    iface.launch(share=True)

frame_interpolation

@@ -1 +0,0 @@
-Subproject commit e9031587b441d679e841b59ccc5407778ba5f493

pyproject.toml

@@ -1,14 +0,0 @@
-[tool.poetry]
-name = "stablepod"
-version = "0.1.0"
-description = ""
-authors = ["Your Name <[email protected]>"]
-readme = "README.md"
-
-[tool.poetry.dependencies]
-python = "3.10.8"
-
-
-[build-system]
-requires = ["poetry-core"]
-build-backend = "poetry.core.masonry.api"

requirements.txt

@@ -13,7 +13,7 @@ natsort==8.1.0
 
 # Our dependencies 
 fastapi==0.74.1 # This is because of reasons
-gradio
+gradio==3.4 # This is also because of reasons
 spodcast
 moviepy
 huggingface-hub

test.py

@@ -0,0 +1,17 @@
+import os
+from app import animate_images, AudioInput
+
+def get_test_image_paths():
+    gallery_dir = "temp/images"
+    return [os.path.join(gallery_dir, img) for img in os.listdir(gallery_dir)][:2]
+    
+def get_test_audio_paths():
+    audio_path = "temp/audio.mp3"
+    audio_input = AudioInput(audio_path, 10, 10)
+    return audio_input
+
+
+image_paths = get_test_image_paths()
+audio_input = get_test_audio_paths()
+
+animate_images(image_paths, audio_input)