work in progress

src/app/interface/generate/index.tsx

@@ -45,8 +45,10 @@ export function Generate() {
       try {
         console.log("starting transition, calling generateAnimation")
         const newAssetUrl = await generateAnimation({
-          prompt: promptDraft,
+          positivePrompt: promptDraft,
+          negativePrompt: "",
           huggingFaceLora: "KappaNeuro/studio-ghibli-style",
+          triggerWord: "Studio Ghibli Style",
           // huggingFaceLora: "veryVANYA/ps1-graphics-sdxl-v2", // 
           // huggingFaceLora: "ostris/crayon_style_lora_sdxl", // "https://huggingface.co/ostris/crayon_style_lora_sdxl/resolve/main/crayons_v1_sdxl.safetensors",
           // replicateLora: "https://replicate.com/jbilcke/sdxl-panorama",
@@ -70,7 +72,9 @@ export function Generate() {
 
           size: "768x320", // "1024x512", // "512x512" // "320x768"
 
-          steps: 40,
+          nbFrames: 8, // if duration is 1000ms then it means 8 FPS
+          duration: 1000, // in ms
+          steps: 25,
       })
         setAssetUrl(newAssetUrl)
       } catch (err) {
@@ -90,7 +94,7 @@ export function Generate() {
       )}>
       <div className={cn(
         `flex flex-col md:flex-row`,
-        `w-full md:max-w-3xl lg:max-w-4xl xl:max-w-5xl max-h-[80vh]`,
+        `w-full md:max-w-4xl lg:max-w-5xl xl:max-w-6xl max-h-[80vh]`,
         `space-y-3 md:space-y-0 md:space-x-6`,
         `transition-all duration-300 ease-in-out`,
 
@@ -111,13 +115,20 @@ export function Generate() {
                 `space-y-3 md:space-y-6`,
                 `items-center`,
               )}>
+                {assetUrl.startsWith("data:video/mp4")
+                 ? <video
+                    muted
+                    autoPlay
+                    loop
+                    src={assetUrl}
+                  /> :
                 <img
                   src={assetUrl}
                   className={cn(
                     `w-[512px] object-cover`,
                     `rounded-2xl`
                     )}
-                />
+                />}
             </div> : null}
 
             <div className={cn(

src/app/server/actions/animation.ts

@@ -16,6 +16,7 @@ export async function generateAnimation({
   size = "512x512",
   huggingFaceLora,
   replicateLora,
+  triggerWord,
   nbFrames = 8,
   duration = 1000,
   steps = 30,
@@ -26,6 +27,7 @@ export async function generateAnimation({
 
   // pimp the prompt
   positivePrompt = [
+    triggerWord,
     positivePrompt,
     "beautiful",
     "hd"

src/app/server/actions/generateWithGradioApi.ts

@@ -14,6 +14,17 @@ export async function generateVideoWithGradioAPI({
   duration = 1000,
   steps = 30,
 }: VideoOptions): Promise<string> {
+  console.log(`SEND TO ${gradioApi + (gradioApi.endsWith("/") ? "" : "/") + "api/predict"}:`, [
+    accessToken,
+    positivePrompt,
+    negativePrompt,
+    huggingFaceLora,
+    size,
+    generateSeed(),
+    steps,
+    nbFrames,
+    duration,
+  ])
   const res = await fetch(gradioApi + (gradioApi.endsWith("/") ? "" : "/") + "api/predict", {
     method: "POST",
     headers: {
@@ -21,7 +32,7 @@ export async function generateVideoWithGradioAPI({
       // Authorization: `Bearer ${token}`,
     },
     body: JSON.stringify({
-      fn_index: 0,
+      fn_index: 1, // <- important!
       data: [
         accessToken,
         positivePrompt,
@@ -41,6 +52,7 @@ export async function generateVideoWithGradioAPI({
 
   const { data } = await res.json()
 
+  // console.log("data:", data)
   // Recommendation: handle errors
   if (res.status !== 200) {
     // This will activate the closest `error.js` Error Boundary
@@ -48,5 +60,5 @@ export async function generateVideoWithGradioAPI({
   }
   // console.log("data:", data.slice(0, 50))
 
-  return data
+  return data[0]
 }

src/app/server/actions/generateWithReplicateAPI.ts

@@ -14,7 +14,9 @@ export async function generateVideoWithReplicateAPI({
   size = "512x512",
   huggingFaceLora,
   replicateLora,
-  steps = 30
+  nbFrames = 8,
+  duration = 1000,
+  steps = 30,
 }: VideoOptions): Promise<string> {
   if (!replicateToken) {
     throw new Error(`you need to configure your AUTH_REPLICATE_API_TOKEN in order to use the REPLICATE rendering engine`)

src/types.ts

@@ -295,6 +295,8 @@ export type VideoOptions = {
 
   replicateLora?: string
 
+  triggerWord?: string
+  
   nbFrames?: number // FPS (eg. 8)
   duration?: number // in milliseconds