Modifying AiTube to support Stories Factory use cases

.env

@@ -1,4 +1,8 @@
 
+API_SECRET_JWT_KEY=""
+API_SECRET_JWT_ISSUER=""
+API_SECRET_JWT_AUDIENCE=""
+
 NEXT_PUBLIC_DOMAIN="https://aitube.at"
 
 NEXT_PUBLIC_SHOW_BETA_FEATURES="false"

package-lock.json

@@ -58,13 +58,14 @@
         "eslint": "8.45.0",
         "eslint-config-next": "13.4.10",
         "fastest-levenshtein": "^1.0.16",
-        "gsplat": "^1.2.3",
+        "gsplat": "^1.2.4",
         "hash-wasm": "^4.11.0",
+        "jose": "^5.2.4",
         "lodash.debounce": "^4.0.8",
         "lucide-react": "^0.260.0",
         "markdown-yaml-metadata-parser": "^3.0.0",
         "minisearch": "^6.3.0",
-        "next": "^14.1.4",
+        "next": "^14.2.2",
         "openai": "^4.36.0",
         "photo-sphere-viewer-lensflare-plugin": "^2.1.2",
         "pick": "^0.0.1",
@@ -1492,9 +1493,9 @@
       }
     },
     "node_modules/@mediapipe/tasks-vision": {
-      "version": "0.10.13-rc.20240419",
-      "resolved": "https://registry.npmjs.org/@mediapipe/tasks-vision/-/tasks-vision-0.10.13-rc.20240419.tgz",
-      "integrity": "sha512-IfIFVSkektnlo9FjqhDYe5LzbOJTdx9kjs65lXmerqPQSOA3akYwp5zt2Rtm1G4JrG1Isv1A15yfiSlqm5pGdQ=="
+      "version": "0.10.13-rc.20240422",
+      "resolved": "https://registry.npmjs.org/@mediapipe/tasks-vision/-/tasks-vision-0.10.13-rc.20240422.tgz",
+      "integrity": "sha512-yKUS+Qidsw0pttv8Bx/EOdGkcWLH0b1tO4D0HfM6PaBjBAFUr7l5OmRfToFh4k8/XPto6d3X8Ujvm37Da0n2nw=="
     },
     "node_modules/@next/env": {
       "version": "14.2.2",
@@ -1677,9 +1678,9 @@
       }
     },
     "node_modules/@photo-sphere-viewer/core": {
-      "version": "5.7.2",
-      "resolved": "https://registry.npmjs.org/@photo-sphere-viewer/core/-/core-5.7.2.tgz",
-      "integrity": "sha512-5RznXVRwuO+Izceae2SbwYM/H8GHtwxKlT26P4UcRFZYsYKllMAggAz9hhU729Vu+r1+il5PHvomIsmPHVTTaw==",
+      "version": "5.7.3",
+      "resolved": "https://registry.npmjs.org/@photo-sphere-viewer/core/-/core-5.7.3.tgz",
+      "integrity": "sha512-F2YYQVHwRxrFFvBXdfx0o9rBVOiHgHyMCGgtnJvo4dKVtoUzJdTjXXKVYiOG1ZCVpx1jsyhZeY5DykHnU+7NSw==",
       "dependencies": {
         "three": "^0.161.0"
       }
@@ -1690,77 +1691,77 @@
       "integrity": "sha512-LC28VFtjbOyEu5b93K0bNRLw1rQlMJ85lilKsYj6dgTu+7i17W+JCCEbvrpmNHF1F3NAUqDSWq50UD7w9H2xQw=="
     },
     "node_modules/@photo-sphere-viewer/equirectangular-video-adapter": {
-      "version": "5.7.2",
-      "resolved": "https://registry.npmjs.org/@photo-sphere-viewer/equirectangular-video-adapter/-/equirectangular-video-adapter-5.7.2.tgz",
-      "integrity": "sha512-cAaot52nPqa2p77Xp1humRvuxRIa8cqbZ/XRhA8kBToFLT1Ugh9YBcDD7pM/358JtAjicUbLpT7Ioap9iEigxQ==",
+      "version": "5.7.3",
+      "resolved": "https://registry.npmjs.org/@photo-sphere-viewer/equirectangular-video-adapter/-/equirectangular-video-adapter-5.7.3.tgz",
+      "integrity": "sha512-a0vUihauMhWuNVkp6dWvE1dkmHjMNIe1GEQYUwpduBJlGNabzx7mp3iJNll9NqmLpz85iHvGNpunn5J+zVhfsg==",
       "peerDependencies": {
-        "@photo-sphere-viewer/core": "5.7.2"
+        "@photo-sphere-viewer/core": "5.7.3"
       }
     },
     "node_modules/@photo-sphere-viewer/gyroscope-plugin": {
-      "version": "5.7.2",
-      "resolved": "https://registry.npmjs.org/@photo-sphere-viewer/gyroscope-plugin/-/gyroscope-plugin-5.7.2.tgz",
-      "integrity": "sha512-GHbm96fBfcxzo1RXYsuuhCRxR0TIv3S+3HszyyNKF4x9Nc6dledl26s98ND5ivnviHtAzH/u4eIWHTO3t2+HMg==",
+      "version": "5.7.3",
+      "resolved": "https://registry.npmjs.org/@photo-sphere-viewer/gyroscope-plugin/-/gyroscope-plugin-5.7.3.tgz",
+      "integrity": "sha512-hS5ePszcR80Lb6ItLRK5xcUMDHsHyf1ebWeEeIwgMfURMdpenRA3phrrlSz8nVmeG/IQB0NRahHubUFfrIv/8g==",
       "peerDependencies": {
-        "@photo-sphere-viewer/core": "5.7.2"
+        "@photo-sphere-viewer/core": "5.7.3"
       }
     },
     "node_modules/@photo-sphere-viewer/markers-plugin": {
-      "version": "5.7.2-fix.1",
-      "resolved": "https://registry.npmjs.org/@photo-sphere-viewer/markers-plugin/-/markers-plugin-5.7.2-fix.1.tgz",
-      "integrity": "sha512-zNJet/ACLBfZgKEl1fCz1qNd6aYey5/2Zqr2++xts0hx179qzA8V3ka2w55SQb/X3WQEWMjomgjtroNd/4wSwg==",
+      "version": "5.7.3",
+      "resolved": "https://registry.npmjs.org/@photo-sphere-viewer/markers-plugin/-/markers-plugin-5.7.3.tgz",
+      "integrity": "sha512-m4f/vqCAMnwEssHiN1akvnsmD6yWdREI2t7Hs/k+nsbWd/vJ2XKb0iio/JyZWbCJhgsKGZ5sasqzhdxSOQBI8A==",
       "peerDependencies": {
-        "@photo-sphere-viewer/core": "5.7.2"
+        "@photo-sphere-viewer/core": "5.7.3"
       }
     },
     "node_modules/@photo-sphere-viewer/overlays-plugin": {
-      "version": "5.7.2",
-      "resolved": "https://registry.npmjs.org/@photo-sphere-viewer/overlays-plugin/-/overlays-plugin-5.7.2.tgz",
-      "integrity": "sha512-0Jp6yWqMzBWr+TRrp90Ua9+YqzmE1a/m+Trg+8FFnXWDc4u1SWPGvwCa4xk0N/x8b5Vx24UnMCRCTQo8B+Ldxw==",
+      "version": "5.7.3",
+      "resolved": "https://registry.npmjs.org/@photo-sphere-viewer/overlays-plugin/-/overlays-plugin-5.7.3.tgz",
+      "integrity": "sha512-OaoDXjsG6r5RuC7sKft+tfFFTJE1dbQokZM3/rB34kKbVpLWt9L8NJNBr1oBYyZt+3Fv5EYhL0MsmpTqf/gZTw==",
       "peerDependencies": {
-        "@photo-sphere-viewer/core": "5.7.2"
+        "@photo-sphere-viewer/core": "5.7.3"
       }
     },
     "node_modules/@photo-sphere-viewer/resolution-plugin": {
-      "version": "5.7.2",
-      "resolved": "https://registry.npmjs.org/@photo-sphere-viewer/resolution-plugin/-/resolution-plugin-5.7.2.tgz",
-      "integrity": "sha512-4gUd4yI8o4GEmHMmBeF3+rN+ElrMXxflDxXKRll+VBGCYAsx3vBw0cHm4kVjYh07ep+6uCa35tyg2VFi1P4mrA==",
+      "version": "5.7.3",
+      "resolved": "https://registry.npmjs.org/@photo-sphere-viewer/resolution-plugin/-/resolution-plugin-5.7.3.tgz",
+      "integrity": "sha512-DbSnIWnwFNa6jeIMGnVsmmuQGn5yZfUN5J9Ew7Xg3CYn5y3HT8EMF/A+yIZQCe7J1AnA30BRhOK1sqgLivDHjA==",
       "peerDependencies": {
-        "@photo-sphere-viewer/core": "5.7.2",
-        "@photo-sphere-viewer/settings-plugin": "5.7.2"
+        "@photo-sphere-viewer/core": "5.7.3",
+        "@photo-sphere-viewer/settings-plugin": "5.7.3"
       }
     },
     "node_modules/@photo-sphere-viewer/settings-plugin": {
-      "version": "5.7.2",
-      "resolved": "https://registry.npmjs.org/@photo-sphere-viewer/settings-plugin/-/settings-plugin-5.7.2.tgz",
-      "integrity": "sha512-RnDG0dPmLtZ6mWnUhL9dSDhztMYTVCd/PwwvhqL3rIHMxcteFxjVKRdVQyLvxUoWGj9O6bvrHYGLEQM1wMNX1g==",
+      "version": "5.7.3",
+      "resolved": "https://registry.npmjs.org/@photo-sphere-viewer/settings-plugin/-/settings-plugin-5.7.3.tgz",
+      "integrity": "sha512-YYhDd2xKYpxwnwgTq2h04+Aq19UALJbySc7B7GNlLilWuGE9Uy/u47PfUHkA5rJiUdJ0cOaXi7DOtPQZvPKiBQ==",
       "peerDependencies": {
-        "@photo-sphere-viewer/core": "5.7.2"
+        "@photo-sphere-viewer/core": "5.7.3"
       }
     },
     "node_modules/@photo-sphere-viewer/stereo-plugin": {
-      "version": "5.7.2",
-      "resolved": "https://registry.npmjs.org/@photo-sphere-viewer/stereo-plugin/-/stereo-plugin-5.7.2.tgz",
-      "integrity": "sha512-vL0a0An5rlcMpERkuWRWlAkt5KoS8LovbMImAB0zZ9YFuorp/xZCKuODQ8HADz3YknoH0ohTAnuxusV7c5FQmA==",
+      "version": "5.7.3",
+      "resolved": "https://registry.npmjs.org/@photo-sphere-viewer/stereo-plugin/-/stereo-plugin-5.7.3.tgz",
+      "integrity": "sha512-UGl9M3YilHcb1HhlTSl+hK+wfVdHrqKj4xSseJ5WDfFV3EErWhpSbg796GX+KWRrvF11EamhkXzAlR7ei5Y4hw==",
       "peerDependencies": {
-        "@photo-sphere-viewer/core": "5.7.2",
-        "@photo-sphere-viewer/gyroscope-plugin": "5.7.2"
+        "@photo-sphere-viewer/core": "5.7.3",
+        "@photo-sphere-viewer/gyroscope-plugin": "5.7.3"
       }
     },
     "node_modules/@photo-sphere-viewer/video-plugin": {
-      "version": "5.7.2",
-      "resolved": "https://registry.npmjs.org/@photo-sphere-viewer/video-plugin/-/video-plugin-5.7.2.tgz",
-      "integrity": "sha512-vrPV9RCr4HsYiORkto1unDPeUkbN2kbyogvNUoLiQ78M4xkPOqoKxtfxCxTYoM+7gECwNL9VTF81+okck498qA==",
+      "version": "5.7.3",
+      "resolved": "https://registry.npmjs.org/@photo-sphere-viewer/video-plugin/-/video-plugin-5.7.3.tgz",
+      "integrity": "sha512-bUo6qLT2Tnbc8d/Q7iZEDor2jWqL91ZKgAozxtXB86FqkYtzVoqmjhP008fMRxtoNCJFe8biisjTryYJ7oYp9g==",
       "peerDependencies": {
-        "@photo-sphere-viewer/core": "5.7.2"
+        "@photo-sphere-viewer/core": "5.7.3"
       }
     },
     "node_modules/@photo-sphere-viewer/visible-range-plugin": {
-      "version": "5.7.2",
-      "resolved": "https://registry.npmjs.org/@photo-sphere-viewer/visible-range-plugin/-/visible-range-plugin-5.7.2.tgz",
-      "integrity": "sha512-vvLoDOkQbuG0YQC8NgKPORmfbKXpAn4AFWi/bjB9ziYPsXU9LTVobiIjqAwbHxh6/yzCZoAscRFb601K85H1mA==",
+      "version": "5.7.3",
+      "resolved": "https://registry.npmjs.org/@photo-sphere-viewer/visible-range-plugin/-/visible-range-plugin-5.7.3.tgz",
+      "integrity": "sha512-dO+qHQsTxuiyrhd3ESRxHOZbE0ZCtAotzv8XSPcOrgnnbHp+2AMKKJhCwuQB58vUb9sxHf4NBo13MWU37vZm1g==",
       "peerDependencies": {
-        "@photo-sphere-viewer/core": "5.7.2"
+        "@photo-sphere-viewer/core": "5.7.3"
       }
     },
     "node_modules/@pkgjs/parseargs": {
@@ -3698,9 +3699,9 @@
       }
     },
     "node_modules/caniuse-lite": {
-      "version": "1.0.30001611",
-      "resolved": "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001611.tgz",
-      "integrity": "sha512-19NuN1/3PjA3QI8Eki55N8my4LzfkMCRLgCVfrl/slbSAchQfV0+GwjPrK3rq37As4UCLlM/DHajbKkAqbv92Q==",
+      "version": "1.0.30001612",
+      "resolved": "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001612.tgz",
+      "integrity": "sha512-lFgnZ07UhaCcsSZgWW0K5j4e69dK1u/ltrL9lTUiFOwNHs12S3UMIEYgBV0Z6C6hRDev7iRnMzzYmKabYdXF9g==",
       "funding": [
         {
           "type": "opencollective",
@@ -4272,9 +4273,9 @@
       "integrity": "sha512-I88TYZWc9XiYHRQ4/3c5rjjfgkjhLyW2luGIheGERbNQ6OY7yTybanSpDXZa8y7VUP9YmDcYa+eyq4ca7iLqWA=="
     },
     "node_modules/electron-to-chromium": {
-      "version": "1.4.744",
-      "resolved": "https://registry.npmjs.org/electron-to-chromium/-/electron-to-chromium-1.4.744.tgz",
-      "integrity": "sha512-nAGcF0yeKKfrP13LMFr5U1eghfFSvFLg302VUFzWlcjPOnUYd52yU5x6PBYrujhNbc4jYmZFrGZFK+xasaEzVA=="
+      "version": "1.4.746",
+      "resolved": "https://registry.npmjs.org/electron-to-chromium/-/electron-to-chromium-1.4.746.tgz",
+      "integrity": "sha512-jeWaIta2rIG2FzHaYIhSuVWqC6KJYo7oSBX4Jv7g+aVujKztfvdpf+n6MGwZdC5hQXbax4nntykLH2juIQrfPg=="
     },
     "node_modules/elliptic": {
       "version": "6.5.4",
@@ -6005,6 +6006,14 @@
         "jiti": "bin/jiti.js"
       }
     },
+    "node_modules/jose": {
+      "version": "5.2.4",
+      "resolved": "https://registry.npmjs.org/jose/-/jose-5.2.4.tgz",
+      "integrity": "sha512-6ScbIk2WWCeXkmzF6bRPmEuaqy1m8SbsRFMa/FLrSCkGIhj8OLVG/IH+XHVmNMx/KUo8cVWEE6oKR4dJ+S0Rkg==",
+      "funding": {
+        "url": "https://github.com/sponsors/panva"
+      }
+    },
     "node_modules/js-sha3": {
       "version": "0.8.0",
       "resolved": "https://registry.npmjs.org/js-sha3/-/js-sha3-0.8.0.tgz",
@@ -6606,9 +6615,9 @@
       }
     },
     "node_modules/openai": {
-      "version": "4.38.1",
-      "resolved": "https://registry.npmjs.org/openai/-/openai-4.38.1.tgz",
-      "integrity": "sha512-nmSKE9O2piuoh9+AgDqwGHojIFSxToQ2jJqwaxjbzz2ebdD5LYY9s+bMe25b18t4QEgvtgW70JfK8BU3xf5dRw==",
+      "version": "4.38.3",
+      "resolved": "https://registry.npmjs.org/openai/-/openai-4.38.3.tgz",
+      "integrity": "sha512-mIL9WtrFNOanpx98mJ+X/wkoepcxdqqu0noWFoNQHl/yODQ47YM7NEYda7qp8JfjqpLFVxY9mQhshoS/Fqac0A==",
       "dependencies": {
         "@types/node": "^18.11.18",
         "@types/node-fetch": "^2.6.4",
@@ -8217,9 +8226,9 @@
       }
     },
     "node_modules/type-fest": {
-      "version": "4.15.0",
-      "resolved": "https://registry.npmjs.org/type-fest/-/type-fest-4.15.0.tgz",
-      "integrity": "sha512-tB9lu0pQpX5KJq54g+oHOLumOx+pMep4RaM6liXh2PKmVRFF+/vAtUP0ZaJ0kOySfVNjF6doBWPHhBhISKdlIA==",
+      "version": "4.16.0",
+      "resolved": "https://registry.npmjs.org/type-fest/-/type-fest-4.16.0.tgz",
+      "integrity": "sha512-z7Rf5PXxIhbI6eJBTwdqe5bO02nUUmctq4WqviFSstBAWV0YNtEQRhEnZw73WJ8sZOqgFG6Jdl8gYZu7NBJZnA==",
       "engines": {
         "node": ">=16"
       },

package.json

@@ -59,13 +59,14 @@
     "eslint": "8.45.0",
     "eslint-config-next": "13.4.10",
     "fastest-levenshtein": "^1.0.16",
-    "gsplat": "^1.2.3",
+    "gsplat": "^1.2.4",
     "hash-wasm": "^4.11.0",
+    "jose": "^5.2.4",
     "lodash.debounce": "^4.0.8",
     "lucide-react": "^0.260.0",
     "markdown-yaml-metadata-parser": "^3.0.0",
     "minisearch": "^6.3.0",
-    "next": "^14.1.4",
+    "next": "^14.2.2",
     "openai": "^4.36.0",
     "photo-sphere-viewer-lensflare-plugin": "^2.1.2",
     "pick": "^0.0.1",

src/app/api/auth/getToken.ts

@@ -0,0 +1,20 @@
+import { createSecretKey } from "crypto"
+import { SignJWT } from "jose"
+
+// https://jmswrnr.com/blog/protecting-next-js-api-routes-query-parameters
+
+export async function getToken(data: Record<string, any> = {}): Promise<string> {
+  const secretKey = createSecretKey(`${process.env.API_SECRET_JWT_KEY || ""}`, 'utf-8');
+
+  const jwtToken = await new SignJWT(data)
+   .setProtectedHeader({
+    alg: 'HS256'
+   }) // algorithm
+   .setIssuedAt()
+   .setIssuer(`${process.env.API_SECRET_JWT_ISSUER || ""}`) // issuer
+   .setAudience(`${process.env.API_SECRET_JWT_AUDIENCE || ""}`) // audience
+   .setExpirationTime("1 day") // token expiration time - to prevent hackers from re-using our URLs more than a day
+   .sign(secretKey); // secretKey generated from previous step
+
+  return jwtToken
+}

src/app/api/generate/story/route.ts

@@ -0,0 +1,28 @@
+import { NextResponse, NextRequest } from "next/server"
+
+import { generateClapFromSimpleStory } from "@/lib/clap/generateClapFromSimpleStory"
+import { serializeClap } from "@/lib/clap/serializeClap"
+
+// a helper to generate Clap stories from a few sentences
+// this is mostly used by external apps such as the Stories Factory
+export async function POST(req: NextRequest) {
+
+  const request = await req.json() as {
+    story: string[]
+    // can add more stuff for the V2 of Stories Factory
+  }
+  
+  const story = Array.isArray(request?.story) ? request.story : []
+
+  if (!story.length) { throw new Error(`please provide at least oen sentence for the story`) }
+
+  const clap = generateClapFromSimpleStory({
+    story,
+    // can add more stuff for the V2 of Stories Factory
+  })
+
+  return new NextResponse(await serializeClap(clap), {
+    status: 200,
+    headers: new Headers({ "content-type": "application/x-gzip" }),
+  })
+}

src/app/api/generate/storyboards/route.ts

@@ -0,0 +1,84 @@
+import { NextResponse, NextRequest } from "next/server"
+
+import { serializeClap } from "@/lib/clap/serializeClap"
+import { parseClap } from "@/lib/clap/parseClap"
+import { startOfSegment1IsWithinSegment2 } from "@/lib/utils/startOfSegment1IsWithinSegment2"
+import { getVideoPrompt } from "@/components/interface/latent-engine/core/prompts/getVideoPrompt"
+import { newSegment } from "@/lib/clap/newSegment"
+import { generateImage } from "@/components/interface/latent-engine/resolvers/image/generateImage"
+import { getToken } from "@/app/api/auth/getToken"
+
+// a helper to generate storyboards for a Clap
+// this is mostly used by external apps such as the Stories Factory
+// this function will:
+//
+// - add missing storyboards to the shots
+// - add missing storyboard prompts
+// - add missing storyboard images
+export async function POST(req: NextRequest) {
+
+  const jwtToken = await getToken({ user: "anonymous" })
+
+  const blob = await req.blob()
+  const clap = await parseClap(blob)
+
+  if (!clap.segments) { throw new Error(`no segment found in the provided clap!`) }
+
+  const shotsSegments = clap.segments.filter(s => s.category === "camera")
+
+  if (shotsSegments.length > 32) {
+    throw new Error(`Error, this endpoint being synchronous, it is designed for short stories only (max 32 shots).`)
+  }
+
+  for (const shotSegment of shotsSegments) {
+
+    const shotSegments = clap.segments.filter(s =>
+      startOfSegment1IsWithinSegment2(s, shotSegment)
+    )
+
+    const shotStoryboardSegments = shotSegments.filter(s =>
+      s.category === "storyboard"
+    )
+
+    let shotStoryboardSegment = shotStoryboardSegments.at(0)
+
+    // TASK 1: GENERATE MISSING STORYBOARD SEGMENT
+    if (!shotStoryboardSegment) {
+      shotStoryboardSegment = newSegment({
+        track: 1,
+        startTimeInMs: shotSegment.startTimeInMs,
+        endTimeInMs: shotSegment.endTimeInMs,
+        assetDurationInMs: shotSegment.assetDurationInMs,
+        category: "storyboard",
+        prompt: "",
+        assetUrl: "",
+        outputType: "image"
+      })
+    }
+
+    // TASK 2: GENERATE MISSING STORYBOARD PROMPT
+    if (!shotStoryboardSegment.prompt) {
+      // storyboard is missing, let's generate it
+      shotStoryboardSegment.prompt = getVideoPrompt(shotSegments, {}, [])
+    }
+
+    // TASK 3: GENERATE MISSING STORYBOARD BITMAP
+    if (!shotStoryboardSegment.assetUrl) {
+      // note this will do a fetch to AiTube API
+      // which is a bit weird since we are already inside the API, but it works
+      //TODO Julian: maybe we could use an internal function call instead?
+      shotStoryboardSegment.assetUrl = await generateImage({
+        prompt: shotStoryboardSegment.prompt,
+         width: clap.meta.width,
+        height: clap.meta.height,
+        token: jwtToken,
+      })
+    }
+  }
+  // TODO: generate the storyboards for the clap 
+
+  return new NextResponse(await serializeClap(clap), {
+    status: 200,
+    headers: new Headers({ "content-type": "application/x-gzip" }),
+  })
+}

src/app/api/generate/video/route.ts

@@ -0,0 +1,19 @@
+import { NextResponse, NextRequest } from "next/server"
+
+// we hide/wrap the micro-service under a unified AiTube API
+export async function POST(req: NextRequest, res: NextResponse) {
+  NextResponse.redirect("https://jbilcke-hf-ai-tube-clap-exporter.hf.space")
+}
+/*
+Alternative solution (in case the redirect doesn't work):
+
+We could also grab the blob and forward it, like this:
+
+  const data = fetch(
+    "https://jbilcke-hf-ai-tube-clap-exporter.hf.space",
+    { method: "POST", body: await req.blob() }
+  )
+  const blob = data.blob()
+
+Then return the blob with the right Content-Type using NextResponse
+*/

src/app/api/generators/clap/generateClap.ts

@@ -39,7 +39,8 @@ export async function generateClap({
       defaultVideoModel: "SDXL",
       extraPositivePrompt: [],
       screenplay: "",
-      streamType: "interactive"
+      isLoop: true,
+      isInteractive: true,
     } 
   })
 

src/app/api/resolvers/image/route.ts

@@ -1,4 +1,6 @@
 import { NextResponse, NextRequest } from "next/server"
+import { createSecretKey } from "node:crypto"
+
 import queryString from "query-string"
 
 import { newRender, getRender } from "../../providers/videochain/renderWithVideoChain"
@@ -6,13 +8,37 @@ import { generateSeed } from "@/lib/utils/generateSeed"
 import { sleep } from "@/lib/utils/sleep"
 import { getNegativePrompt, getPositivePrompt } from "../../utils/imagePrompts"
 import { getContentType } from "@/lib/data/getContentType"
+import { getValidNumber } from "@/lib/utils/getValidNumber"
+
+const secretKey = createSecretKey(`${process.env.API_SECRET_JWT_KEY || ""}`, 'utf-8');
 
 export async function GET(req: NextRequest) {
 
-const qs = queryString.parseUrl(req.url || "")
-const query = (qs || {}).query
+  const qs = queryString.parseUrl(req.url || "")
+  const query = (qs || {}).query
 
-let prompt = ""
+  /*
+  TODO: check the validity of the JWT token
+  let token = ""
+  try {
+    token = decodeURIComponent(query?.t?.toString() || "").trim()
+
+    // verify token
+    const { payload, protectedHeader } = await jwtVerify(token, secretKey, {
+      issuer: `${process.env.API_SECRET_JWT_ISSUER || ""}`, // issuer
+      audience: `${process.env.API_SECRET_JWT_AUDIENCE || ""}`, // audience
+    });
+      // log values to console
+      console.log(payload);
+      console.log(protectedHeader);
+  } catch (err) {
+    // token verification failed
+    console.log("Token is invalid");
+    return NextResponse.json({ error: `access denied ${err}` }, { status: 400 });
+  }
+  */
+
+  let prompt = ""
   try {
     prompt = decodeURIComponent(query?.p?.toString() || "").trim()
   } catch (err) {}
@@ -20,6 +46,19 @@ let prompt = ""
     return NextResponse.json({ error: 'no prompt provided' }, { status: 400 });
   }
 
+  let width = 512
+  try {
+    const rawString = decodeURIComponent(query?.w?.toString() || "").trim()
+    width = getValidNumber(rawString, 256, 8192, 512)
+  } catch (err) {}
+
+  let height = 288
+  try {
+    const rawString = decodeURIComponent(query?.h?.toString() || "").trim()
+    height = getValidNumber(rawString, 256, 8192, 288)
+  } catch (err) {}
+
+
   let format = "binary"
   try {
     const f = decodeURIComponent(query?.f?.toString() || "").trim()
@@ -36,8 +75,8 @@ let prompt = ""
     nbFrames: 1,
     nbFPS: 1,
     nbSteps: 8,
-    width: 1024,
-    height: 576,
+    width,
+    height,
     turbo: true,
     shouldRenewCache: true,
     seed: generateSeed()

src/app/api/resolvers/video/route.ts

@@ -1,17 +1,43 @@
 import { NextResponse, NextRequest } from "next/server"
 import queryString from "query-string"
+import { createSecretKey } from "crypto"
+import { jwtVerify } from "jose"
 
 import { newRender, getRender } from "../../providers/videochain/renderWithVideoChain"
 import { generateSeed } from "@/lib/utils/generateSeed"
 import { sleep } from "@/lib/utils/sleep"
 import { getNegativePrompt, getPositivePrompt } from "../../utils/imagePrompts"
 import { getContentType } from "@/lib/data/getContentType"
+import { whoAmI, WhoAmIUser } from "@huggingface/hub"
+import { getValidNumber } from "@/lib/utils/getValidNumber"
+
+const secretKey = createSecretKey(`${process.env.API_SECRET_JWT_KEY || ""}`, 'utf-8');
 
 export async function GET(req: NextRequest) {
 
   const qs = queryString.parseUrl(req.url || "")
   const query = (qs || {}).query
 
+  /*
+  TODO Julian: check the validity of the JWT token
+  let token = ""
+  try {
+    token = decodeURIComponent(query?.t?.toString() || "").trim()
+
+    // verify token
+    const { payload, protectedHeader } = await jwtVerify(token, secretKey, {
+      issuer: `${process.env.API_SECRET_JWT_ISSUER || ""}`, // issuer
+      audience: `${process.env.API_SECRET_JWT_AUDIENCE || ""}`, // audience
+    });
+      // log values to console
+      console.log(payload);
+      console.log(protectedHeader);
+  } catch (err) {
+    // token verification failed
+    console.log("Token is invalid");
+    return NextResponse.json({ error: `access denied ${err}` }, { status: 400 });
+  }
+  */
 
   let prompt = ""
   try {
@@ -22,6 +48,19 @@ export async function GET(req: NextRequest) {
     return NextResponse.json({ error: 'no prompt provided' }, { status: 400 });
   }
 
+  let width = 512
+  try {
+    const rawString = decodeURIComponent(query?.w?.toString() || "").trim()
+    width = getValidNumber(rawString, 256, 8192, 512)
+  } catch (err) {}
+
+  let height = 288
+  try {
+    const rawString = decodeURIComponent(query?.h?.toString() || "").trim()
+    height = getValidNumber(rawString, 256, 8192, 288)
+  } catch (err) {}
+
+
   let format = "binary"
   try {
     const f = decodeURIComponent(query?.f?.toString() || "").trim()
@@ -40,19 +79,52 @@ export async function GET(req: NextRequest) {
     // ATTENTION: changing those will slow things to 5-6s of loading time (compared to 3-4s)
     // and with no real visible change
 
-    nbFrames: 20, // apparently the model can only do 2 seconds at 10, so be it
+    // ATTENTION! if you change those values,
+    // please make sure that the backend API can support them,
+    // and also make sure to update the Zustand store values in the frontend:
+    // videoModelFPS: number
+    // videoModelNumOfFrames: number
+    // videoModelDurationInSec: number
+    //
+    // note: internally, the model can only do 16 frames at 10 FPS
+    // (1.6 second of video)
+    // but I have added a FFmpeg interpolation step, which adds some
+    // overhead (2-3 secs) but at least can help smooth things out, or make
+    // them artificially longer
+
+    // those settings are pretty good, takes about 2.9,, 3.1 seconds to compute
+    // represents 3 secs of 16fps
+  
+
+    // with those parameters, we can generate a 2.584s long video at 24 FPS
+    // note that there is a overhead due to smoothing,
+    // on the A100 it takes betwen 5.3 and 7 seconds to compute
+    // although it will appear a bit "slo-mo"
+    // since the original is a 1.6s long video at 10 FPS
+    nbFrames: 80,
+    nbFPS: 24,
+
+    
+    // nbFrames: 48,
+    // nbFPS: 24,
 
-    nbFPS: 10,
+    // it generated about:
+    // 24 frames
+    // 2.56s run time
 
     // possibles values are 1, 2, 4, and 8
-    // but I don't see much improvements with 8 to be honest
-    // the best one seems to be 4
+    // but with 2 steps the video "flashes" and it creates monstruosity
+    // like fishes with 2 tails etc
+    // and with 8 steps I don't see much improvements with 8 to be honest
     nbSteps: 4,
 
     // this corresponds roughly to 16:9
     // which is the aspect ratio video used by AiTube
 
-    // unfortunately, this is too compute intensive, so we have to take half of that
+    // unfortunately, this is too compute intensive,
+    // and it creates monsters like two-headed fishes
+    // (although this artifact could probably be fixed with more steps,
+    // but we cannot afford those)
     // width: 1024,
     // height: 576,
 
@@ -68,10 +140,15 @@ export async function GET(req: NextRequest) {
     //
     // that's not the only constraint: you also need to respect this:
     // `height` and `width` have to be divisible by 8 (use 32 to be safe)
-    // width: 512,
-    // height: 288,
-    width: 456, // 512,
-    height: 256, // 288,
+    width,
+    height,
+
+    // we save about 500ms if we go below,
+    // but there we will be some deformed artifacts as the model
+    // doesn't perform well below 512px
+    // it also makes things more "flashy"
+    // width: 456, // 512,
+    // height: 256, // 288,
 
     turbo: true, // without much effect for videos as of now, as we only supports turbo (AnimateDiff Lightning)
     shouldRenewCache: true,

src/app/dream/{page.tsx → spoiler.tsx}

@@ -1,18 +1,20 @@
-
-
 import { LatentQueryProps } from "@/types/general"
 
 import { Main } from "../main"
-import { searchResultToMediaInfo } from "../api/generators/search/searchResultToMediaInfo"
-import { LatentSearchResult } from "../api/generators/search/types"
-import { serializeClap } from "@/lib/clap/serializeClap"
-import { getMockClap } from "@/lib/clap/getMockClap"
+import { generateClapFromSimpleStory } from "@/lib/clap/generateClapFromSimpleStory"
 import { clapToDataUri } from "@/lib/clap/clapToDataUri"
 import { getNewMediaInfo } from "../api/generators/search/getNewMediaInfo"
+import { getToken } from "../api/auth/getToken"
 
-export default async function DreamPage({ searchParams: {
-  l: latentContent,
-} }: LatentQueryProps) {
+// https://jmswrnr.com/blog/protecting-next-js-api-routes-query-parameters
+
+export default async function DreamPage({
+  searchParams: {
+    l: latentContent,
+  },
+  ...rest
+}: LatentQueryProps) {
+  const jwtToken = await getToken({ user: "anonymous" })
 
   // const latentSearchResult = JSON.parse(atob(`${latentContent}`)) as LatentSearchResult
 
@@ -24,10 +26,13 @@ export default async function DreamPage({ searchParams: {
   const latentMedia = getNewMediaInfo()
 
   latentMedia.clapUrl = await clapToDataUri(
-    getMockClap({showDisclaimer: true })
+    generateClapFromSimpleStory({
+      showIntroPoweredByEngine: false,
+      showIntroDisclaimerAboutAI: false
+    })
   )
 
   return (
-    <Main latentMedia={latentMedia} />
-   )
+    <Main latentMedia={latentMedia} jwtToken={jwtToken} />
+  )
 }

src/app/main.tsx

@@ -29,6 +29,8 @@ import { PublicLatentMediaView } from "./views/public-latent-media-view"
 // one benefit of doing this is that we will able to add some animations/transitions
 // more easily
 export function Main({
+  jwtToken,
+
   // view,
   publicMedia,
   publicMedias,
@@ -42,25 +44,31 @@ export function Main({
   publicTrack,
   channel,
 }: {
- // server side params
- // view?: InterfaceView
- publicMedia?: MediaInfo
- publicMedias?: MediaInfo[]
-
- latentMedia?: MediaInfo
- latentMedias?: MediaInfo[]
-
- publicChannelVideos?: MediaInfo[]
-
- publicTracks?: MediaInfo[]
- publicTrack?: MediaInfo
-
- channel?: ChannelInfo
+  // token used to secure communications between the Next frontend and the Next API
+  // this doesn't necessarily mean the user has to be logged it:
+  // we can use this for anonymous visitors too.
+  jwtToken?: string
+ 
+  // server side params
+  // view?: InterfaceView
+  publicMedia?: MediaInfo
+  publicMedias?: MediaInfo[]
+ 
+  latentMedia?: MediaInfo
+  latentMedias?: MediaInfo[]
+ 
+  publicChannelVideos?: MediaInfo[]
+ 
+  publicTracks?: MediaInfo[]
+  publicTrack?: MediaInfo
+ 
+  channel?: ChannelInfo
 }) {
   // this could be also a parameter of main, where we pass this manually
   const pathname = usePathname()
   const router = useRouter()
 
+  const setJwtToken = useStore(s => s.setJwtToken)
   const setPublicMedia = useStore(s => s.setPublicMedia)
   const setView = useStore(s => s.setView)
   const setPathname = useStore(s => s.setPathname)
@@ -72,6 +80,12 @@ export function Main({
   const setPublicTracks = useStore(s => s.setPublicTracks)
   const setPublicTrack = useStore(s => s.setPublicTrack)
 
+  useEffect(() => {
+    if (typeof jwtToken !== "string" && !jwtToken) { return }
+    setJwtToken(jwtToken)
+  }, [jwtToken])
+
+
   useEffect(() => {
     if (!publicMedias?.length) { return }
     // note: it is important to ALWAYS set the current video to videoId

src/app/state/useStore.ts

@@ -19,6 +19,9 @@ export const useStore = create<{
 
   setPathname: (pathname: string) => void
 
+  jwtToken: string
+  setJwtToken: (jwtToken: string) => void
+
   searchQuery: string
   setSearchQuery: (searchQuery?: string) => void
 
@@ -131,6 +134,11 @@ export const useStore = create<{
     set({ view: routes[pathname] || "not_found" })
   },
 
+  jwtToken: "",
+  setJwtToken: (jwtToken: string) => {
+    set({ jwtToken })
+  },
+
   searchAutocompleteQuery: "",
   setSearchAutocompleteQuery: (searchAutocompleteQuery?: string) => {
     set({ searchAutocompleteQuery })

src/components/interface/latent-engine/components/content-layer/index.tsx

@@ -26,7 +26,7 @@ export const ContentLayer = forwardRef(function ContentLayer({
       ref={ref}
       onClick={onClick}
       >
-      <div className="h-full aspect-video opacity-60">
+      <div className="h-full aspect-video opacity-100">
         {children}
       </div>
     </div>

src/components/interface/latent-engine/components/{disclaimers/this-is-ai.tsx → intros/ai-content-disclaimer.tsx}

@@ -4,12 +4,11 @@ import React from "react"
 import { cn } from "@/lib/utils/cn"
 
 import { arimoBold, arimoNormal } from "@/lib/fonts"
-import { ClapStreamType } from "@/lib/clap/types"
 
-export function ThisIsAI({
-  streamType,
+export function AIContentDisclaimer({
+  isInteractive = false,
 }: {
-  streamType?: ClapStreamType
+  isInteractive?: boolean
 } = {}) {
 
   return (
@@ -59,7 +58,7 @@ export function ThisIsAI({
             */
             } content
           </div> {
-          streamType !== "static"
+          isInteractive
           ? "will be"
           : "has been"
           } <div className={cn(`

src/components/interface/latent-engine/components/intros/powered-by.tsx

@@ -0,0 +1,31 @@
+import React from "react"
+import Image from 'next/image'
+
+import { cn } from "@/lib/utils/cn"
+
+import latentEngineLogo from "./latent-engine.png"
+
+export function PoweredBy() {
+
+  return (
+  <div className={cn(`
+  flex flex-col flex-1
+  items-center justify-center
+  w-full h-full
+
+  bg-black
+  `,
+  )}>
+  
+    <div className={cn(`
+       flex flex-col items-center justify-center
+    `)}>
+      <Image
+        src={latentEngineLogo}
+        alt="Latent Engine logo"
+        className="w-[80%]"
+      />
+    </div>
+  </div>
+  )
+}

src/components/interface/latent-engine/core/engine.tsx

@@ -1,16 +1,20 @@
 "use client"
 
 import React, { MouseEventHandler, useEffect, useRef, useState } from "react"
+import { useLocalStorage } from "usehooks-ts"
 
 import { cn } from "@/lib/utils/cn"
+import { MediaInfo } from "@/types/general"
+import { serializeClap } from "@/lib/clap/serializeClap"
+import { generateClapFromSimpleStory } from "@/lib/clap/generateClapFromSimpleStory"
 
-import { useLatentEngine } from "../store/useLatentEngine"
+import { useLatentEngine } from "./useLatentEngine"
 import { PlayPauseButton } from "../components/play-pause-button"
-import { StreamTag } from "../../stream-tag"
+import { StaticOrInteractiveTag } from "../../static-or-interactive-tag"
 import { ContentLayer } from "../components/content-layer"
-import { MediaInfo } from "@/types/general"
-import { getMockClap } from "@/lib/clap/getMockClap"
-import { serializeClap } from "@/lib/clap/serializeClap"
+import { localStorageKeys } from "@/app/state/localStorageKeys"
+import { defaultSettings } from "@/app/state/defaultSettings"
+import { useStore } from "@/app/state/useStore"
 
 function LatentEngine({
   media,
@@ -22,19 +26,35 @@ function LatentEngine({
   height?: number
   className?: string
 }) {
+  // used to prevent people from opening multiple sessions at the same time
+  // note: this should also be enforced with the Hugging Face ID
+  const [multiTabsLock, setMultiTabsLock] = useLocalStorage<number>(
+    "AI_TUBE_ENGINE_MULTI_TABS_LOCK",
+    Date.now()
+  )
+
+  const [huggingfaceApiKey, setHuggingfaceApiKey] = useLocalStorage<string>(
+    localStorageKeys.huggingfaceApiKey,
+    defaultSettings.huggingfaceApiKey
+  )
+  
+  // note here how we transfer the info from one store to another
+  const jwtToken = useStore(s => s.jwtToken)
+  const setJwtToken = useLatentEngine(s => s.setJwtToken)
+  useEffect(() => {
+    setJwtToken(jwtToken)
+  }, [jwtToken])
+
+
   const setContainerDimension = useLatentEngine(s => s.setContainerDimension)
   const isLoaded = useLatentEngine(s => s.isLoaded)
   const imagine = useLatentEngine(s => s.imagine)
   const open = useLatentEngine(s => s.open)
 
-  const setImageElement = useLatentEngine(s => s.setImageElement)
-  const setVideoElement = useLatentEngine(s => s.setVideoElement)
-  const setSegmentationElement = useLatentEngine(s => s.setSegmentationElement)
-
-  const simulationVideoPlaybackFPS = useLatentEngine(s => s.simulationVideoPlaybackFPS)
-  const simulationRenderingTimeFPS = useLatentEngine(s => s.simulationRenderingTimeFPS)
+  const videoSimulationVideoPlaybackFPS = useLatentEngine(s => s.videoSimulationVideoPlaybackFPS)
+  const videoSimulationRenderingTimeFPS = useLatentEngine(s => s.videoSimulationRenderingTimeFPS)
 
-  const streamType = useLatentEngine(s => s.streamType)
+  const isLoop = useLatentEngine(s => s.isLoop)
   const isStatic = useLatentEngine(s => s.isStatic)
   const isLive = useLatentEngine(s => s.isLive)
   const isInteractive = useLatentEngine(s => s.isInteractive)
@@ -42,11 +62,8 @@ function LatentEngine({
   const isPlaying = useLatentEngine(s => s.isPlaying)
   const togglePlayPause = useLatentEngine(s => s.togglePlayPause)
 
-  const videoLayer = useLatentEngine(s => s.videoLayer)
-  const segmentationLayer = useLatentEngine(s => s.segmentationLayer)
-  const interfaceLayer = useLatentEngine(s => s.interfaceLayer)
-  const videoElement = useLatentEngine(s => s.videoElement)
-  const imageElement = useLatentEngine(s => s.imageElement)
+  const videoLayers = useLatentEngine(s => s.videoLayers)
+  const interfaceLayers = useLatentEngine(s => s.interfaceLayers)
 
   const onClickOnSegmentationLayer = useLatentEngine(s => s.onClickOnSegmentationLayer)
 
@@ -70,7 +87,7 @@ function LatentEngine({
         // there is a bug, we can't unpack the .clap when it's from a data-uri :/
         
         // open(mediaUrl)
-        const mockClap = getMockClap()
+        const mockClap = generateClapFromSimpleStory()
         const mockArchive = await serializeClap(mockClap)
         // for some reason conversion to data uri doesn't work
         // const mockDataUri = await blobToDataUri(mockArchive, "application/x-gzip")
@@ -91,7 +108,7 @@ function LatentEngine({
         setOverlayVisible(!isPlayingRef.current)
       }
       clearTimeout(overlayTimerRef.current)
-    }, 1000)
+    }, 3000)
   }
 
   /*
@@ -109,32 +126,6 @@ function LatentEngine({
   }, [isPlaying])
   */
 
-  useEffect(() => {
-    if (!videoLayerRef.current) { return }
- 
-    // note how in both cases we are pulling from the videoLayerRef
-    // that's because one day everything will be a video, but for now we
-    // "fake it until we make it"
-    const videoElements = Array.from(
-      videoLayerRef.current.querySelectorAll('.latent-video')
-    ) as HTMLVideoElement[]
-    setVideoElement(videoElements.at(0))
-
-    // images are used for simpler or static experiences
-    const imageElements = Array.from(
-      videoLayerRef.current.querySelectorAll('.latent-image')
-    ) as HTMLImageElement[]
-    setImageElement(imageElements.at(0))
-
-
-    if (!segmentationLayerRef.current) { return }
- 
-     const segmentationElements = Array.from(
-       segmentationLayerRef.current.querySelectorAll('.segmentation-canvas')
-     ) as HTMLCanvasElement[]
-     setSegmentationElement(segmentationElements.at(0))
-
-  })
 
   useEffect(() => {
     setContainerDimension({ width: width || 256, height: height || 256 })
@@ -161,9 +152,26 @@ function LatentEngine({
         height={height}
         ref={videoLayerRef}
         onClick={onClickOnSegmentationLayer}
-      >{videoLayer}</ContentLayer>
-
-
+      >{videoLayers.map(({ id }) => (
+        <video
+          key={id}
+          id={id}
+          style={{ width, height }}
+          className={cn(
+            `video-buffer`,
+            `video-buffer-${id}`,
+          )}
+          data-segment-id="0"
+          data-segment-start-at="0"
+          data-z-index-depth="0"
+          playsInline={true}
+          muted={true}
+          autoPlay={false}
+          loop={true}
+          src="/blanks/blank_1sec_512x288.webm"
+      />))}
+      </ContentLayer>
+        
       <ContentLayer
         className="pointer-events-none"
         width={width}
@@ -174,14 +182,20 @@ function LatentEngine({
         style={{ width, height }}
       ></canvas></ContentLayer>
 
+      
       {/*
       <ContentLayer
         className="pointer-events-auto"
         width={width}
         height={height}
-  >{interfaceLayer}</ContentLayer>
-  */}
-      
+     >{interfaceLayers.map(({ id, element }) => (
+      <div
+        key={id}
+        id={id}
+        style={{ width, height }}
+        className={`interface-layer-${id}`}>{element}</div>))}</ContentLayer>
+     */}
+    
       {/* content overlay, with the gradient, buttons etc */}
       <div className={cn(`
         absolute
@@ -247,8 +261,8 @@ function LatentEngine({
                 isToggledOn={isPlaying}
                 onClick={togglePlayPause}
               />
-              <StreamTag
-                streamType={streamType}
+              <StaticOrInteractiveTag
+                isInteractive={isInteractive}
                 size="md"
                 className=""
               />
@@ -266,8 +280,8 @@ function LatentEngine({
               TODO: put a fullscreen button (and mode) here
 
              */}
-             <div className="mono text-xs text-center">playback: {Math.round(simulationVideoPlaybackFPS * 100) / 100} FPS</div>
-             <div className="mono text-xs text-center">rendering: {Math.round(simulationRenderingTimeFPS * 100) / 100} FPS</div>
+             <div className="mono text-xs text-center">playback: {Math.round(videoSimulationVideoPlaybackFPS * 100) / 100} FPS</div>
+             <div className="mono text-xs text-center">rendering: {Math.round(videoSimulationRenderingTimeFPS * 100) / 100} FPS</div>
             </div>
           </div>
         </div>

src/components/interface/latent-engine/core/prompts/getCharacterPrompt.ts

@@ -0,0 +1,26 @@
+import { ClapModel } from "@/lib/clap/types"
+
+export function getCharacterPrompt(model: ClapModel): string {
+
+  let characterPrompt = ""
+  if (model.description) {
+    characterPrompt = [
+      // the label (character name) can help making the prompt more unique
+      // this might backfires however, if the name is
+      // something like "SUN", "SILVER" etc
+      // I'm not sure stable diffusion really needs this,
+      // so let's skip it for now (might still be useful for locations, though)
+      // we also want to avoid triggering "famous people" (BARBOSSA etc)
+      // model.label,
+
+      model.description
+    ].join(", ")
+  } else {
+    characterPrompt = [
+      model.gender !== "object" ? model.gender : "",
+      model.age ? `aged ${model.age}yo` : '',
+      model.label ? `named ${model.label}` : '',
+    ].map(i => i.trim()).filter(i => i).join(", ")
+  }
+  return characterPrompt
+}

src/components/interface/latent-engine/core/prompts/getVideoPrompt.ts

@@ -0,0 +1,104 @@
+import { ClapModel, ClapSegment } from "@/lib/clap/types"
+
+import { deduplicatePrompt } from "../../utils/prompting/deduplicatePrompt"
+
+import { getCharacterPrompt } from "./getCharacterPrompt"
+
+/**
+ * Construct a video prompt from a list of active segments
+ * 
+ * @param segments 
+ * @returns 
+ */
+export function getVideoPrompt(
+  segments: ClapSegment[],
+  modelsById: Record<string, ClapModel>,
+  extraPositivePrompt: string[]
+): string {
+
+  // console.log("modelsById:", modelsById)
+
+  // to construct the video we need to collect all the segments describing it
+  // we ignore unrelated categories (music, dialogue) or non-prompt items (eg. an audio sample)
+  const tmp = segments
+    .filter(({ category, outputType }) => {
+      if (outputType === "audio") {
+        return false
+      }
+
+      if (category === "music" ||
+          category === "sound") {
+        return false
+      }
+
+      if (category === "event" ||
+          category === "interface" ||
+          category === "phenomenon"
+      ) {
+        return false
+      }
+      
+      if (category === "splat" ||
+          category === "mesh" ||
+          category === "depth"
+      ) {
+        return false
+      }
+
+      if (category === "storyboard" ||
+          category === "video") {
+        return false
+      }
+
+      if (category === "transition") {
+        return false
+      }
+
+      return true
+    })
+
+  tmp.sort((a, b) => b.label.localeCompare(a.label))
+
+  let videoPrompt = tmp.map(segment => {
+    const model: ClapModel | undefined = modelsById[segment?.modelId || ""] || undefined
+    
+    if (segment.category === "dialogue") {
+
+      // if we can't find the model, then we are unable
+      // to make any assumption about the gender, age or appearance
+      if (!model) {
+        console.log("ERROR: this is a dialogue, but couldn't find the model!")
+        return `portrait of a person speaking, blurry background, bokeh`
+      }
+
+      const characterTrigger = model?.triggerName || ""
+      const characterLabel = model?.label || ""
+      const characterDescription = model?.description || ""
+      const dialogueLine = segment?.prompt || ""
+     
+      const characterPrompt = getCharacterPrompt(model)
+
+      // in the context of a video, we some something additional:
+      // we create a "bokeh" style
+      return `portrait of a person speaking, blurry background, bokeh, ${characterPrompt}`
+      
+    } else if (segment.category === "location") {
+  
+      // if we can't find the location's model, we default to returning the prompt
+      if (!model) {
+        console.log("ERROR: this is a location, but couldn't find the model!")
+        return segment.prompt
+      }
+
+      return model.description
+    } else {
+      return segment.prompt
+    }
+  }).filter(x => x)
+
+  videoPrompt = videoPrompt.concat([
+    ...extraPositivePrompt
+  ])
+
+  return deduplicatePrompt(videoPrompt.join(", "))
+}

src/components/interface/latent-engine/core/types.ts

@@ -1,4 +1,4 @@
-import { ClapProject, ClapSegment, ClapStreamType } from "@/lib/clap/types"
+import { ClapProject, ClapSegment } from "@/lib/clap/types"
 import { InteractiveSegmenterResult } from "@mediapipe/tasks-vision"
 import { MouseEventHandler, ReactNode } from "react"
 
@@ -24,18 +24,31 @@ export type LayerCategory =
   | "video"
   | "splat"
 
-export type LatentComponentResolver = (segment: ClapSegment, clap: ClapProject) => Promise<JSX.Element>
+export type LatentComponentResolver = (segment: ClapSegment, clap: ClapProject) => Promise<LayerElement>
+
+export type LayerElement = {
+  id: string;
+  element: JSX.Element;
+}
 
 export type LatentEngineStore = {
+  // the token use to communicate with the NextJS backend
+  // note that this isn't the Hugging Face token,
+  // it is something more anynomous
+  jwtToken: string
+
   width: number
   height: number
 
   clap: ClapProject
   debug: boolean
 
-  streamType: ClapStreamType
+  // whether the engine is headless or not
+  // (pure chatbot apps won't need the live UI for instance)
+  headless: boolean
 
   // just some aliases for convenience
+  isLoop: boolean
   isStatic: boolean
   isLive: boolean
   isInteractive: boolean
@@ -51,46 +64,56 @@ export type LatentEngineStore = {
   isPlaying: boolean
   isPaused: boolean
 
-  simulationPromise?: Promise<void>
-  simulationPending: boolean // used as a "lock"
-  simulationStartedAt: number
-  simulationEndedAt: number
-  simulationDurationInMs: number
-  simulationVideoPlaybackFPS: number
-  simulationRenderingTimeFPS: number
+  videoSimulationPromise?: Promise<void>
+  videoSimulationPending: boolean // used as a "lock"
+  videoSimulationStartedAt: number
+  videoSimulationEndedAt: number
+  videoSimulationDurationInMs: number
+  videoSimulationVideoPlaybackFPS: number
+  videoSimulationRenderingTimeFPS: number
+
+  interfaceSimulationPromise?: Promise<void>
+  interfaceSimulationPending: boolean // used as a "lock"
+  interfaceSimulationStartedAt: number
+  interfaceSimulationEndedAt: number
+  interfaceSimulationDurationInMs: number
+
+  entitySimulationPromise?: Promise<void>
+  entitySimulationPending: boolean // used as a "lock"
+  entitySimulationStartedAt: number
+  entitySimulationEndedAt: number
+  entitySimulationDurationInMs: number
 
   renderingIntervalId: NodeJS.Timeout | string | number | undefined
   renderingIntervalDelayInMs: number
+  renderingLastRenderAt: number
+
+  // for our calculations to be correct
+  // those need to match the actual output from the API
+  // don't trust the parameters you send to the API,
+  // instead check the *actual* values with VLC!!
+  videoModelFPS: number
+  videoModelNumOfFrames: number
+  videoModelDurationInSec: number
+
+  playbackSpeed: number
 
   positionInMs: number
   durationInMs: number 
 
-  videoLayerElement?: HTMLDivElement
-  imageElement?: HTMLImageElement
-  videoElement?: HTMLVideoElement
-  segmentationElement?: HTMLCanvasElement
+  // this is the "buffer size"
+  videoLayers: LayerElement[]
+  videoElements: HTMLVideoElement[]
 
-  videoLayer: ReactNode
-  videoBuffer: "A" | "B"
-  videoBufferA: ReactNode
-  videoBufferB: ReactNode
+  interfaceLayers: LayerElement[]
 
-  segmentationLayer: ReactNode
-
-  interfaceLayer: ReactNode
-  interfaceBuffer: "A" | "B"
-  interfaceBufferA: ReactNode
-  interfaceBufferB: ReactNode
+  setJwtToken: (jwtToken: string) => void
 
   setContainerDimension: ({ width, height }: { width: number; height: number }) => void
   imagine: (prompt: string) => Promise<void>
   open: (src?: string | ClapProject | Blob) => Promise<void>
 
-  setVideoLayerElement: (videoLayerElement?: HTMLDivElement) => void
-  setImageElement: (imageElement?: HTMLImageElement) => void
-  setVideoElement: (videoElement?: HTMLVideoElement) => void
-  setSegmentationElement: (segmentationElement?: HTMLCanvasElement) => void
-
+  setVideoElements: (videoElements?: HTMLVideoElement[]) => void
   processClickOnSegment: (data: InteractiveSegmenterResult) => void
   onClickOnSegmentationLayer: MouseEventHandler<HTMLDivElement> 
   
@@ -98,8 +121,14 @@ export type LatentEngineStore = {
   play: () => boolean
   pause: () => boolean
 
-  // a slow rendering function (async - might call a third party LLM)
-  runSimulationLoop: () => Promise<void>
+  // a slow simulation function (async - might call a third party LLM)
+  runVideoSimulationLoop: () => Promise<void>
+  
+  // a slow simulation function (async - might call a third party LLM)
+  runInterfaceSimulationLoop: () => Promise<void>
+  
+  // a slow simulation function (async - might call a third party LLM)
+  runEntitySimulationLoop: () => Promise<void>
   
   // a fast rendering function; whose sole role is to filter the component
   // list to put into the buffer the one that should be displayed

src/components/interface/latent-engine/{store → core}/useLatentEngine.ts

@@ -1,29 +1,43 @@
 
 import { create } from "zustand"
 
-import { ClapProject } from "@/lib/clap/types"
+import { ClapModel, ClapProject } from "@/lib/clap/types"
 import { newClap } from "@/lib/clap/newClap"
 import { sleep } from "@/lib/utils/sleep"
 // import { getSegmentationCanvas } from "@/lib/on-device-ai/getSegmentationCanvas"
 
-import { LatentEngineStore } from "../core/types"
+import { LatentEngineStore } from "./types"
 import { resolveSegments } from "../resolvers/resolveSegments"
-import { fetchLatentClap } from "../core/fetchLatentClap"
+import { fetchLatentClap } from "./generators/fetchLatentClap"
 import { dataUriToBlob } from "@/app/api/utils/dataUriToBlob"
 import { parseClap } from "@/lib/clap/parseClap"
 import { InteractiveSegmenterResult, MPMask } from "@mediapipe/tasks-vision"
 import { segmentFrame } from "@/lib/on-device-ai/segmentFrameOnClick"
-import { drawSegmentation } from "../core/drawSegmentation"
+import { drawSegmentation } from "../utils/canvas/drawSegmentation"
 import { filterImage } from "@/lib/on-device-ai/filterImage"
+import { getZIndexDepth } from "../utils/data/getZIndexDepth"
+import { getSegmentStartAt } from "../utils/data/getSegmentStartAt"
+import { getSegmentId } from "../utils/data/getSegmentId"
+import { getElementsSortedByStartAt } from "../utils/data/getElementsSortedByStartAt"
+import { getSegmentEndAt } from "../utils/data/getSegmentEndAt"
+import { getVideoPrompt } from "./prompts/getVideoPrompt"
+import { setZIndexDepthId } from "../utils/data/setZIndexDepth"
+import { setSegmentStartAt } from "../utils/data/setSegmentStartAt"
+import { setSegmentEndAt } from "../utils/data/setSegmentEndAt"
+import { setSegmentId } from "../utils/data/setSegmentId"
 
 export const useLatentEngine = create<LatentEngineStore>((set, get) => ({
-  width: 1024,
-  height: 576,
+  jwtToken: "",
+
+  width: 512,
+  height: 288,
 
   clap: newClap(),
   debug: true,
 
-  streamType: "static",
+  headless: false, // false by default
+
+  isLoop: false,
   isStatic: false,
   isLive: false,
   isInteractive: false,
@@ -37,36 +51,73 @@ export const useLatentEngine = create<LatentEngineStore>((set, get) => ({
   hasDisclaimer: true,
   hasPresentedDisclaimer: false,
 
-  simulationPromise: undefined,
-  simulationPending: false,
-  simulationStartedAt: performance.now(),
-  simulationEndedAt: performance.now(),
-  simulationDurationInMs: 0,
-  simulationVideoPlaybackFPS: 0,
-  simulationRenderingTimeFPS: 0,
+  videoSimulationPromise: undefined,
+  videoSimulationPending: false,
+  videoSimulationStartedAt: performance.now(),
+  videoSimulationEndedAt: performance.now(),
+  videoSimulationDurationInMs: 0,
+  videoSimulationVideoPlaybackFPS: 0,
+  videoSimulationRenderingTimeFPS: 0,
+
+  interfaceSimulationPromise: undefined,
+  interfaceSimulationPending: false,
+  interfaceSimulationStartedAt: performance.now(),
+  interfaceSimulationEndedAt: performance.now(),
+  interfaceSimulationDurationInMs: 0,
+
+  entitySimulationPromise: undefined,
+  entitySimulationPending: false,
+  entitySimulationStartedAt: performance.now(),
+  entitySimulationEndedAt: performance.now(),
+  entitySimulationDurationInMs: 0,
 
   renderingIntervalId: undefined,
-  renderingIntervalDelayInMs: 2000, // 2 sec
-  
-  positionInMs: 0,
-  durationInMs: 0,
+  renderingIntervalDelayInMs: 150, // 0.2s
+  renderingLastRenderAt: performance.now(),
 
-  videoLayerElement: undefined,
-  imageElement: undefined,
-  videoElement: undefined,
-  segmentationElement: undefined,
-  
-  videoLayer: undefined,
-  videoBuffer: "A",
-  videoBufferA: null,
-  videoBufferB: undefined,
+  // for our calculations to be correct
+  // those need to match the actual output from the API
+  // don't trust the parameters you send to the API,
+  // instead check the *actual* values with VLC!!
+  videoModelFPS: 24,
+  videoModelNumOfFrames: 60, // 80,
+  videoModelDurationInSec: 2.584,
 
-  segmentationLayer: undefined,
+  playbackSpeed: 1,
 
-  interfaceLayer: undefined,
-  interfaceBuffer: "A",
-  interfaceBufferA: undefined,
-  interfaceBufferB: undefined,
+  positionInMs: 0,
+  durationInMs: 0,
+
+  // this is the "buffer size"
+  videoLayers: [
+    {
+      id: "video-buffer-0",
+      element: null as unknown as JSX.Element,
+    },
+    {
+      id: "video-buffer-1",
+      element: null as unknown as JSX.Element,
+    },
+    /*
+    {
+      id: "video-buffer-2",
+      element: null as unknown as JSX.Element,
+    },
+    {
+      id: "video-buffer-3",
+      element: null as unknown as JSX.Element,
+    },
+    */
+  ],
+  videoElements: [],
+
+  interfaceLayers: [],
+
+  setJwtToken: (jwtToken: string) => {
+    set({
+      jwtToken
+    })
+  },
 
   setContainerDimension: ({ width, height }: { width: number; height: number }) => {
     set({
@@ -117,27 +168,24 @@ export const useLatentEngine = create<LatentEngineStore>((set, get) => ({
     }
 
     if (!clap) { return }
-
+ 
     set({
       clap,
       isLoading: false,
       isLoaded: true,
-      streamType: clap.meta.streamType,
-      isStatic: clap.meta.streamType !== "interactive" && clap.meta.streamType !== "live",
-      isLive: clap.meta.streamType === "live",
-      isInteractive: clap.meta.streamType === "interactive",
+      isLoop: clap.meta.isLoop,
+      isStatic: !clap.meta.isInteractive,
+      isLive: false,
+      isInteractive: clap.meta.isInteractive,
     })
   },
 
-  setVideoLayerElement: (videoLayerElement?: HTMLDivElement) => { set({ videoLayerElement }) },
-  setImageElement: (imageElement?: HTMLImageElement) => { set({ imageElement }) },
-  setVideoElement: (videoElement?: HTMLVideoElement) => { set({ videoElement }) },
-  setSegmentationElement: (segmentationElement?: HTMLCanvasElement) => { set({ segmentationElement }) },
+  setVideoElements: (videoElements: HTMLVideoElement[] = []) => { set({ videoElements }) },
 
   processClickOnSegment: (result: InteractiveSegmenterResult) => {
     console.log(`processClickOnSegment: user clicked on something:`, result)
 
-    const { videoElement, imageElement, segmentationElement, debug } = get()
+    const { videoElements, debug } = get()
 
     if (!result?.categoryMask) {
       if (debug) {
@@ -151,10 +199,20 @@ export const useLatentEngine = create<LatentEngineStore>((set, get) => ({
         console.log(`processClickOnSegment: callling drawSegmentation`)
       }
 
+      const firstVisibleVideo = videoElements.find(element => 
+        getZIndexDepth(element) > 0
+      )
+
+      const segmentationElements = Array.from(
+        document.querySelectorAll('.segmentation-canvas')
+      ) as HTMLCanvasElement[]
+
+      const segmentationElement = segmentationElements.at(0)
+   
       const canvasMask: HTMLCanvasElement = drawSegmentation({
         mask: result.categoryMask,
         canvas: segmentationElement,
-        backgroundImage: imageElement,
+        backgroundImage: firstVisibleVideo,
         fillStyle: "rgba(255, 255, 255, 1.0)"
       })
       // TODO: read the canvas te determine on what the user clicked
@@ -174,12 +232,15 @@ export const useLatentEngine = create<LatentEngineStore>((set, get) => ({
   },
   onClickOnSegmentationLayer: (event) => {
 
-    const { videoElement, imageElement, segmentationLayer, segmentationElement, debug } = get()
+    const { videoElements, debug } = get()
     if (debug) {
       console.log("onClickOnSegmentationLayer")
     }
-    // TODO use the videoElement if this is is video!
-    if (!videoElement) { return }
+
+    const firstVisibleVideo = videoElements.find(element => 
+      getZIndexDepth(element) > 0
+    )
+    if (!firstVisibleVideo) { return }
 
     const box = event.currentTarget.getBoundingClientRect()
 
@@ -188,27 +249,36 @@ export const useLatentEngine = create<LatentEngineStore>((set, get) => ({
 
     const x = px / box.width
     const y = py / box.height
-    console.log(`onClickOnSegmentationLayer: user clicked on `, { x, y, px, py, box, videoElement })
+    console.log(`onClickOnSegmentationLayer: user clicked on `, { x, y, px, py, box, videoElements })
 
     const fn = async () => {
-      const results: InteractiveSegmenterResult = await segmentFrame(videoElement, x, y)
+      // todo julian: this should use the visible element instead
+      const results: InteractiveSegmenterResult = await segmentFrame(firstVisibleVideo, x, y)
       get().processClickOnSegment(results)
     }
     fn()
   },
   
   togglePlayPause: (): boolean => {
-    const { isLoaded, isPlaying, renderingIntervalId, videoElement } = get()
+    const { isLoaded, isPlaying, playbackSpeed, renderingIntervalId, videoElements } = get()
     if (!isLoaded) { return false }
     
     const newValue = !isPlaying
 
     clearInterval(renderingIntervalId)
 
+    const firstVisibleVideo = videoElements.find(element => 
+      getZIndexDepth(element) > 0
+    )
+
+    // Note Julian: we could also let the background scheduler
+    // (runRenderingLoop) do its work of advancing the cursor here
+
     if (newValue) {
-      if (videoElement) {
+      if (firstVisibleVideo) {
         try {
-          videoElement.play()
+          firstVisibleVideo.playbackRate = playbackSpeed
+          firstVisibleVideo.play()
         } catch (err) {
           console.error(`togglePlayPause: failed to start the video (${err})`)
         }
@@ -218,9 +288,10 @@ export const useLatentEngine = create<LatentEngineStore>((set, get) => ({
         renderingIntervalId: setTimeout(() => { get().runRenderingLoop() }, 0)
       })
     } else {
-      if (videoElement) {
+      if (firstVisibleVideo) {
         try {
-          videoElement.pause()
+          firstVisibleVideo.playbackRate = playbackSpeed
+          firstVisibleVideo.pause()
         } catch (err) {
           console.error(`togglePlayPause: failed to pause the video (${err})`)
         }
@@ -260,129 +331,254 @@ export const useLatentEngine = create<LatentEngineStore>((set, get) => ({
   },
 
   // a slow rendering function (async - might call a third party LLM)
-  runSimulationLoop: async () => {
+  runVideoSimulationLoop: async () => {
     const {
       isLoaded,
       isPlaying,
       clap,
-      segmentationLayer,
-      imageElement,
-      videoElement,
-      height,
-      width,
+      playbackSpeed,
+      positionInMs,
+      videoModelFPS,
+      videoModelNumOfFrames,
+      videoModelDurationInSec,
+      videoElements,
+      jwtToken,
     } = get()
 
     if (!isLoaded || !isPlaying) {
-        
-      set({
-        simulationPending: false,
-      })
-
+      set({ videoSimulationPending: false })
       return
     }
 
     set({
-      simulationPending: true,
-      simulationStartedAt: performance.now(),
+      videoSimulationPending: true,
+      videoSimulationStartedAt: performance.now(),
     })
 
-    try {
+    const videosSortedByStartAt = getElementsSortedByStartAt(videoElements)
 
-      /*
-      // console.log("doing stuff")
-      let timestamp = performance.now()
-
-      if (imageElement) {
-        // console.log("we have an image element:", imageElement)
-        const segmentationLayer = await getSegmentationCanvas({
-          frame: imageElement,
-          timestamp,
-          width,
-          height,
-        })
-        set({ segmentationLayer })
+    // videos whose timestamp is behind the current cursor
+    let toRecycle: HTMLVideoElement[] = []
+    let toPlay: HTMLVideoElement[] = []
+    let toPreload: HTMLVideoElement[] = []
+    
+    for (let i = 0; i < videosSortedByStartAt.length; i++) {
+      const video = videosSortedByStartAt[i]
+      
+      const segmentStartAt = getSegmentStartAt(video)
+      const segmentEndAt = getSegmentEndAt(video)
+
+      // this segment has been spent, it should be discared
+      if (segmentEndAt < positionInMs) {
+        toRecycle.push(video)
+      } else if (segmentStartAt < positionInMs) {
+        toPlay.push(video)
+        video.play()
+        setZIndexDepthId(video, 10)
+      } else {
+        toPreload.push(video)
+        video.pause()
+        setZIndexDepthId(video, 0)
       }
-      */
-
-      // await sleep(500)
+    }
 
-      // note: since we are asynchronous, we need to regularly check if
-      // the user asked to pause the system or no
-      if (get().isPlaying) {
-        // console.log(`runSimulationLoop: rendering video content layer..`)
-        // we only grab the first one
+    const videoDurationInMs = videoModelDurationInSec * 1000
+
+    // TODO julian: this is an approximation
+    // to grab the max number of segments
+    const maxBufferDurationInMs = positionInMs + (videoDurationInMs * 4)
+    console.log(`DEBUG: `, {
+      positionInMs,
+      videoModelDurationInSec,
+      videoDurationInMs,
+      "(videoDurationInMs * 4)":  (videoDurationInMs * 4),
+      maxBufferDurationInMs,
+      segments: clap.segments
+    })
     
+    const prefilterSegmentsForPerformanceReasons = clap.segments.filter(s =>
+      s.startTimeInMs >= positionInMs &&
+      s.startTimeInMs < maxBufferDurationInMs
+    )
 
+    console.log(`prefilterSegmentsForPerformanceReasons: `, prefilterSegmentsForPerformanceReasons)
 
-        const videoLayer = (await resolveSegments(clap, "video", 1)).at(0)
+    // this tells us how much time is left
+    let remainingTimeInMs = Math.max(0, clap.meta.durationInMs - positionInMs)
+    // to avoid interruptions, we should jump to the beginning of the project
+    // as soo as we are start playing back the "last" video segment
 
-        if (get().isPlaying) {
+    // now, we need to recycle spent videos,
+    // by discarding their content and replacing it with fresh one
+    //
+    // yes: I know the code is complex and not intuitive - sorry about that
 
-          set({
-            videoLayer
-          })
+    // TODO Julian: use the Clap project to fill in those
+    const modelsById: Record<string, ClapModel> = {}
+    const extraPositivePrompt: string[] = []
 
-          const { videoElement, imageElement, segmentationElement } = get()
+    let bufferAheadOfCurrentPositionInMs = positionInMs
 
-          if (videoElement) {
-            // yes, it is a very a dirty trick
-            // yes, it will look back
-            videoElement.defaultPlaybackRate = 0.5
-          }
+    for (let i = 0; i < toRecycle.length; i++) {
+      console.log(`got a spent video to recycle`)
+      
+      // we select the segments in the current shot
 
-          const canvas = drawSegmentation({
-             // no mask means this will effectively clear the canvas
-            canvas: segmentationElement,
-            backgroundImage: imageElement,
-          })
-          
+      const shotSegmentsToPreload = prefilterSegmentsForPerformanceReasons.filter(s =>
+        s.startTimeInMs >= bufferAheadOfCurrentPositionInMs &&
+        s.startTimeInMs < (bufferAheadOfCurrentPositionInMs + videoDurationInMs)
+      )
+      
+      bufferAheadOfCurrentPositionInMs += videoDurationInMs
 
-          // console.log(`runSimulationLoop: rendered video content layer`)
-        }
-      }
+      const prompt = getVideoPrompt(shotSegmentsToPreload, modelsById, extraPositivePrompt)
 
-    } catch (err) {
-      console.error(`runSimulationLoop failed to render video layer ${err}`)
+      console.log(`video prompt: ${prompt}`)
+      // could also be the camera
+      // after all, we don't necessarily have a shot,
+      // this could also be a gaussian splat
+      const shotData = shotSegmentsToPreload.find(s => s.category === "video")
+
+      console.log(`shotData:`, shotData)
+
+      if (!prompt || !shotData) { continue }
+  
+      const recycled = toRecycle[i]
+
+      recycled.pause()
+
+      setSegmentId(recycled, shotData.id)
+      setSegmentStartAt(recycled, shotData.startTimeInMs)
+      setSegmentEndAt(recycled, shotData.endTimeInMs)
+      setZIndexDepthId(recycled, 0)
+
+      // this is the best compromise for now in term of speed
+      const width = 512
+      const height = 288
+
+      // this is our magic trick: we let the browser do the token-secured,
+      // asynchronous and parallel video generation call for us
+      //
+      // one issue with this approach is that it hopes the video
+      // will be downloaded in time, but it's not an exact science
+      //
+      // first, generation time varies between 4sec and 7sec,
+      // then some people will get 300ms latency due to their ISP,
+      // and finally the video itself is a 150~200 Kb payload)
+      recycled.src = `/api/resolvers/video?t=${
+
+      // to prevent funny people from using this as a free, open-bar video API
+      // we have this system of token with a 24h expiration date
+      // we might even make it tighter in the future
+        jwtToken
+      }&w=${
+        width
+
+      }&h=${
+        height
+      }&p=${
+        // let's re-use the best ideas from the Latent Browser:
+        // a text uri equals a latent resource
+        encodeURIComponent(prompt)
+      }`
+
+      toPreload.push(recycled)
+    }
+
+    const videoSimulationEndedAt = performance.now()
+    const videoSimulationDurationInMs = videoSimulationEndedAt - get().videoSimulationStartedAt 
+    const videoSimulationDurationInSec = videoSimulationDurationInMs / 1000
+
+    const videoSimulationVideoPlaybackFPS = videoModelFPS * playbackSpeed
+    const videoSimulationRenderingTimeFPS = videoModelNumOfFrames / videoSimulationDurationInSec
+    set({
+      videoSimulationPending: false,
+      videoSimulationEndedAt,
+      videoSimulationDurationInMs,
+      videoSimulationVideoPlaybackFPS,
+      videoSimulationRenderingTimeFPS,
+    })
+  },
+
+
+  // a slow rendering function (async - might call a third party LLM)
+  runInterfaceSimulationLoop: async () => {
+    const {
+      isLoaded,
+      isPlaying,
+      clap,
+    } = get()
+
+    if (!isLoaded || !isPlaying) {
+      set({ interfaceSimulationPending: false })
+      return
     }
 
+    set({
+      interfaceSimulationPending: true,
+      interfaceSimulationStartedAt: performance.now(),
+    })
+ 
     try {
       if (get().isPlaying) {
         // console.log(`runSimulationLoop: rendering UI layer..`)
 
-        // note: for now we only display one element, to avoid handing a list of html elements
-        const interfaceLayer = (await resolveSegments(clap, "interface", 1)).at(0)
+        // note: for now we only display one panel at a time,
+        // later we can try to see if we should handle more
+        // for nice gradient transition,
+        const interfaceLayers = await resolveSegments(clap, "interface", 1)
+
         if (get().isPlaying) {
           set({
-            interfaceLayer
+            interfaceLayers
           })
 
           // console.log(`runSimulationLoop: rendered UI layer`)
         }
       }
     } catch (err) {
-      console.error(`runSimulationLoop failed to render UI layer ${err}`)
+      console.error(`runInterfaceSimulationLoop failed to render UI layer ${err}`)
     }
 
-    const simulationEndedAt = performance.now()
-    const simulationDurationInMs = simulationEndedAt - get().simulationStartedAt 
-    const simulationDurationInSec =simulationDurationInMs / 1000
-
-    // I've counted the frames manually, and we indeed have, in term of pure video playback,
-    // 10 fps divided by 2 (the 0.5 playback factor)
-    const videoFPS = 10
-    const videoDurationInSec = 1
-    const videoPlaybackSpeed = 0.5
-    const simulationVideoPlaybackFPS = videoDurationInSec * videoFPS * videoPlaybackSpeed
-    const simulationRenderingTimeFPS = (videoDurationInSec * videoFPS) / simulationDurationInSec
+    const interfaceSimulationEndedAt = performance.now()
+    const interfaceSimulationDurationInMs = interfaceSimulationEndedAt - get().interfaceSimulationStartedAt
+
     set({
-      simulationPending: false,
-      simulationEndedAt,
-      simulationDurationInMs,
-      simulationVideoPlaybackFPS,
-      simulationRenderingTimeFPS,
+      interfaceSimulationPending: false,
+      interfaceSimulationEndedAt,
+      interfaceSimulationDurationInMs,
     })
   },
 
+
+  // a slow rendering function (async - might call a third party LLM)
+  runEntitySimulationLoop: async () => {
+    const {
+      isLoaded,
+      isPlaying,
+      clap,
+    } = get()
+
+
+    if (!isLoaded || !isPlaying) {
+      set({ entitySimulationPending: false })
+      return
+    }
+
+    set({
+      entitySimulationPending: true,
+      entitySimulationStartedAt: performance.now(),
+    })
+
+    const entitySimulationEndedAt = performance.now()
+    const entitySimulationDurationInMs = entitySimulationEndedAt - get().entitySimulationStartedAt 
+
+    set({
+      entitySimulationPending: false,
+      entitySimulationEndedAt,
+      entitySimulationDurationInMs,
+    })
+  },
   // a fast sync rendering function; whose sole role is to filter the component
   // list to put into the buffer the one that should be displayed
   runRenderingLoop: () => {
@@ -391,32 +587,52 @@ export const useLatentEngine = create<LatentEngineStore>((set, get) => ({
       isPlaying,
       renderingIntervalId,
       renderingIntervalDelayInMs,
-      simulationPromise,
-      simulationPending,
-      runSimulationLoop,
-      imageElement,
-      videoElement,
+      renderingLastRenderAt,
+      positionInMs,
+      videoSimulationPending,
+      runVideoSimulationLoop,
+      interfaceSimulationPending,
+      runInterfaceSimulationLoop,
+      entitySimulationPending,
+      runEntitySimulationLoop,
     } = get()
-    if (!isLoaded) { return }
-    if (!isPlaying) { return }
-    try {
-      // console.log(`runRenderingLoop: starting..`)
-    
-      // TODO: some operations with
-      // console.log(`runRenderingLoop: ended`)
-    } catch (err) {
-      console.error(`runRenderingLoop failed ${err}`)
-    }
+    if (!isLoaded || !isPlaying) { return }
+
+    // TODO julian: don't do this here, this is inneficient
+    const videoElements = Array.from(
+      document.querySelectorAll('.video-buffer')
+    ) as HTMLVideoElement[]
+ 
+    const newRenderingLastRenderAt = performance.now()
+    const elapsedInMs = newRenderingLastRenderAt - renderingLastRenderAt
+
+    // let's move inside the Clap file timeline
+    const newPositionInMs = positionInMs + elapsedInMs
+
     clearInterval(renderingIntervalId)
+
     set({
       isPlaying: true,
-      simulationPromise: simulationPending ? simulationPromise : runSimulationLoop(),
+      renderingLastRenderAt: newRenderingLastRenderAt,
+      positionInMs: newPositionInMs,
 
+      videoElements: videoElements,
       // TODO: use requestAnimationFrame somehow
       // https://developers.google.com/mediapipe/solutions/vision/image_segmenter/web_js
       renderingIntervalId: setTimeout(() => { get().runRenderingLoop() }, renderingIntervalDelayInMs)
     })
-    
+
+    // note that having this second set() also helps us to make sure previously values are properly stored
+    // in the state when the simulation loop runs
+    if (!videoSimulationPending) {
+      set({ videoSimulationPromise: runVideoSimulationLoop() }) // <-- note: this is a fire-and-forget operation!
+    }
+    if (!interfaceSimulationPending) {
+      set({ interfaceSimulationPromise: runInterfaceSimulationLoop() }) // <-- note: this is a fire-and-forget operation!
+    }
+    if (!entitySimulationPending) {
+      set({ entitySimulationPromise: runEntitySimulationLoop() }) // <-- note: this is a fire-and-forget operation!
+    }
   },
 
   jumpTo: (positionInMs: number) => {

src/components/interface/latent-engine/core/video-buffer.tsx

@@ -0,0 +1,57 @@
+import { useEffect, useRef, useState } from "react"
+import { v4 as uuidv4 } from "uuid"
+
+import { cn } from "@/lib/utils/cn"
+
+import { LayerElement } from "./types"
+
+export function VideoBuffer({
+  bufferSize = 4,
+  className = "",
+  width = 512,
+  height = 256,
+}: {
+  bufferSize?: number
+  className?: string
+  width?: number
+  height?: number
+}) {
+  const state = useRef<{
+    isInitialized: boolean
+  }>({
+    isInitialized: false,
+  })
+
+  const [layers, setLayers] = useState<LayerElement[]>([])
+  
+  // this initialize the VideoBuffer and keeps the layers in sync with the bufferSize
+  useEffect(() => {
+    if (layers?.length !== bufferSize) {
+      const newLayers: LayerElement[] = []
+      for (let i = 0; i < bufferSize; i++) {
+        newLayers.push({
+          id: uuidv4(),
+          element: <></>
+        })
+      }
+      setLayers(newLayers)
+    }
+  }, [bufferSize, layers?.length])
+
+  return (
+    <div
+      className={cn(className)}
+      style={{
+        width,
+        height
+      }}>
+    {layers.map(({ id, element }) => (
+      <div
+        key={id}
+        id={id}
+        style={{ width, height }}
+        // className={`video-buffer-layer video-buffer-layer-${id}`}
+      >{element}</div>))}
+    </div>
+  )
+}

src/components/interface/latent-engine/resolvers/generic/index.tsx

@@ -1,11 +1,14 @@
 "use client"
 
+import { v4 as uuidv4 } from "uuid"
+
 import { ClapProject, ClapSegment } from "@/lib/clap/types"
 
-export async function resolve(segment: ClapSegment, clap: ClapProject): Promise<JSX.Element> {
-  return (
-    <div
-      className="w-full h-full"
-    />
-  )
+import { LayerElement } from "../../core/types"
+
+export async function resolve(segment: ClapSegment, clap: ClapProject): Promise<LayerElement> {
+  return {
+    id: uuidv4(),
+    element: <div className="w-full h-full" />
+  }
 }

src/components/interface/latent-engine/resolvers/image/generateImage.ts

@@ -1,5 +1,24 @@
-export async function generateImage(prompt: string): Promise<string> {
-  const requestUri = `/api/resolvers/image?p=${encodeURIComponent(prompt)}`
+export async function generateImage({
+  prompt,
+  width,
+  height,
+  token,
+}: {
+  prompt: string
+  width: number
+  height: number
+  token: string
+}): Promise<string> {
+  const requestUri = `/api/resolvers/image?t=${
+    token
+  }&w=${
+    width
+  }&h=${
+    height
+
+  }&p=${
+    encodeURIComponent(prompt)
+  }`
   const res = await fetch(requestUri)
   const blob = await res.blob()
   const url = URL.createObjectURL(blob)

src/components/interface/latent-engine/resolvers/image/index.tsx

@@ -2,8 +2,10 @@
 
 import { ClapProject, ClapSegment } from "@/lib/clap/types"
 import { generateImage } from "./generateImage"
+import { LayerElement } from "../../core/types"
+import { useStore } from "@/app/state/useStore"
 
-export async function resolve(segment: ClapSegment, clap: ClapProject): Promise<JSX.Element> {
+export async function resolve(segment: ClapSegment, clap: ClapProject): Promise<LayerElement> {
 
   const { prompt } = segment
 
@@ -11,21 +13,30 @@ export async function resolve(segment: ClapSegment, clap: ClapProject): Promise<
   try {
     // console.log(`resolveImage: generating video for: ${prompt}`)
 
-    assetUrl = await generateImage(prompt)
+    assetUrl = await generateImage({
+      prompt,
+      width: clap.meta.width,
+      height: clap.meta.height,
+      token: useStore.getState().jwtToken,
+    })
 
     // console.log(`resolveImage: generated ${assetUrl}`)
 
   } catch (err) {
     console.error(`resolveImage failed (${err})`)
-    return <></>
+    return {
+      id: segment.id,
+      element: <></>
+    }
   }
 
   // note: the latent-image class is not used for styling, but to grab the component
   // from JS when we need to segment etc
-  return (
-    <img
+  return {
+    id: segment.id,
+    element: <img
       className="latent-image object-cover h-full"
       src={assetUrl}
     />
-  )
+  }
 }

src/components/interface/latent-engine/resolvers/interface/index.tsx

@@ -4,19 +4,31 @@ import RunCSS, { extendRunCSS } from "runcss"
 
 import { ClapProject, ClapSegment } from "@/lib/clap/types"
 import { generateHtml } from "./generateHtml"
-import { ThisIsAI } from "../../components/disclaimers/this-is-ai"
+import { AIContentDisclaimer } from "../../components/intros/ai-content-disclaimer"
+import { LayerElement } from "../../core/types"
+import { PoweredBy } from "../../components/intros/powered-by"
 
 let state = {
   runCSS: RunCSS({}),
   isWatching: false,
 }
 
-export async function resolve(segment: ClapSegment, clap: ClapProject): Promise<JSX.Element> {
+export async function resolve(segment: ClapSegment, clap: ClapProject): Promise<LayerElement> {
 
   const { prompt } = segment
 
-  if (prompt.toLowerCase() === "<builtin:disclaimer>") {
-    return <ThisIsAI streamType={clap.meta.streamType} />
+  if (prompt.toLowerCase() === "<builtin:powered_by_engine>") {
+    return {
+      id: segment.id,
+      element: <PoweredBy />
+    }
+  }
+
+  if (prompt.toLowerCase() === "<builtin:disclaimer_about_ai>") {
+    return {
+      id: segment.id,
+      element: <AIContentDisclaimer isInteractive={clap.meta.isInteractive} />
+    }
   }
 
   let dangerousHtml = ""
@@ -48,10 +60,11 @@ export async function resolve(segment: ClapSegment, clap: ClapProject): Promise<
     // startWatching(targetNode)
   }
 
-  return (
-    <div
+ return {
+    id: segment.id,
+    element: <div
       className="w-full h-full"
       dangerouslySetInnerHTML={{ __html: dangerousHtml }}
     />
-  )
+  }
 }

src/components/interface/latent-engine/resolvers/resolveSegment.ts

@@ -1,13 +1,13 @@
 import { ClapProject, ClapSegment } from "@/lib/clap/types"
 
-import { LatentComponentResolver } from "../core/types"
+import { LatentComponentResolver, LayerElement } from "../core/types"
 
 import { resolve as genericResolver } from "./generic"
 import { resolve as interfaceResolver } from "./interface"
 import { resolve as videoResolver } from "./video"
 import { resolve as imageResolver } from "./image"
 
-export async function resolveSegment(segment: ClapSegment, clap: ClapProject): Promise<JSX.Element> {
+export async function resolveSegment(segment: ClapSegment, clap: ClapProject): Promise<LayerElement> {
   let latentComponentResolver: LatentComponentResolver = genericResolver
   
   if (segment.category === "interface") {

src/components/interface/latent-engine/resolvers/resolveSegments.ts

@@ -1,17 +1,17 @@
 import { ClapProject, ClapSegmentCategory } from "@/lib/clap/types"
 
 import { resolveSegment } from "./resolveSegment"
+import { LayerElement } from "../core/types"
 
 export async function resolveSegments(
   clap: ClapProject,
   segmentCategory: ClapSegmentCategory,
   nbMax?: number
-) : Promise<JSX.Element[]> {
-  const elements: JSX.Element[] = await Promise.all(
+) : Promise<LayerElement[]> {
+  return Promise.all(
     clap.segments
     .filter(s => s.category === segmentCategory)
     .slice(0, nbMax)
     .map(s => resolveSegment(s, clap))
   )
-  return elements
 }

src/components/interface/latent-engine/resolvers/video/basic-video.tsx

@@ -0,0 +1,47 @@
+
+import React, { useEffect, useRef } from 'react';
+
+export function BasicVideo({
+  className = "",
+  src,
+  playbackSpeed = 1.0,
+  playsInline = true,
+  muted = true,
+  autoPlay = true,
+  loop = true,
+}: {
+  className?: string
+  src?: string
+  playbackSpeed?: number
+  playsInline?: boolean
+  muted?: boolean
+  autoPlay?: boolean
+  loop?: boolean
+}) {
+  const videoRef = useRef<HTMLVideoElement>(null)
+
+  // Setup and handle changing playback rate and video source
+  useEffect(() => {
+    if (videoRef.current) {
+      videoRef.current.playbackRate = playbackSpeed;
+    }
+  }, [videoRef.current, playbackSpeed]);
+
+
+  // Handle UI case for empty playlists
+  if (!src || typeof src !== "string") {
+    return <></>
+  }
+
+  return (
+    <video
+      ref={videoRef}
+      className={className}
+      playsInline={playsInline}
+      muted={muted}
+      autoPlay={autoPlay}
+      loop={loop}
+      src={src}
+    />
+  );
+};

src/components/interface/latent-engine/resolvers/video/generateVideo.ts

@@ -1,8 +1,25 @@
-export async function generateVideo(prompt: string): Promise<string> {
-  const requestUri = `/api/resolvers/video?p=${encodeURIComponent(prompt)}`
+export async function generateVideo({
+  prompt,
+  width,
+  height,
+  token,
+}: {
+  prompt: string
+  width: number
+  height: number
+  token: string
+}): Promise<string> {
+  const requestUri = `/api/resolvers/video?t=${
+    token
+  }&w=${
+    width
+  }&h=${
+    height
+  }&p=${
+    encodeURIComponent(prompt)
+  }`
   const res = await fetch(requestUri)
   const blob = await res.blob()
   const url = URL.createObjectURL(blob)
   return url
-
 }

src/components/interface/latent-engine/resolvers/video/index.tsx

@@ -1,40 +1,47 @@
 "use client"
 
 import { ClapProject, ClapSegment } from "@/lib/clap/types"
+
+import { LayerElement } from "../../core/types"
+
 import { generateVideo } from "./generateVideo"
+import { BasicVideo } from "./basic-video"
+import { useStore } from "@/app/state/useStore"
 
-export async function resolve(segment: ClapSegment, clap: ClapProject): Promise<JSX.Element> {
+export async function resolve(segment: ClapSegment, clap: ClapProject): Promise<LayerElement> {
 
   const { prompt } = segment
 
-  let assetUrl = ""
-  try {
-    // console.log(`resolveVideo: generating video for: ${prompt}`)
-
-    assetUrl = await generateVideo(prompt)
+  let src: string = ""
 
+  try {
+    src = await generateVideo({
+      prompt,
+      width: clap.meta.width,
+      height: clap.meta.height,
+      token: useStore.getState().jwtToken,
+    })
     // console.log(`resolveVideo: generated ${assetUrl}`)
 
   } catch (err) {
-    console.error(`resolveVideo failed (${err})`)
-    return <></>
+    console.error(`resolveVideo failed: ${err}`)
+    return {
+      id: segment.id,
+      element: <></>
+    }
   }
 
   // note: the latent-video class is not used for styling, but to grab the component
   // from JS when we need to segment etc
-  return (
-    <video
-      loop
+  return {
+    id: segment.id,
+    element: <BasicVideo
       className="latent-video object-cover h-full"
+      src={src}
       playsInline
-
-      // muted needs to be enabled for iOS to properly autoplay
       muted
       autoPlay
-
-      // we hide the controls
-      // controls
-      src={assetUrl}>
-    </video>
-  )
+      loop
+    />
+  }
 }

src/components/interface/latent-engine/resolvers/video/index_legacy.tsx

@@ -0,0 +1,86 @@
+"use client"
+
+import { ClapProject, ClapSegment } from "@/lib/clap/types"
+import { PromiseResponseType, waitPromisesUntil } from "@/lib/utils/waitPromisesUntil"
+import { generateVideo } from "./generateVideo"
+import { BasicVideo } from "./basic-video"
+import { useStore } from "@/app/state/useStore"
+
+export async function resolve(segment: ClapSegment, clap: ClapProject): Promise<JSX.Element> {
+
+  const { prompt } = segment
+
+
+  // this is were the magic happen, and we create our buffer of N videos
+  // we need to adopt a conservative approach, ideally no more than 3 videos in parallel per user
+  const numberOfParallelRequests = 3
+
+  // the playback speed is the second trick:
+  // it allows us to get more "value" per video (a run time of 2 sec instead of 1)
+  // at the expense of a more sluggish appearance (10 fps -> 5 fps, if you pick a speed of 0.5)
+  const playbackSpeed = 0.5
+
+  // running multiple requests in parallel increase the risk that a server is down
+  // we also cannot afford to wait for all videos for too long as this increase latency,
+  // so we should keep a tight execution window
+
+   // with current resolution and step settings,
+   // we achieve 3463.125 rendering time on a A10 Large
+   const maxRenderingTimeForAllVideos = 5000
+
+  // this is how long we wait after we received our first video
+  // this represents the variability between rendering time
+  //
+  // this parameters helps us "squeeze" the timeout,
+  // if the hardware settings changed for instance
+  // this is a way to say "how, this is faster than I expected, other videos should be fast too then"
+  //
+  // I've noticed it is between 0.5s and 1s with current settings
+  // so let's a a slightly larger value
+  const maxWaitTimeAfterFirstVideo = 1500
+
+  let playlist: string[] = []
+
+  try {
+    // console.log(`resolveVideo: generating video for: ${prompt}`)
+    const promises: Array<Promise<string>> = []
+
+    for (let i = 0; i < numberOfParallelRequests; i++) {
+      // TODO use the Clap segments instead to bufferize the next scenes,
+      // otherwise we just clone the current segment, which is not very interesting
+      promises.push(generateVideo({
+        prompt,
+        width: clap.meta.width,
+        height: clap.meta.height,
+        token: useStore.getState().jwtToken,
+      }))
+    }
+
+    const results = await waitPromisesUntil(promises, maxWaitTimeAfterFirstVideo, maxRenderingTimeForAllVideos)
+    
+    playlist = results
+      .filter(result => result?.status === PromiseResponseType.Resolved && typeof result?.value === "string")
+      .map(result => result?.value || "")
+
+
+    // console.log(`resolveVideo: generated ${assetUrl}`)
+
+  } catch (err) {
+    console.error(`resolveVideo failed: ${err}`)
+    return <></>
+  }
+
+  // note: the latent-video class is not used for styling, but to grab the component
+  // from JS when we need to segment etc
+  return (
+    <BasicVideo
+      className="latent-video object-cover h-full"
+      playbackSpeed={playbackSpeed}
+      src={playlist[0]}
+      playsInline
+      muted
+      autoPlay
+      loop
+    />
+  )
+}

src/components/interface/latent-engine/resolvers/video/index_notSoGood.tsx

@@ -0,0 +1,82 @@
+"use client"
+
+import { ClapProject, ClapSegment } from "@/lib/clap/types"
+import { PromiseResponseType, waitPromisesUntil } from "@/lib/utils/waitPromisesUntil"
+import { VideoLoop } from "./video-loop"
+import { generateVideo } from "./generateVideo"
+import { useStore } from "@/app/state/useStore"
+
+export async function resolve(segment: ClapSegment, clap: ClapProject): Promise<JSX.Element> {
+
+  const { prompt } = segment
+
+
+  // this is were the magic happen, and we create our buffer of N videos
+  // we need to adopt a conservative approach, ideally no more than 3 videos in parallel per user
+  const numberOfParallelRequests = 3
+
+  // the playback speed is the second trick:
+  // it allows us to get more "value" per video (a run time of 2 sec instead of 1)
+  // at the expense of a more sluggish appearance (10 fps -> 5 fps, if you pick a speed of 0.5)
+  const playbackSpeed = 0.5
+
+  // running multiple requests in parallel increase the risk that a server is down
+  // we also cannot afford to wait for all videos for too long as this increase latency,
+  // so we should keep a tight execution window
+
+   // with current resolution and step settings,
+   // we achieve 3463.125 rendering time on a A10 Large
+   const maxRenderingTimeForAllVideos = 5000
+
+  // this is how long we wait after we received our first video
+  // this represents the variability between rendering time
+  //
+  // this parameters helps us "squeeze" the timeout,
+  // if the hardware settings changed for instance
+  // this is a way to say "how, this is faster than I expected, other videos should be fast too then"
+  //
+  // I've noticed it is between 0.5s and 1s with current settings
+  // so let's a a slightly larger value
+  const maxWaitTimeAfterFirstVideo = 1500
+
+  let playlist: string[] = []
+
+  try {
+    // console.log(`resolveVideo: generating video for: ${prompt}`)
+    const promises: Array<Promise<string>> = []
+
+    for (let i = 0; i < numberOfParallelRequests; i++) {
+      // TODO use the Clap segments instead to bufferize the next scenes,
+      // otherwise we just clone the current segment, which is not very interesting
+      promises.push(generateVideo({
+        prompt,
+        width: clap.meta.width,
+        height: clap.meta.height,
+        token: useStore.getState().jwtToken,
+      }))
+    }
+
+    const results = await waitPromisesUntil(promises, maxWaitTimeAfterFirstVideo, maxRenderingTimeForAllVideos)
+    
+    playlist = results
+      .filter(result => result?.status === PromiseResponseType.Resolved && typeof result?.value === "string")
+      .map(result => result?.value || "")
+
+
+    // console.log(`resolveVideo: generated ${assetUrl}`)
+
+  } catch (err) {
+    console.error(`resolveVideo failed: ${err}`)
+    return <></>
+  }
+
+  // note: the latent-video class is not used for styling, but to grab the component
+  // from JS when we need to segment etc
+  return (
+    <VideoLoop
+      className="latent-video object-cover h-full"
+      playbackSpeed={playbackSpeed}
+      playlist={playlist}
+    />
+  )
+}

src/components/interface/latent-engine/resolvers/video/video-loop.tsx

@@ -0,0 +1,83 @@
+
+import React, { useState, useEffect, useRef } from 'react';
+
+interface VideoLoopProps {
+  className?: string;
+  playlist?: string[];
+  playbackSpeed?: number;
+}
+
+export const VideoLoop: React.FC<VideoLoopProps> = ({
+  className = "",
+  playlist = [],
+  playbackSpeed = 1.0
+}) => {
+  const [currentVideoIndex, setCurrentVideoIndex] = useState(0);
+  const videoRef = useRef<HTMLVideoElement>(null);
+
+  const handleVideoEnd = () => {
+    // Loop only if there is more than one video
+    if (playlist.length > 1) {
+      setCurrentVideoIndex(prevIndex => (prevIndex + 1) % playlist.length);
+    }
+  };
+
+  // Setup and handle changing playback rate and video source
+  useEffect(() => {
+    if (videoRef.current) {
+      videoRef.current.playbackRate = playbackSpeed;
+      videoRef.current.src = playlist[currentVideoIndex] || ''; // Resort to empty string if undefined
+      videoRef.current.load();
+      if (videoRef.current.src) {
+        videoRef.current.play().catch(error => {
+          console.error('Video play failed', error);
+        });
+      } else {
+        console.log("VideoLoop: cannot start (no video)")
+      }
+    }
+  }, [playbackSpeed, currentVideoIndex, playlist]);
+
+  // Handle native video controls interaction
+  useEffect(() => {
+    const videoElement = videoRef.current;
+    if (!videoElement || playlist.length === 0) return;
+
+    const handlePlay = () => {
+      if (videoElement.paused && !videoElement.ended) {
+        if (videoRef.current?.src) {
+          videoElement.play().catch((error) => {
+            console.error('Error playing the video', error);
+          });
+        } else {
+          console.log("VideoLoop: cannot start (no video)")
+        }
+      }
+    };
+
+    videoElement.addEventListener('play', handlePlay);
+    videoElement.addEventListener('ended', handleVideoEnd);
+
+    return () => {
+      videoElement.removeEventListener('play', handlePlay);
+      videoElement.removeEventListener('ended', handleVideoEnd);
+    };
+  }, [playlist]);
+
+  // Handle UI case for empty playlists
+  if (playlist.length === 0 || !playlist[currentVideoIndex]) {
+    return <></>
+  }
+
+  return (
+    <video
+      ref={videoRef}
+      loop={false}
+      className={className}
+      playsInline
+      muted
+      autoPlay
+      src={playlist[currentVideoIndex]}
+    />
+  );
+};

src/components/interface/latent-engine/{core → utils/canvas}/drawSegmentation.ts

@@ -3,7 +3,7 @@ import { MPMask } from "@mediapipe/tasks-vision";
 interface DrawSegmentationOptions {
   mask?: MPMask;
   canvas?: HTMLCanvasElement;
-  backgroundImage?: HTMLImageElement;
+  backgroundImage?: HTMLVideoElement | HTMLImageElement;
   fillStyle?: string;
 }
 

src/components/interface/latent-engine/utils/data/getElementsSortedByStartAt.ts

@@ -0,0 +1,13 @@
+import { getSegmentStartAt } from "./getSegmentStartAt"
+
+export function getElementsSortedByStartAt<T extends HTMLElement>(elements: T[], createCopy = true): T[] {
+  
+  const array = createCopy ? [...elements]:  elements
+
+  // this sort from the smallest (oldest) to biggest (youngest)
+  return array.sort((a, b) => {
+    const aSegmentStartAt = getSegmentStartAt(a)
+    const bSegmentStartAt = getSegmentStartAt(b)
+    return aSegmentStartAt - bSegmentStartAt
+  })
+}

src/components/interface/latent-engine/utils/data/getSegmentEndAt.ts

@@ -0,0 +1,3 @@
+export function getSegmentEndAt(element: HTMLElement, defaultValue = 0): number {
+  return Number(element.getAttribute('data-segment-end-at') || defaultValue)
+}

src/components/interface/latent-engine/utils/data/getSegmentId.ts

@@ -0,0 +1,3 @@
+export function getSegmentId(element: HTMLElement, defaultValue = ""): string {
+  return element.getAttribute('data-segment-id') || defaultValue
+}

src/components/interface/latent-engine/utils/data/getSegmentStartAt.ts

@@ -0,0 +1,3 @@
+export function getSegmentStartAt(element: HTMLElement, defaultValue = 0): number {
+  return Number(element.getAttribute('data-segment-start-at') || defaultValue)
+}

src/components/interface/latent-engine/utils/data/getZIndexDepth.ts

@@ -0,0 +1,3 @@
+export function getZIndexDepth(element: HTMLElement, defaultValue = 0) {
+  return Number(element.getAttribute('data-z-index-depth') || 0)
+}

src/components/interface/latent-engine/utils/data/setSegmentEndAt.ts

@@ -0,0 +1,3 @@
+export function setSegmentEndAt(element: HTMLElement, value = 0): void {
+  return element.setAttribute('data-segment-end-at', `${value || "0"}`)
+}

src/components/interface/latent-engine/utils/data/setSegmentId.ts

@@ -0,0 +1,3 @@
+export function setSegmentId(element: HTMLElement, value = ""): void {
+  return element.setAttribute('data-segment-id', value)
+}