add experimental support for 360° videos

src/app/interface/equirectangular-video-player/index.tsx

@@ -0,0 +1,40 @@
+"use client"
+
+import AutoSizer from "react-virtualized-auto-sizer"
+
+import { cn } from "@/lib/utils"
+import { VideoInfo } from "@/types"
+
+import { VideoSphereViewer } from "./viewer"
+
+export function EquirectangularVideoPlayer({
+  video,
+  className = "",
+ }: {
+  video?: VideoInfo
+  className?: string
+}) {
+  // we shield the VideeoSphere viewer from bad data
+  if (!video?.assetUrl) { return null }
+
+  return (
+    <div
+      className={cn(
+        `w-full`,
+        // note: for AutoSizer to work properly it needs to be inside a normal div with no display: "flex"
+        `aspect-video`,
+        className
+      )}>
+      <AutoSizer>
+      {({ height, width }) => (
+         <VideoSphereViewer
+            video={video}
+            className={className}
+            width={width}
+            height={height}
+          />
+        )}
+      </AutoSizer>
+    </div>
+  )
+}

src/app/interface/equirectangular-video-player/viewer.tsx

@@ -0,0 +1,91 @@
+"use client"
+
+import { useEffect, useRef, useState } from "react"
+import AutoSizer from "react-virtualized-auto-sizer"
+import { PanoramaPosition, PluginConstructor, Point, Position, SphericalPosition, Viewer } from "@photo-sphere-viewer/core"
+import { EquirectangularVideoAdapter, LensflarePlugin, ReactPhotoSphereViewer, ResolutionPlugin, SettingsPlugin, VideoPlugin } from "react-photo-sphere-viewer"
+
+import { cn } from "@/lib/utils"
+import { VideoInfo } from "@/types"
+
+type PhotoSpherePlugin = (PluginConstructor | [PluginConstructor, any])
+
+export function VideoSphereViewer({
+  video,
+  className = "",
+  width,
+  height,
+  muted = false,
+ }: {
+  video: VideoInfo
+  className?: string
+  width: number
+  height: number
+  muted?: boolean
+}) {
+  const rootContainerRef = useRef<HTMLDivElement>(null)
+  const viewerContainerRef = useRef<HTMLElement>()
+  const viewerRef = useRef<Viewer>()
+
+  useEffect(() => {
+    if (!viewerRef.current) { return }
+    viewerRef.current.setOptions({
+      size: {
+        width: `${width}px`,
+        height: `${height}px`
+      }
+    })
+  }, [width, height])
+
+  if (!video.assetUrl) { return null }
+
+  return (
+    <div
+      // will be used later, if we need overlays and stuff
+      ref={rootContainerRef}
+    >
+      <ReactPhotoSphereViewer
+
+        container=""
+        containerClass={cn(
+          "rounded-xl overflow-hidden",
+          className
+        )}
+
+        width={`${width}px`}
+        height={`${height}px`}
+
+        onReady={(instance) => {
+          viewerRef.current = instance
+          viewerContainerRef.current = instance.container
+        }}
+        
+        // to access a plugin we must use viewer.getPlugin()
+        // plugins={[[LensflarePlugin, { lensflares: [] }]]}
+
+        adapter={[EquirectangularVideoAdapter, { muted }]}
+        navbar="video"
+        src="" 
+        plugins={[
+          [VideoPlugin, {
+            muted,
+            // progressbar: true,
+            bigbutton: false
+          }],
+          // SettingsPlugin,
+          [ResolutionPlugin, {
+            defaultResolution: 'HD',
+            resolutions: [
+              {
+                id: 'HD',
+                label: 'Standard',
+                panorama: { source: video.assetUrl },
+              },
+            ],
+          }],
+        ]}
+      />
+    </div>
+  )
+}
+  

src/app/interface/video-player/cartesian.tsx

@@ -0,0 +1,53 @@
+"use client"
+
+import { Player } from "react-tuby"
+import "react-tuby/css/main.css"
+
+import { cn } from "@/lib/utils"
+import { VideoInfo } from "@/types"
+
+export function CartesianVideoPlayer({
+  video,
+  enableShortcuts = true,
+  className = "",
+  // currentTime,
+ }: {
+  video: VideoInfo
+  enableShortcuts?: boolean
+  className?: string
+  // currentTime?: number
+}) {
+  return (
+    <div className={cn(
+      `w-full`,
+      `flex flex-col items-center justify-center`,
+      `rounded-xl overflow-hidden`,
+      className
+      )}>
+      <div className={cn(
+        `w-[calc(100%+16px)]`,
+        `-ml-2 -mr-2`,
+        `flex flex-col items-center justify-center`,
+        )}>
+        <Player
+        
+          // playerRef={ref}
+
+          src={[
+            {
+              quality: "Full HD",
+              url: video.assetUrl,
+            }
+          ]}
+
+          keyboardShortcut={enableShortcuts}
+
+          subtitles={[]}
+          poster={
+            `https://huggingface.co/datasets/jbilcke-hf/ai-tube-index/resolve/main/videos/${video.id}.webp`
+          }
+        />
+      </div>
+    </div>
+  )
+}

src/app/interface/video-player/equirectangular.tsx

@@ -0,0 +1,91 @@
+"use client"
+
+import { useEffect, useRef, useState } from "react"
+
+import { PanoramaPosition, PluginConstructor, Point, Position, SphericalPosition, Viewer } from "@photo-sphere-viewer/core"
+import { EquirectangularVideoAdapter, LensflarePlugin, ReactPhotoSphereViewer, ResolutionPlugin, SettingsPlugin, VideoPlugin } from "react-photo-sphere-viewer"
+
+import { cn } from "@/lib/utils"
+import { VideoInfo } from "@/types"
+
+type PhotoSpherePlugin = (PluginConstructor | [PluginConstructor, any])
+
+export function EquirectangularVideoPlayer({
+  video,
+  className = "",
+  width,
+  height,
+  muted = false,
+ }: {
+  video: VideoInfo
+  className?: string
+  width: number
+  height: number
+  muted?: boolean
+}) {
+  const rootContainerRef = useRef<HTMLDivElement>(null)
+  const viewerContainerRef = useRef<HTMLElement>()
+  const viewerRef = useRef<Viewer>()
+
+  useEffect(() => {
+    if (!viewerRef.current) { return }
+    viewerRef.current.setOptions({
+      size: {
+        width: `${width}px`,
+        height: `${height}px`
+      }
+    })
+  }, [width, height])
+
+  if (!video.assetUrl) { return null }
+
+  return (
+    <div
+      // will be used later, if we need overlays and stuff
+      ref={rootContainerRef}
+    >
+      <ReactPhotoSphereViewer
+
+        container=""
+        containerClass={cn(
+          "rounded-xl overflow-hidden",
+          className
+        )}
+
+        width={`${width}px`}
+        height={`${height}px`}
+
+        onReady={(instance) => {
+          viewerRef.current = instance
+          viewerContainerRef.current = instance.container
+        }}
+        
+        // to access a plugin we must use viewer.getPlugin()
+        // plugins={[[LensflarePlugin, { lensflares: [] }]]}
+
+        adapter={[EquirectangularVideoAdapter, { muted }]}
+        navbar="video"
+        src="" 
+        plugins={[
+          [VideoPlugin, {
+            muted,
+            // progressbar: true,
+            bigbutton: false
+          }],
+          // SettingsPlugin,
+          [ResolutionPlugin, {
+            defaultResolution: 'HD',
+            resolutions: [
+              {
+                id: 'HD',
+                label: 'Standard',
+                panorama: { source: video.assetUrl },
+              },
+            ],
+          }],
+        ]}
+      />
+    </div>
+  )
+}
+  

src/app/interface/video-player/index.tsx

@@ -1,12 +1,13 @@
 "use client"
 
-import { Player } from "react-tuby"
-import "react-tuby/css/main.css"
+import AutoSizer from "react-virtualized-auto-sizer"
 
 import { cn } from "@/lib/utils"
 import { VideoInfo } from "@/types"
-import { MutableRefObject, useEffect, useRef } from "react"
-import { isValidNumber } from "@/app/server/actions/utils/isValidNumber"
+import { parseProjectionFromLoRA } from "@/app/server/actions/utils/parseProjectionFromLoRA"
+
+import { EquirectangularVideoPlayer } from "./equirectangular"
+import { CartesianVideoPlayer } from "./cartesian"
 
 export function VideoPlayer({
   video,
@@ -19,51 +20,26 @@ export function VideoPlayer({
   className?: string
   // currentTime?: number
 }) {
+  // we should our video players from  misssing data
+  if (!video?.assetUrl) { return null }
 
-  /*
-  const ref = useRef(null)
-
-  useEffect(() => {
-    if (!ref.current) { return }
-    if (!isValidNumber(currentTime)) { return }
-    
-    (ref.current as any).currentTime = currentTime 
-    // $(".tuby-container video").currentTime = 2
-  }, [currentTime])
-  */
+  const isEquirectangular = (
+    video.projection === "equirectangular" ||
+    parseProjectionFromLoRA(video.lora) === "equirectangular"
+  )
 
-  // TODO: keep the same form factor?
-  if (!video) { return null }
+  if (isEquirectangular) {
+    // note: for AutoSizer to work properly it needs to be inside a normal div with no display: "flex"
+    return (
+      <div className={cn(`w-full aspect-video`, className)}>
+        <AutoSizer>{({ height, width }) => (
+           <EquirectangularVideoPlayer video={video} className={className} width={width} height={height} />
+          )}</AutoSizer>
+      </div>
+    )
+  }
 
   return (
-    <div className={cn(
-      `w-full`,
-      `flex flex-col items-center justify-center`,
-      `rounded-xl overflow-hidden`,
-      className
-      )}>
-      <div className={cn(
-        `w-[calc(100%+16px)]`,
-        `-ml-2 -mr-2`,
-        `flex flex-col items-center justify-center`,
-        )}>
-        <Player
-        
-          // playerRef={ref}
-
-          src={[
-            {
-              quality: "Full HD",
-              url: video.assetUrl,
-            }
-          ]}
-
-          keyboardShortcut={enableShortcuts}
-
-          subtitles={[]}
-          // poster="https://cdn.jsdelivr.net/gh/naptestdev/video-examples@master/poster.png"
-        />
-      </div>
-    </div>
+    <CartesianVideoPlayer video={video} enableShortcuts={enableShortcuts} className={className} />
   )
 }

src/app/server/actions/ai-tube-hf/getChannelVideos.ts

@@ -7,6 +7,7 @@ import { adminApiKey } from "../config"
 import { getVideoIndex } from "./getVideoIndex"
 import { extendVideosWithStats } from "./extendVideosWithStats"
 import { orientationToWidthHeight } from "../utils/orientationToWidthHeight"
+import { parseProjectionFromLoRA } from "../utils/parseProjectionFromLoRA"
 
 // return 
 export async function getChannelVideos({
@@ -46,6 +47,7 @@ export async function getChannelVideos({
         thumbnailUrl: v.thumbnailUrl,
         model: v.model,
         lora: v.lora,
+        projection: parseProjectionFromLoRA(v.lora),
         style: v.style,
         voice: v.voice,
         music: v.music,

src/app/server/actions/ai-tube-hf/getVideoRequestsFromChannel.ts

@@ -8,6 +8,7 @@ import { downloadFileAsText } from "./downloadFileAsText"
 import { parseDatasetPrompt } from "../utils/parseDatasetPrompt"
 import { parseVideoModelName } from "../utils/parseVideoModelName"
 import { orientationToWidthHeight } from "../utils/orientationToWidthHeight"
+import { parseProjectionFromLoRA } from "../utils/parseProjectionFromLoRA"
 
 /**
  * Return all the videos requests created by a user on their channel

src/app/server/actions/ai-tube-hf/uploadVideoRequestToDataset.ts

@@ -6,6 +6,7 @@ import { Credentials, uploadFile, whoAmI } from "@/huggingface/hub/src"
 import { ChannelInfo, VideoGenerationModel, VideoInfo, VideoOrientation, VideoRequest } from "@/types"
 import { formatPromptFileName } from "../utils/formatPromptFileName"
 import { orientationToWidthHeight } from "../utils/orientationToWidthHeight"
+import { parseProjectionFromLoRA } from "../utils/parseProjectionFromLoRA"
 
 /**
  * Save the video request to the user's own dataset
@@ -142,6 +143,7 @@ ${prompt}
     model,
     style,
     lora,
+    projection: parseProjectionFromLoRA(lora),
     voice,
     music,
     thumbnailUrl: channel.thumbnail, // will be generated in async

src/app/server/actions/utils/parseProjectionFromLoRA.ts

@@ -0,0 +1,17 @@
+import { VideoProjection } from "@/types"
+
+export function parseProjectionFromLoRA(input?: any): VideoProjection {
+ const name = `${input || ""}`.trim().toLowerCase()
+
+  const isEquirectangular = (
+    name.includes("equirectangular") ||
+    name.includes("panorama") ||
+    name.includes("360")
+  )
+
+  return (
+    isEquirectangular
+    ? "equirectangular"
+    : "cartesian"
+  )
+}

src/app/views/user-account-view/index.tsx

@@ -42,7 +42,7 @@ export function UserAccountView() {
         }
       })
     }
-  }, [isLoaded, huggingfaceApiKey])
+  }, [isLoaded, huggingfaceApiKey, setUserChannels, setLoaded])
 
   return (
     <div className={cn(`flex flex-col space-y-4`)}>

src/lib/usePlaylist.ts

@@ -109,7 +109,7 @@ export function usePlaylist() {
         dequeue();
       }
     }
-  }, [audio?.currentTime, dequeue, setProgress, isPlaying]);
+  }, [audio, audio?.currentTime, dequeue, setProgress, isPlaying]);
 
   const playback = useCallback(async (options?: PlaybackOptions<VideoInfo>): Promise<void> => {
     if (!audio) { return }

src/types.ts

@@ -330,6 +330,11 @@ export type VideoOrientation =
   | "landscape"
   | "square"
 
+export type VideoProjection =
+  | "cartesian" // this is the default
+  | "equirectangular"
+
+
 export type VideoInfo = {
   /**
    * UUID (v4)
@@ -446,6 +451,11 @@ export type VideoInfo = {
    * General video aspect ratio
    */
   orientation: VideoOrientation
+
+  /**
+   * Video projection (cartesian by default)
+   */
+  projection: VideoProjection
 }
 
 export type CollectionInfo = {