Update app.py

app.py

@@ -8,7 +8,7 @@ import torch
 import torch.nn.functional as F
 from torchvision import transforms
 from torchvision.transforms import Compose
-import trimesh
+import open3d as o3d
 #from geometry import create_triangles
 import tempfile
 from functools import partial
@@ -136,7 +136,7 @@ def make_video(video_path, outdir='./vis_video_depth', encoder='vits'):
                 break
 
             frame = cv2.cvtColor(raw_frame, cv2.COLOR_BGR2RGB) / 255.0
-            frame_pil =  Image.fromarray((frame * 255).astype(np.uint8))
+            frame_pil = Image.fromarray((frame * 255).astype(np.uint8))
             frame = transform({'image': frame})['image']
             
             frame = torch.from_numpy(frame).unsqueeze(0).to(DEVICE)
@@ -322,9 +322,23 @@ def get_mesh(image, depth, blur_data, loadall):
             #if (clrs[j] == pcolors[i]).all():
                 #cverts[i].append(verts[j])
                 #j=j+1
-    mesh.append(trimesh.PointCloud(verts, colors=clrs))
+
+    depth_o3d = o3d.geometry.Image(Image.fromarray(gdepth.astype(np.uint8)))
+    image_o3d = o3d.geometry.Image(Image.fromarray(rgba.astype(np.uint8)))
+    rgbd_image = o3d.geometry.RGBDImage.create_from_color_and_depth(
+        image_o3d, depth_o3d, convert_rgb_to_intensity=False)
+    w = int(depth_image.shape[1])
+    h = int(depth_image.shape[0])
+
+    camera_intrinsic = o3d.camera.PinholeCameraIntrinsic()
+    camera_intrinsic.set_intrinsics(w, h, 500, 500, w/2, h/2)
+
+    pcd = o3d.geometry.PointCloud.create_from_rgbd_image(
+        rgbd_image, camera_intrinsic)
+
+    mesh.append(pcd)
     print(mesh[len(mesh)-1])
-    mesh[len(mesh)-1].merge_vertices()
+    #mesh[len(mesh)-1].merge_vertices()
         #i=i+1
     mesh_n.append(str(fnum))
 
@@ -362,7 +376,7 @@ def get_mesh(image, depth, blur_data, loadall):
     #texture = trimesh.visual.TextureVisuals(uv=uv, image=image, material=material)
     #mesh.visual = texture
         
-    scene = trimesh.Scene(mesh)
+    #scene = trimesh.Scene(mesh)
     print('mesh - ok')
 
     # Save as glb
@@ -370,7 +384,7 @@ def get_mesh(image, depth, blur_data, loadall):
     #f = open(glb_file.name, "wb")
     #f.write(binary)
     #f.close()
-    scene.export(glb_file.name)
+    o3d.io.write_point_cloud(glb_file.name, pcd)
     print('file - ok')
     return glb_file.name, glb_file.name, ",".join(mesh_n)
 
@@ -613,40 +627,6 @@ async(f, c, o, b, p, d, n)=>{
 
 on_click="""
 async(p, d, n)=>{
-  function decycle(obj, stack = []) {
-    if (!obj || typeof obj !== 'object')
-        return obj;
-    
-    if (stack.includes(obj))
-        return null;
-
-    let s = stack.concat([obj]);
-
-    return Array.isArray(obj)
-        ? obj.map(x => decycle(x, s))
-        : Object.fromEntries(
-            Object.entries(obj)
-                .map(([k, v]) => [k, decycle(v, s)]));
-  }
-  var objectUrl;
-  function download3D(filename, scene) {
-    if(objectUrl) {
-        window.URL.revokeObjectURL(objectUrl);
-    }
-    var serializedScene = BABYLON.SceneSerializer.Serialize(scene);
-    var strMesh = JSON.stringify(decycle(serializedScene));
-    if (filename.toLowerCase().lastIndexOf(".babylon") !== filename.length - 8 || filename.length < 9){
-        filename += ".babylon";
-    }
-    var blob = new Blob ( [ strMesh ], { type : "octet/stream" } );
-            
-    // turn blob into an object URL; saved as a member, so can be cleaned out later
-    objectUrl = (window.webkitURL || window.URL).createObjectURL(blob);
-
-    document.getElementById("file3D").getElementsByTagName("a")[0].download = filename;
-    document.getElementById("file3D").getElementsByTagName("a")[0].innerText = blob.size / 1024 + " KB";
-    document.getElementById("file3D").getElementsByTagName("a")[0].href = objectUrl;
-  }
   var intv = setInterval(function(){
     if (document.getElementById("iframe3D")===null || typeof document.getElementById("iframe3D")==="undefined") {
       if (BABYLON && BABYLON.Engine.LastCreatedScene) {
@@ -709,10 +689,6 @@ async(p, d, n)=>{
           mesh.dispose(false, true);
         }
         BABYLON.Engine.LastCreatedScene.getMeshByName("dome_mesh").applyDisplacementMap(d.background.url, 0, 255, function(m){try{alert(BABYLON.Engine.Version);}catch(e){alert(e);}}, null, null, true, function(e){alert(e);});
-
-        try {
-        download3D("filename", BABYLON.Engine.LastCreatedScene);
-        } catch (e){alert(e);}
         
         clearInterval(intv);
       }
@@ -745,8 +721,7 @@ css = """
 
 title = "# Depth Anything Video Demo"
 description = """Depth Anything on full video files.  
-Please refer to our [paper](https://arxiv.org/abs/2401.10891), [project page](https://depth-anything.github.io), or [github](https://github.com/LiheYoung/Depth-Anything) for more details.  
-Mesh rendering from [ZoeDepth](https://huggingface.co/spaces/shariqfarooq/ZoeDepth) ([github](https://github.com/isl-org/ZoeDepth/tree/main/ui))."""
+Please refer to our [paper](https://arxiv.org/abs/2401.10891), [project page](https://depth-anything.github.io), or [github](https://github.com/LiheYoung/Depth-Anything) for more details."""
 
 transform = Compose([
         Resize(