semantic-aware hand+face replacement

LICENSE

@@ -39,12 +39,12 @@ You acknowledge that the Data & Software is a valuable scientific resource and a
 
 Citation:
 
-@inproceedings{xiu2022econ,
-  title={{ECON}: {E}xplicit {C}lothed humans {O}btained from {N}ormals},
-  author={Xiu, Yuliang and Yang, Jinlong and Cao, Xu and Tzionas, Dimitrios and Black, Michael J.},
-  booktitle=arXiv,
-  month = Dec,
-  year={2022}
+@misc{xiu2022econ,
+    title={ECON: Explicit Clothed humans Obtained from Normals},
+    author={Xiu, Yuliang and Yang, Jinlong and Cao, Xu and Tzionas, Dimitrios and Black, Michael J.},
+    year={2022}
+    publisher={arXiv},
+    primaryClass={cs.CV}
 }
 
 Commercial licensing opportunities

README.md

@@ -25,8 +25,10 @@
     <a href="https://pytorchlightning.ai/"><img alt="Lightning" src="https://img.shields.io/badge/-Lightning-792ee5?logo=pytorchlightning&logoColor=white"></a>
     <br></br>
     <a href=''>
-      <img src='https://img.shields.io/badge/Paper-PDF-green?style=for-the-badge&logo=arXiv&logoColor=green' alt='Paper PDF'>
+      <img src='https://img.shields.io/badge/Paper-PDF (coming soon)-green?style=for-the-badge&logo=arXiv&logoColor=green' alt='Paper PDF'>
     </a>
+    <a href='https://xiuyuliang.cn/econ/'>
+      <img src='https://img.shields.io/badge/ECON-Page-orange?style=for-the-badge&logo=Google%20chrome&logoColor=orange' alt='Project Page'></a>
     <a href="https://discord.gg/Vqa7KBGRyk"><img src="https://img.shields.io/discord/940240966844035082?color=7289DA&labelColor=4a64bd&logo=discord&logoColor=white&style=for-the-badge"></a>
     <a href="https://youtu.be/j5hw4tsWpoY"><img alt="youtube views" title="Subscribe to my YouTube channel" src="https://img.shields.io/youtube/views/j5hw4tsWpoY?logo=youtube&labelColor=ce4630&style=for-the-badge"/></a>
   </p>
@@ -39,9 +41,11 @@ ECON is designed for **"Human digitization from a color image"**, which combines
 <br/>
 
 ## News :triangular_flag_on_post:
-- [2022/03/05] <a href="">arXiv</a> and <a href="#demo">demo</a> are available.
+
+- [2022/12/09] <a href="#demo">Demo</a> is available.
 
 ## TODO
+
 - [ ] Blender add-on for FBX export
 - [ ] Full RGB texture generation
 
@@ -72,29 +76,33 @@ ECON is designed for **"Human digitization from a color image"**, which combines
 
 - See [docs/installation.md](docs/installation.md) to install all the required packages and setup the models
 
-
 ## Demo
 
 ```bash
-# For image-based reconstruction
+# For single-person image-based reconstruction
 python -m apps.infer -cfg ./configs/econ.yaml -in_dir ./examples -out_dir ./results
 
-# For video rendering
+# For multi-person image-based reconstruction (see config/econ.yaml)
+python -m apps.infer -cfg ./configs/econ.yaml -in_dir ./examples -out_dir ./results -multi
+
+# To generate the demo video of reconstruction results
 python -m apps.multi_render -n {filename}
 ```
 
 ## Tricks
-### Some adjustable parameters in *config/econ.yaml*
-- `use_ifnet`
+
+### Some adjustable parameters in _config/econ.yaml_
+
+- `use_ifnet: True`
   - True: use IF-Nets+ for mesh completion ( $\text{ECON}_\text{IF}$ - Better quality)
   - False: use SMPL-X for mesh completion ( $\text{ECON}_\text{EX}$ - Faster speed)
-- `use_smpl`
+- `use_smpl: ["hand", "face"]`
   - [ ]: don't use either hands or face parts from SMPL-X
   - ["hand"]: only use the **visible** hands from SMPL-X
   - ["hand", "face"]: use both **visible** hands and face from SMPL-X
-- `thickness` (default 2cm)
+- `thickness: 2cm`
   - could be increased accordingly in case **xx_full.obj** looks flat
-- `hps_type`
+- `hps_type: pixie`
   - "pixie": more accurate for face and hands
   - "pymafx": more robust for challenging poses
 
@@ -102,16 +110,15 @@ python -m apps.multi_render -n {filename}
 
 ## More Qualitative Results
 
-|![OOD Poses](assets/OOD-poses.jpg)|
-| :----------------------: |
-|_Challenging Poses_|
-|![OOD Clothes](assets/OOD-outfits.jpg)|
-|_Loose Clothes_|
-|![SHHQ](assets/SHHQ.gif)|
-|_ECON Results on [SHHQ Dataset](https://github.com/stylegan-human/StyleGAN-Human)_|
-|![crowd](assets/crowd.gif)|
-|_ECON Results on Multi-Person Image_|
-
+|                         ![OOD Poses](assets/OOD-poses.jpg)                         |
+| :--------------------------------------------------------------------------------: |
+|                                _Challenging Poses_                                 |
+|                       ![OOD Clothes](assets/OOD-outfits.jpg)                       |
+|                                  _Loose Clothes_                                   |
+|                              ![SHHQ](assets/SHHQ.gif)                              |
+| _ECON Results on [SHHQ Dataset](https://github.com/stylegan-human/StyleGAN-Human)_ |
+|                             ![crowd](assets/crowd.gif)                             |
+|                        _ECON Results on Multi-Person Image_                        |
 
 <br/>
 <br/>
@@ -119,14 +126,15 @@ python -m apps.multi_render -n {filename}
 ## Citation
 
 ```bibtex
-@inproceedings{xiu2022econ,
-  title     = {{ECON}: {E}xplicit {C}lothed humans {O}btained from {N}ormals},
-  author    = {Xiu, Yuliang and Yang, Jinlong and Cao, Xu and Tzionas, Dimitrios and Black, Michael J.},
-  booktitle = arXiv,
-  month     = {Dec},
-  year      = {2022},
+@misc{xiu2022econ,
+    title={ECON: Explicit Clothed humans Obtained from Normals},
+    author={Xiu, Yuliang and Yang, Jinlong and Cao, Xu and Tzionas, Dimitrios and Black, Michael J.},
+    year={2022}
+    publisher={arXiv},
+    primaryClass={cs.CV}
 }
 ```
+
 <br/>
 
 ## Acknowledgments
@@ -146,7 +154,7 @@ Some images used in the qualitative examples come from [pinterest.com](https://w
 
 This project has received funding from the European Union’s Horizon 2020 research and innovation programme under the Marie Skłodowska-Curie grant agreement No.860768 ([CLIPE Project](https://www.clipe-itn.eu)).
 
---------------
+---
 
 <br>
 
@@ -156,10 +164,11 @@ This code and model are available for non-commercial scientific research purpose
 
 ## Disclosure
 
-MJB has received research gift funds from Adobe, Intel, Nvidia, Meta/Facebook, and Amazon.  MJB has financial interests in Amazon, Datagen Technologies, and Meshcapade GmbH.
+MJB has received research gift funds from Adobe, Intel, Nvidia, Meta/Facebook, and Amazon. MJB has financial interests in Amazon, Datagen Technologies, and Meshcapade GmbH.
 
 ## Contact
 
 For technical questions, please contact [email protected]
 
-For commercial licensing, please contact [email protected] and [email protected]
+For commercial licensing, please contact [email protected]
+

apps/infer.py

@@ -31,6 +31,7 @@ from termcolor import colored
 from tqdm.auto import tqdm
 from apps.Normal import Normal
 from apps.IFGeo import IFGeo
+from pytorch3d.ops import SubdivideMeshes
 from lib.common.config import cfg
 from lib.common.train_util import init_loss, load_normal_networks, load_networks
 from lib.common.BNI import BNI
@@ -91,6 +92,11 @@ if __name__ == "__main__":
         "vol_res": cfg.vol_res,
         "single": args.multi,
     }
+    
+    if cfg.bni.use_ifnet:
+        print(colored("Use IF-Nets (Implicit)+ for completion", "green"))
+    else:
+        print(colored("Use SMPL-X (Explicit) for completion", "green"))
 
     dataset = TestDataset(dataset_param, device)
 
@@ -378,6 +384,7 @@ if __name__ == "__main__":
             side_mesh = smpl_obj_lst[idx].copy()
             face_mesh = smpl_obj_lst[idx].copy()
             hand_mesh = smpl_obj_lst[idx].copy()
+            smplx_mesh = smpl_obj_lst[idx].copy()
 
             # save normals, depths and masks
             BNI_dict = save_normal_tensor(
@@ -404,7 +411,6 @@ if __name__ == "__main__":
             # replace SMPL by completed mesh as side_mesh
 
             if cfg.bni.use_ifnet:
-                print(colored("Use IF-Nets+ for completion\n", "green"))
 
                 side_mesh_path = f"{args.out_dir}/{cfg.name}/obj/{data['name']}_{idx}_IF.obj"
 
@@ -436,13 +442,21 @@ if __name__ == "__main__":
                 side_mesh = remesh(side_mesh, side_mesh_path)
 
             else:
-                print(colored("Use SMPL-X body for completion\n", "green"))
                 side_mesh = apply_vertex_mask(
                     side_mesh,
                     (SMPLX_object.front_flame_vertex_mask + SMPLX_object.mano_vertex_mask +
                      SMPLX_object.eyeball_vertex_mask).eq(0).float(),
                 )
 
+                # upsample the side mesh
+                side_sub_mesh = Meshes(
+                    verts=[torch.tensor(side_mesh.vertices).float()],
+                    faces=[torch.tensor(side_mesh.faces).long()],
+                )
+                sm = SubdivideMeshes(side_sub_mesh)
+                new_mesh = sm(side_sub_mesh)
+                side_mesh = trimesh.Trimesh(new_mesh.verts_padded().squeeze(), new_mesh.faces_padded().squeeze())
+
             side_verts = torch.tensor(side_mesh.vertices).float().to(device)
             side_faces = torch.tensor(side_mesh.faces).long().to(device)
 
@@ -464,9 +478,8 @@ if __name__ == "__main__":
 
                 # remove face neighbor triangles
                 BNI_object.F_B_trimesh = part_removal(
-                    BNI_object.F_B_trimesh, None, face_mesh, cfg.bni.face_thres, device, camera_ray=True)
-                side_mesh = part_removal(
-                    side_mesh, torch.zeros_like(side_verts[:, 0:1]), face_mesh, cfg.bni.face_thres, device, camera_ray=True)
+                    BNI_object.F_B_trimesh, face_mesh, cfg.bni.face_thres, device, smplx_mesh, region="face")
+                side_mesh = part_removal(side_mesh, face_mesh, cfg.bni.face_thres, device, smplx_mesh, region="face")
                 face_mesh.export(f"{args.out_dir}/{cfg.name}/obj/{data['name']}_{idx}_face.obj")
                 full_lst += [face_mesh]
 
@@ -480,18 +493,18 @@ if __name__ == "__main__":
 
                 # only hands
                 hand_mesh = apply_vertex_mask(hand_mesh, hand_mask)
-                # remove face neighbor triangles
+
+                # remove hand neighbor triangles
                 BNI_object.F_B_trimesh = part_removal(
-                    BNI_object.F_B_trimesh, None, hand_mesh, cfg.bni.hand_thres, device, camera_ray=True)
-                side_mesh = part_removal(
-                    side_mesh, torch.zeros_like(side_verts[:, 0:1]), hand_mesh, cfg.bni.hand_thres, device, camera_ray=True)
+                    BNI_object.F_B_trimesh, hand_mesh, cfg.bni.hand_thres, device, smplx_mesh, region="hand")
+                side_mesh = part_removal(side_mesh, hand_mesh, cfg.bni.hand_thres, device, smplx_mesh, region="hand")
                 hand_mesh.export(f"{args.out_dir}/{cfg.name}/obj/{data['name']}_{idx}_hand.obj")
                 full_lst += [hand_mesh]
 
             full_lst += [BNI_object.F_B_trimesh]
 
             # initial side_mesh could be SMPLX or IF-net
-            side_mesh = part_removal(side_mesh, torch.zeros_like(side_verts[:, 0:1]), sum(full_lst), 2e-2, device, clean=False)
+            side_mesh = part_removal(side_mesh, sum(full_lst), 2e-2, device, smplx_mesh, region="", clean=False)
 
             full_lst += [side_mesh]
 

configs/econ.yaml

@@ -14,22 +14,24 @@ batch_size: 1
 dataset:
   prior_type: "SMPL"
 
-# user defined
-vol_res: 256      # IF-Net volume resolution
-mcube_res: 256
-clean_mesh: True  # if True, will remove floating pieces
+vol_res: 256
+mcube_res: 128
+clean_mesh: True
 cloth_overlap_thres: 0.50
-body_overlap_thres: 0.98
+body_overlap_thres: 0.00
+
+# For crowded / occluded scene
+# body_overlap_thres: 0.98
 
 bni:
-  k: 2
+  k: 4
   lambda1: 1e-4
   boundary_consist: 1e-6
   poisson_depth: 10
   use_smpl: ["hand", "face"]
   use_ifnet: True
   use_poisson: True
-  hand_thres: 4e-2
+  hand_thres: 8e-2
   face_thres: 6e-2
   thickness: 0.02
   hps_type: "pixie"

lib/common/BNI.py

@@ -25,9 +25,6 @@ class BNI:
         # k --> smaller, keep continuity
         # lambda --> larger, more depth-awareness
 
-        # self.k = self.cfg.k
-        # self.lambda1 = self.cfg.lambda1
-        # self.boundary_consist = self.cfg.boundary_consist
         self.k = self.cfg['k']
         self.lambda1 = self.cfg['lambda1']
         self.boundary_consist = self.cfg['boundary_consist']

lib/common/BNI_utils.py

@@ -657,22 +657,6 @@ def save_normal_tensor(in_tensor, idx, png_path, thickness=0.0):
 
     BNI_dict = {}
     
-    # add random masks
-    # normal_F_arr[200:300,200:300,:] *= 0
-    # normal_B_arr[200:300,200:300,:] *= 0
-    # mask_normal_arr[200:300,200:300] *= 0
-    
-    # normal_F_arr[:,:200,:] *= 0
-    # normal_B_arr[:,:200,:] *= 0
-    # mask_normal_arr[:,:200] *= 0
-    
-    # normal_F_arr[:200,:,:] *= 0
-    # normal_B_arr[:200,:,:] *= 0
-    # mask_normal_arr[:200,:] *= 0
-    
-    # Image.fromarray(((normal_F_arr+1.0)*0.5*255).astype(np.uint8)).save(png_path+"_F.png")
-    # Image.fromarray(((normal_B_arr+1.0)*0.5*255).astype(np.uint8)).save(png_path+"_B.png")
-    
     # clothed human
     BNI_dict["normal_F"] = normal_F_arr
     BNI_dict["normal_B"] = normal_B_arr

lib/dataset/TestDataset.py

@@ -80,7 +80,7 @@ class TestDataset:
 
         self.smpl_model = PIXIE_SMPLX(pixie_cfg.model).to(self.device)
 
-        print(colored(f"Using -- {self.hps_type} -- as HPS Estimator\n", "green"))
+        print(colored(f"Use {self.hps_type.upper()} to estimate human pose and shape", "green"))
 
         self.render = Render(size=512, device=self.device)
 

lib/dataset/mesh_util.py

@@ -24,17 +24,19 @@ import os
 from termcolor import colored
 import os.path as osp
 import _pickle as cPickle
+from scipy.spatial import cKDTree
 
 from pytorch3d.structures import Meshes
 import torch.nn.functional as F
 import lib.smplx as smplx
-from lib.common.imutils import uncrop
-from lib.common.render_utils import Pytorch3dRasterizer
 from pytorch3d.renderer.mesh import rasterize_meshes
 from PIL import Image, ImageFont, ImageDraw
 from pytorch3d.loss import mesh_laplacian_smoothing, mesh_normal_consistency
 import tinyobjloader
 
+from lib.common.imutils import uncrop
+from lib.common.render_utils import Pytorch3dRasterizer
+
 
 class SMPLX:
 
@@ -55,11 +57,13 @@ class SMPLX:
         self.smplx_flame_vid_path = osp.join(self.current_dir, "smpl_data/FLAME_SMPLX_vertex_ids.npy")
         self.smplx_mano_vid_path = osp.join(self.current_dir, "smpl_data/MANO_SMPLX_vertex_ids.pkl")
         self.front_flame_path = osp.join(self.current_dir, "smpl_data/FLAME_face_mask_ids.npy")
+        self.smplx_vertex_lmkid_path = osp.join(self.current_dir, "smpl_data/smplx_vertex_lmkid.npy")
 
         self.smplx_faces = np.load(self.smplx_faces_path)
         self.smplx_verts = np.load(self.smplx_verts_path)
         self.smpl_verts = np.load(self.smpl_verts_path)
         self.smpl_faces = np.load(self.smpl_faces_path)
+        self.smplx_vertex_lmkid = np.load(self.smplx_vertex_lmkid_path)
 
         self.smplx_eyeball_fid_mask = np.load(self.smplx_eyeball_fid_path)
         self.smplx_mouth_fid = np.load(self.smplx_fill_mouth_fid_path)
@@ -264,28 +268,32 @@ def apply_vertex_face_mask(mesh, vertex_mask, face_mask):
     return mesh
 
 
-def part_removal(full_mesh, vis_mask, part_mesh, thres, device, clean=True, camera_ray=False):
+def part_removal(full_mesh, part_mesh, thres, device, smpl_obj, region, clean=True):
 
-    # thres: face 3e-2, hand 4e-2
+    smpl_tree = cKDTree(smpl_obj.vertices)
+    SMPL_container = SMPLX()
 
     from lib.dataset.PointFeat import PointFeat
+
     part_extractor = PointFeat(
         torch.tensor(part_mesh.vertices).unsqueeze(0).to(device),
         torch.tensor(part_mesh.faces).unsqueeze(0).to(device))
 
-    (part_dist, part_cos) = part_extractor.query(torch.tensor(full_mesh.vertices).unsqueeze(0).to(device))
+    (part_dist, _) = part_extractor.query(torch.tensor(full_mesh.vertices).unsqueeze(0).to(device))
 
-    if camera_ray:
-        remove_mask = torch.logical_and(part_dist < thres, part_cos > 0.5)
-    else:
-        remove_mask = part_dist < thres
+    remove_mask = part_dist < thres
 
-    if vis_mask is not None:
-        BNI_verts_mask = ~(torch.logical_or(remove_mask, vis_mask.to(device))).flatten()
-    else:
-        BNI_verts_mask = ~(remove_mask).flatten()
+    if region == "hand":
+        _, idx = smpl_tree.query(full_mesh.vertices, k=1)
+        full_lmkid = SMPL_container.smplx_vertex_lmkid[idx]
+        remove_mask = torch.logical_and(remove_mask, torch.tensor(full_lmkid >= 20).type_as(remove_mask).unsqueeze(0))
+
+    elif region == "face":
+        _, idx = smpl_tree.query(full_mesh.vertices, k=5)
+        face_space_mask = torch.isin(torch.tensor(idx), torch.tensor(SMPL_container.smplx_front_flame_vid))
+        remove_mask = torch.logical_and(remove_mask, face_space_mask.any(dim=1).type_as(remove_mask).unsqueeze(0))
 
-    BNI_part_mask = BNI_verts_mask[full_mesh.faces].any(dim=1)
+    BNI_part_mask = ~(remove_mask).flatten()[full_mesh.faces].any(dim=1)
     full_mesh.update_faces(BNI_part_mask.detach().cpu())
     full_mesh.remove_unreferenced_vertices()
 
@@ -544,6 +552,7 @@ def poisson_remesh(obj_path):
     ms.meshing_decimation_quadric_edge_collapse(targetfacenum=50000)
     # ms.apply_coord_laplacian_smoothing()
     ms.save_current_mesh(obj_path)
+    ms.save_current_mesh(obj_path.replace(".obj", ".ply"))
     polished_mesh = trimesh.load_mesh(obj_path)
 
     return polished_mesh