Add files

.gitattributes

@@ -32,3 +32,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+*.obj filter=lfs diff=lfs merge=lfs -text

.gitignore

@@ -0,0 +1,163 @@
+cache/
+experiments/
+
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+*.py,cover
+.hypothesis/
+.pytest_cache/
+cover/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+db.sqlite3-journal
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+.pybuilder/
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# IPython
+profile_default/
+ipython_config.py
+
+# pyenv
+#   For a library or package, you might want to ignore these files since the code is
+#   intended to run in multiple environments; otherwise, check them in:
+# .python-version
+
+# pipenv
+#   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
+#   However, in case of collaboration, if having platform-specific dependencies or dependencies
+#   having no cross-platform support, pipenv may install dependencies that don't work, or not
+#   install all needed dependencies.
+#Pipfile.lock
+
+# poetry
+#   Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
+#   This is especially recommended for binary packages to ensure reproducibility, and is more
+#   commonly ignored for libraries.
+#   https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
+#poetry.lock
+
+# pdm
+#   Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
+#pdm.lock
+#   pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it
+#   in version control.
+#   https://pdm.fming.dev/#use-with-ide
+.pdm.toml
+
+# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
+__pypackages__/
+
+# Celery stuff
+celerybeat-schedule
+celerybeat.pid
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre/
+
+# pytype static type analyzer
+.pytype/
+
+# Cython debug symbols
+cython_debug/
+
+# PyCharm
+#  JetBrains specific template is maintained in a separate JetBrains.gitignore that can
+#  be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
+#  and can be added to the global gitignore or merged into this file.  For a more nuclear
+#  option (not recommended) you can uncomment the following to ignore the entire idea folder.
+#.idea/

.gitmodules

@@ -0,0 +1,3 @@
+[submodule "TEXTurePaper"]
+	path = TEXTurePaper
+	url = https://github.com/TEXTurePaper/TEXTurePaper

.pre-commit-config.yaml

@@ -0,0 +1,36 @@
+exclude: patch
+repos:
+- repo: https://github.com/pre-commit/pre-commit-hooks
+  rev: v4.2.0
+  hooks:
+  - id: check-executables-have-shebangs
+  - id: check-json
+  - id: check-merge-conflict
+  - id: check-shebang-scripts-are-executable
+  - id: check-toml
+  - id: check-yaml
+  - id: double-quote-string-fixer
+  - id: end-of-file-fixer
+  - id: mixed-line-ending
+    args: ['--fix=lf']
+  - id: requirements-txt-fixer
+  - id: trailing-whitespace
+- repo: https://github.com/myint/docformatter
+  rev: v1.4
+  hooks:
+  - id: docformatter
+    args: ['--in-place']
+- repo: https://github.com/pycqa/isort
+  rev: 5.12.0
+  hooks:
+    - id: isort
+- repo: https://github.com/pre-commit/mirrors-mypy
+  rev: v0.991
+  hooks:
+    - id: mypy
+      args: ['--ignore-missing-imports']
+- repo: https://github.com/google/yapf
+  rev: v0.32.0
+  hooks:
+  - id: yapf
+    args: ['--parallel', '--in-place']

.style.yapf

@@ -0,0 +1,5 @@
+[style]
+based_on_style = pep8
+blank_line_before_nested_class_or_def = false
+spaces_before_comment = 2
+split_before_logical_operator = true

Dockerfile

@@ -0,0 +1,59 @@
+FROM nvidia/cuda:11.7.1-cudnn8-devel-ubuntu22.04
+ENV DEBIAN_FRONTEND=noninteractive
+RUN apt-get update && \
+    apt-get upgrade -y && \
+    apt-get install -y --no-install-recommends \
+    git \
+    wget \
+    curl \
+    zip \
+    # ffmpeg \
+    ffmpeg \
+    x264 \
+    # python build dependencies \
+    build-essential \
+    libssl-dev \
+    zlib1g-dev \
+    libbz2-dev \
+    libreadline-dev \
+    libsqlite3-dev \
+    libncursesw5-dev \
+    xz-utils \
+    tk-dev \
+    libxml2-dev \
+    libxmlsec1-dev \
+    libffi-dev \
+    liblzma-dev && \
+    apt-get clean && \
+    rm -rf /var/lib/apt/lists/*
+
+RUN useradd -m -u 1000 user
+USER user
+ENV HOME=/home/user \
+    PATH=/home/user/.local/bin:${PATH}
+WORKDIR ${HOME}/app
+
+RUN curl https://pyenv.run | bash
+ENV PATH=${HOME}/.pyenv/shims:${HOME}/.pyenv/bin:${PATH}
+ENV PYTHON_VERSION=3.9.16
+RUN pyenv install ${PYTHON_VERSION} && \
+    pyenv global ${PYTHON_VERSION} && \
+    pyenv rehash && \
+    pip install --no-cache-dir -U pip setuptools wheel
+
+RUN pip install --no-cache-dir -U torch==1.12.1+cu113 torchvision==0.13.1+cu113 --extra-index-url https://download.pytorch.org/whl/cu113
+COPY --chown=1000 requirements.txt /tmp
+RUN pip install --no-cache-dir -r /tmp/requirements.txt
+RUN pip install --no-cache-dir -U kaolin==0.13.0 -f https://nvidia-kaolin.s3.us-east-2.amazonaws.com/torch-1.12.1_cu113.html
+RUN pip install --no-cache-dir -U gradio==3.17.1
+
+COPY --chown=1000 . ${HOME}/app
+RUN cd TEXTurePaper && patch -p1 < ../patch
+ENV PYTHONPATH=${HOME}/app \
+    PYTHONUNBUFFERED=1 \
+    GRADIO_ALLOW_FLAGGING=never \
+    GRADIO_NUM_PORTS=1 \
+    GRADIO_SERVER_NAME=0.0.0.0 \
+    GRADIO_THEME=huggingface \
+    SYSTEM=spaces
+CMD ["python", "app.py"]

TEXTurePaper

@@ -0,0 +1 @@
+Subproject commit 3f393aa2b91cdb58f571d37a6304e068d791af10

app.py

@@ -0,0 +1,67 @@
+#!/usr/bin/env python
+
+from __future__ import annotations
+
+import pathlib
+
+import gradio as gr
+
+from model import Model
+
+DESCRIPTION = '''# TEXTure
+
+This is an unofficial demo for [https://github.com/TEXTurePaper/TEXTurePaper](https://github.com/TEXTurePaper/TEXTurePaper).
+
+This demo only accepts as input `.obj` files with less than 100,000 faces.
+
+Inference takes about 10 minutes on a T4 GPU.
+'''
+
+model = Model()
+
+with gr.Blocks(css='style.css') as demo:
+    gr.Markdown(DESCRIPTION)
+    with gr.Row():
+        with gr.Column():
+            input_shape = gr.Model3D(label='Input 3D mesh')
+            text = gr.Text(label='Text')
+            seed = gr.Slider(label='Seed',
+                             minimum=0,
+                             maximum=100000,
+                             value=3,
+                             step=1)
+            guidance_scale = gr.Slider(label='Guidance scale',
+                                       minimum=0,
+                                       maximum=50,
+                                       value=7.5,
+                                       step=0.1)
+            run_button = gr.Button('Run')
+        with gr.Column():
+            result = gr.Video(label='Result')
+            output_file = gr.File(label='Output mesh file')
+    with gr.Row():
+        examples = [
+            ['shapes/dragon1.obj', 'a photo of a dragon', 0, 7.5],
+            ['shapes/dragon2.obj', 'a photo of a dragon', 0, 7.5],
+            ['shapes/eagle.obj', 'a photo of an eagle', 0, 7.5],
+            ['shapes/napoleon.obj', 'a photo of Napoleon Bonaparte', 3, 7.5],
+            ['shapes/nascar.obj', 'A next gen nascar', 2, 10],
+        ]
+        gr.Examples(examples=examples,
+                    inputs=[input_shape, text, seed, guidance_scale],
+                    outputs=[result, output_file],
+                    cache_examples=False)
+
+    run_button.click(fn=model.run,
+                     inputs=[
+                         input_shape,
+                         text,
+                         seed,
+                         guidance_scale,
+                     ],
+                     outputs=[
+                         result,
+                         output_file,
+                     ])
+
+demo.queue(max_size=5).launch()

model.py

@@ -0,0 +1,63 @@
+from __future__ import annotations
+
+import datetime
+import pathlib
+import shlex
+import subprocess
+import sys
+
+import gradio as gr
+
+sys.path.append('TEXTurePaper')
+
+from src.configs.train_config import GuideConfig, LogConfig, TrainConfig
+from src.training.trainer import TEXTure
+
+
+class Model:
+    def __init__(self):
+        self.max_num_faces = 100000
+
+    def load_config(self, shape_path: str, text: str, seed: int,
+                    guidance_scale: float) -> TrainConfig:
+        text += ', {} view'
+
+        log = LogConfig(exp_name=self.gen_exp_name())
+        guide = GuideConfig(text=text)
+        guide.background_img = 'TEXTurePaper/textures/brick_wall.png'
+        guide.shape_path = 'TEXTurePaper/shapes/spot_triangulated.obj'
+        config = TrainConfig(log=log, guide=guide)
+
+        config.guide.shape_path = shape_path
+        config.optim.seed = seed
+        config.guide.guidance_scale = guidance_scale
+        return config
+
+    def gen_exp_name(self) -> str:
+        now = datetime.datetime.now()
+        return now.strftime('%Y-%m-%d-%H-%M-%S')
+
+    def check_num_faces(self, path: str) -> bool:
+        with open(path) as f:
+            lines = [line for line in f.readlines() if line.startswith('f')]
+        return len(lines) <= self.max_num_faces
+
+    def zip_results(self, exp_dir: pathlib.Path) -> str:
+        mesh_dir = exp_dir / 'mesh'
+        out_path = f'{exp_dir.name}.zip'
+        subprocess.run(shlex.split(f'zip -r {out_path} {mesh_dir}'))
+        return out_path
+
+    def run(self, shape_path: str, text: str, seed: int,
+            guidance_scale: float) -> tuple[str, str]:
+        if not shape_path.endswith('.obj'):
+            raise gr.Error('The input file is not .obj file.')
+        if not self.check_num_faces(shape_path):
+            raise gr.Error('The number of faces is over 100,000.')
+
+        config = self.load_config(shape_path, text, seed, guidance_scale)
+        trainer = TEXTure(config)
+        trainer.paint()
+        video_path = config.log.exp_dir / 'results' / 'step_00010_rgb.mp4'
+        zip_path = self.zip_results(config.log.exp_dir)
+        return video_path.as_posix(), zip_path

patch

@@ -0,0 +1,34 @@
+diff --git a/src/models/textured_mesh.py b/src/models/textured_mesh.py
+index 75ffb6d..40d30e6 100644
+--- a/src/models/textured_mesh.py
++++ b/src/models/textured_mesh.py
+@@ -122,7 +122,7 @@ class TexturedMeshModel(nn.Module):
+ 
+         self.renderer = Renderer(device=self.device, dim=(render_grid_size, render_grid_size),
+                                  interpolation_mode=self.opt.texture_interpolation_mode)
+-        self.env_sphere, self.mesh = self.init_meshes()
++        self.env_sphere, self.mesh = self.init_meshes(opt.shape_path)
+         self.default_color = [0.8, 0.1, 0.8]
+         self.background_sphere_colors, self.texture_img = self.init_paint()
+         self.meta_texture_img = nn.Parameter(torch.zeros_like(self.texture_img))
+diff --git a/src/stable_diffusion_depth.py b/src/stable_diffusion_depth.py
+index 610d2de..e172080 100644
+--- a/src/stable_diffusion_depth.py
++++ b/src/stable_diffusion_depth.py
+@@ -1,3 +1,5 @@
++import os
++
+ from diffusers import AutoencoderKL, UNet2DConditionModel, PNDMScheduler
+ from huggingface_hub import hf_hub_download
+ from transformers import CLIPTextModel, CLIPTokenizer, logging
+@@ -24,9 +26,7 @@ class StableDiffusion(nn.Module):
+         super().__init__()
+ 
+         try:
+-            with open('./TOKEN', 'r') as f:
+-                self.token = f.read().replace('\n', '')  # remove the last \n!
+-                logger.info(f'loaded hugging face access token from ./TOKEN!')
++            self.token = os.environ['HF_TOKEN']
+         except FileNotFoundError as e:
+             self.token = True
+             logger.warning(

requirements.txt

@@ -0,0 +1,15 @@
+accelerate==0.16.0
+diffusers==0.12.1
+einops==0.6.0
+huggingface-hub==0.12.0
+imageio[ffmpeg,pyav]==2.25.0
+loguru==0.6.0
+matplotlib==3.6.3
+ninja==1.11.1
+opencv-python-headless==4.7.0.68
+pyrallis==0.3.1
+torch==1.12.1
+torchvision==0.13.1
+tqdm==4.64.1
+transformers==4.26.0
+xatlas==0.0.7

shapes/dragon1.obj

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2b8c4ed26150539c33c15db2711e3f682621dccbcb500a69951e9a506442219a
+size 7186630

shapes/dragon2.obj

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:75fafd97c70db71f6635d21357aaf8c86864e999d04a809ded62659aa0999eb9
+size 7176659

shapes/eagle.obj

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b31932d34387664fa740f8792abe47213ed37372d9471cd3adbe2ce8fdde960f
+size 7282898

shapes/napoleon.obj

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2f2adeafaa5c50398ca8a75552332414564b0de5bcecb81a0fc89eb52cfed60b
+size 7249885

shapes/nascar.obj

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:343014429d93c7bc0bbf7586b22e96fe84db773beb74feae99976a65e1ce3f77
+size 483133

shapes/shell.obj

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:20a239e4c10392da488f6fdd6ef53f326537f341a04e97b09a6ad067fe8f05cc
+size 7176643

style.css

@@ -0,0 +1,3 @@
+h1 {
+  text-align: center;
+}