Upload folder using huggingface_hub

.dockerignore

@@ -0,0 +1,40 @@
+# The .dockerignore file excludes files from the container build process.
+#
+# https://docs.docker.com/engine/reference/builder/#dockerignore-file
+
+# Replicate
+/safety-cache/
+/gradio_cached_examples/
+*.mp4
+*.pth
+*.pt
+*.bin
+*.ckpt
+*.onnx
+*.tar
+*.tar.gz
+*.h5
+*.pb
+*.caffemodel
+*.weights
+*.tar
+*.jpg
+*.jpeg
+*.png
+*.webp
+.vscode
+
+# Exclude Git files
+.git
+.github
+.gitignore
+
+# Exclude Python cache files
+__pycache__
+.pytest_cache/
+.mypy_cache
+.pytest_cache
+.ruff_cache
+
+# Exclude Python virtual environment
+/venv

.gitattributes

@@ -33,3 +33,9 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+generated_images/20240723_053704_668578_0.png filter=lfs diff=lfs merge=lfs -text
+generated_images/20240723_053801_148984_0.png filter=lfs diff=lfs merge=lfs -text
+generated_images/20240723_053853_022841_0.png filter=lfs diff=lfs merge=lfs -text
+generated_images/20240723_053948_468290_0.png filter=lfs diff=lfs merge=lfs -text
+generated_images/20240723_054025_692605_0.png filter=lfs diff=lfs merge=lfs -text
+generated_images/20240723_054124_697176_0.png filter=lfs diff=lfs merge=lfs -text

.gitignore

@@ -0,0 +1,186 @@
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+*.py,cover
+.hypothesis/
+.pytest_cache/
+cover/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+db.sqlite3-journal
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+.pybuilder/
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# IPython
+profile_default/
+ipython_config.py
+
+# pyenv
+#   For a library or package, you might want to ignore these files since the code is
+#   intended to run in multiple environments; otherwise, check them in:
+# .python-version
+
+# pipenv
+#   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
+#   However, in case of collaboration, if having platform-specific dependencies or dependencies
+#   having no cross-platform support, pipenv may install dependencies that don't work, or not
+#   install all needed dependencies.
+#Pipfile.lock
+
+# poetry
+#   Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
+#   This is especially recommended for binary packages to ensure reproducibility, and is more
+#   commonly ignored for libraries.
+#   https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
+#poetry.lock
+
+# pdm
+#   Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
+#pdm.lock
+#   pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it
+#   in version control.
+#   https://pdm.fming.dev/#use-with-ide
+.pdm.toml
+
+# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
+__pypackages__/
+
+# Celery stuff
+celerybeat-schedule
+celerybeat.pid
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre/
+
+# pytype static type analyzer
+.pytype/
+
+# Cython debug symbols
+cython_debug/
+
+# PyCharm
+#  JetBrains specific template is maintained in a separate JetBrains.gitignore that can
+#  be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
+#  and can be added to the global gitignore or merged into this file.  For a more nuclear
+#  option (not recommended) you can uncomment the following to ignore the entire idea folder.
+#.idea/
+huggingface/
+
+# Cog
+/.cog/
+/safety-cache/
+*.tar
+.vscode
+gradio_cached_examples
+cog/test_batchsize.py
+input.png
+output_*.png
+output.*.png
+output_image_*.png
+output_image.*.png
+output_*.webp
+output.*.webp
+output_image_*.webp
+output_image.*.webp
+output_*.jpg
+output.*.jpg
+output_image_*.jpg
+output_image.*.jpg
+output_*.jpeg
+output.*.jpeg
+output_image_*.jpeg
+output_image.*.jpeg

LICENSE

@@ -0,0 +1,201 @@
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright [yyyy] [name of copyright owner]
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.

README.md

@@ -1,12 +1,6 @@
 ---
-title: IDfy Avatarifyy
-emoji: 🐢
-colorFrom: gray
-colorTo: pink
+title: IDfy-Avatarifyy
+app_file: gradio_demo/app.py
 sdk: gradio
-sdk_version: 4.42.0
-app_file: app.py
-pinned: false
+sdk_version: 4.38.1
 ---
-
-Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference

Untitled.ipynb

@@ -0,0 +1,432 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "id": "82c97d2c-16bf-4b2c-b16c-d9f7a8a5b12f",
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Collecting diffusers==0.25.1 (from -r gradio_demo/requirements.txt (line 1))\n",
+      "  Downloading diffusers-0.25.1-py3-none-any.whl.metadata (19 kB)\n",
+      "Requirement already satisfied: torch==2.0.0 in /opt/conda/lib/python3.10/site-packages (from -r gradio_demo/requirements.txt (line 2)) (2.0.0+cu118)\n",
+      "Requirement already satisfied: torchvision==0.15.1 in /opt/conda/lib/python3.10/site-packages (from -r gradio_demo/requirements.txt (line 3)) (0.15.1+cu118)\n",
+      "Collecting transformers==4.37.1 (from -r gradio_demo/requirements.txt (line 4))\n",
+      "  Downloading transformers-4.37.1-py3-none-any.whl.metadata (129 kB)\n",
+      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m129.4/129.4 kB\u001b[0m \u001b[31m1.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0ma \u001b[36m0:00:01\u001b[0m\n",
+      "\u001b[?25hCollecting accelerate (from -r gradio_demo/requirements.txt (line 5))\n",
+      "  Downloading accelerate-0.32.1-py3-none-any.whl.metadata (18 kB)\n",
+      "Collecting safetensors (from -r gradio_demo/requirements.txt (line 6))\n",
+      "  Downloading safetensors-0.4.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata (3.8 kB)\n",
+      "Collecting einops (from -r gradio_demo/requirements.txt (line 7))\n",
+      "  Downloading einops-0.8.0-py3-none-any.whl.metadata (12 kB)\n",
+      "Collecting onnxruntime-gpu (from -r gradio_demo/requirements.txt (line 8))\n",
+      "  Downloading onnxruntime_gpu-1.18.1-cp310-cp310-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl.metadata (4.4 kB)\n",
+      "Collecting spaces==0.19.4 (from -r gradio_demo/requirements.txt (line 9))\n",
+      "  Downloading spaces-0.19.4-py3-none-any.whl.metadata (972 bytes)\n",
+      "Collecting omegaconf (from -r gradio_demo/requirements.txt (line 10))\n",
+      "  Downloading omegaconf-2.3.0-py3-none-any.whl.metadata (3.9 kB)\n",
+      "Collecting peft (from -r gradio_demo/requirements.txt (line 11))\n",
+      "  Downloading peft-0.11.1-py3-none-any.whl.metadata (13 kB)\n",
+      "Collecting huggingface-hub==0.20.2 (from -r gradio_demo/requirements.txt (line 12))\n",
+      "  Downloading huggingface_hub-0.20.2-py3-none-any.whl.metadata (12 kB)\n",
+      "Collecting opencv-python (from -r gradio_demo/requirements.txt (line 13))\n",
+      "  Downloading opencv_python-4.10.0.84-cp37-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata (20 kB)\n",
+      "Collecting insightface (from -r gradio_demo/requirements.txt (line 14))\n",
+      "  Downloading insightface-0.7.3.tar.gz (439 kB)\n",
+      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m439.5/439.5 kB\u001b[0m \u001b[31m10.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m00:01\u001b[0m\n",
+      "\u001b[?25h  Installing build dependencies ... \u001b[?25ldone\n",
+      "\u001b[?25h  Getting requirements to build wheel ... \u001b[?25ldone\n",
+      "\u001b[?25h  Preparing metadata (pyproject.toml) ... \u001b[?25ldone\n",
+      "\u001b[?25hCollecting gradio (from -r gradio_demo/requirements.txt (line 15))\n",
+      "  Downloading gradio-4.38.1-py3-none-any.whl.metadata (15 kB)\n",
+      "Collecting controlnet_aux (from -r gradio_demo/requirements.txt (line 16))\n",
+      "  Downloading controlnet_aux-0.0.9-py3-none-any.whl.metadata (6.5 kB)\n",
+      "Collecting gdown (from -r gradio_demo/requirements.txt (line 17))\n",
+      "  Downloading gdown-5.2.0-py3-none-any.whl.metadata (5.8 kB)\n",
+      "Requirement already satisfied: importlib-metadata in /opt/conda/lib/python3.10/site-packages (from diffusers==0.25.1->-r gradio_demo/requirements.txt (line 1)) (7.0.0)\n",
+      "Requirement already satisfied: filelock in /opt/conda/lib/python3.10/site-packages (from diffusers==0.25.1->-r gradio_demo/requirements.txt (line 1)) (3.15.4)\n",
+      "Requirement already satisfied: numpy in /opt/conda/lib/python3.10/site-packages (from diffusers==0.25.1->-r gradio_demo/requirements.txt (line 1)) (1.25.2)\n",
+      "Collecting regex!=2019.12.17 (from diffusers==0.25.1->-r gradio_demo/requirements.txt (line 1))\n",
+      "  Downloading regex-2024.5.15-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata (40 kB)\n",
+      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m40.9/40.9 kB\u001b[0m \u001b[31m5.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
+      "\u001b[?25hRequirement already satisfied: requests in /opt/conda/lib/python3.10/site-packages (from diffusers==0.25.1->-r gradio_demo/requirements.txt (line 1)) (2.32.3)\n",
+      "Requirement already satisfied: Pillow in /opt/conda/lib/python3.10/site-packages (from diffusers==0.25.1->-r gradio_demo/requirements.txt (line 1)) (10.4.0)\n",
+      "Requirement already satisfied: typing-extensions in /opt/conda/lib/python3.10/site-packages (from torch==2.0.0->-r gradio_demo/requirements.txt (line 2)) (4.12.2)\n",
+      "Requirement already satisfied: sympy in /opt/conda/lib/python3.10/site-packages (from torch==2.0.0->-r gradio_demo/requirements.txt (line 2)) (1.13.0)\n",
+      "Requirement already satisfied: networkx in /opt/conda/lib/python3.10/site-packages (from torch==2.0.0->-r gradio_demo/requirements.txt (line 2)) (3.3)\n",
+      "Requirement already satisfied: jinja2 in /opt/conda/lib/python3.10/site-packages (from torch==2.0.0->-r gradio_demo/requirements.txt (line 2)) (3.1.4)\n",
+      "Requirement already satisfied: triton==2.0.0 in /opt/conda/lib/python3.10/site-packages (from torch==2.0.0->-r gradio_demo/requirements.txt (line 2)) (2.0.0)\n",
+      "Requirement already satisfied: packaging>=20.0 in /opt/conda/lib/python3.10/site-packages (from transformers==4.37.1->-r gradio_demo/requirements.txt (line 4)) (24.1)\n",
+      "Requirement already satisfied: pyyaml>=5.1 in /opt/conda/lib/python3.10/site-packages (from transformers==4.37.1->-r gradio_demo/requirements.txt (line 4)) (6.0.1)\n",
+      "Collecting tokenizers<0.19,>=0.14 (from transformers==4.37.1->-r gradio_demo/requirements.txt (line 4))\n",
+      "  Downloading tokenizers-0.15.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata (6.7 kB)\n",
+      "Requirement already satisfied: tqdm>=4.27 in /opt/conda/lib/python3.10/site-packages (from transformers==4.37.1->-r gradio_demo/requirements.txt (line 4)) (4.66.4)\n",
+      "Requirement already satisfied: httpx>=0.20 in /opt/conda/lib/python3.10/site-packages (from spaces==0.19.4->-r gradio_demo/requirements.txt (line 9)) (0.27.0)\n",
+      "Requirement already satisfied: psutil<6,>=2 in /opt/conda/lib/python3.10/site-packages (from spaces==0.19.4->-r gradio_demo/requirements.txt (line 9)) (5.9.3)\n",
+      "Requirement already satisfied: pydantic<3,>=1 in /opt/conda/lib/python3.10/site-packages (from spaces==0.19.4->-r gradio_demo/requirements.txt (line 9)) (1.10.17)\n",
+      "Requirement already satisfied: fsspec>=2023.5.0 in /opt/conda/lib/python3.10/site-packages (from huggingface-hub==0.20.2->-r gradio_demo/requirements.txt (line 12)) (2024.6.1)\n",
+      "Requirement already satisfied: cmake in /opt/conda/lib/python3.10/site-packages (from triton==2.0.0->torch==2.0.0->-r gradio_demo/requirements.txt (line 2)) (3.30.0)\n",
+      "Requirement already satisfied: lit in /opt/conda/lib/python3.10/site-packages (from triton==2.0.0->torch==2.0.0->-r gradio_demo/requirements.txt (line 2)) (18.1.8)\n",
+      "Collecting coloredlogs (from onnxruntime-gpu->-r gradio_demo/requirements.txt (line 8))\n",
+      "  Downloading coloredlogs-15.0.1-py2.py3-none-any.whl.metadata (12 kB)\n",
+      "Collecting flatbuffers (from onnxruntime-gpu->-r gradio_demo/requirements.txt (line 8))\n",
+      "  Downloading flatbuffers-24.3.25-py2.py3-none-any.whl.metadata (850 bytes)\n",
+      "Requirement already satisfied: protobuf in /opt/conda/lib/python3.10/site-packages (from onnxruntime-gpu->-r gradio_demo/requirements.txt (line 8)) (3.20.3)\n",
+      "Collecting antlr4-python3-runtime==4.9.* (from omegaconf->-r gradio_demo/requirements.txt (line 10))\n",
+      "  Downloading antlr4-python3-runtime-4.9.3.tar.gz (117 kB)\n",
+      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m117.0/117.0 kB\u001b[0m \u001b[31m14.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
+      "\u001b[?25h  Preparing metadata (setup.py) ... \u001b[?25ldone\n",
+      "\u001b[?25hCollecting onnx (from insightface->-r gradio_demo/requirements.txt (line 14))\n",
+      "  Downloading onnx-1.16.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata (16 kB)\n",
+      "Requirement already satisfied: matplotlib in /opt/conda/lib/python3.10/site-packages (from insightface->-r gradio_demo/requirements.txt (line 14)) (3.7.3)\n",
+      "Requirement already satisfied: scipy in /opt/conda/lib/python3.10/site-packages (from insightface->-r gradio_demo/requirements.txt (line 14)) (1.11.4)\n",
+      "Requirement already satisfied: scikit-learn in /opt/conda/lib/python3.10/site-packages (from insightface->-r gradio_demo/requirements.txt (line 14)) (1.5.1)\n",
+      "Requirement already satisfied: scikit-image in /opt/conda/lib/python3.10/site-packages (from insightface->-r gradio_demo/requirements.txt (line 14)) (0.24.0)\n",
+      "Collecting easydict (from insightface->-r gradio_demo/requirements.txt (line 14))\n",
+      "  Downloading easydict-1.13-py3-none-any.whl.metadata (4.2 kB)\n",
+      "Requirement already satisfied: cython in /opt/conda/lib/python3.10/site-packages (from insightface->-r gradio_demo/requirements.txt (line 14)) (3.0.10)\n",
+      "Collecting albumentations (from insightface->-r gradio_demo/requirements.txt (line 14))\n",
+      "  Downloading albumentations-1.4.11-py3-none-any.whl.metadata (39 kB)\n",
+      "Requirement already satisfied: prettytable in /opt/conda/lib/python3.10/site-packages (from insightface->-r gradio_demo/requirements.txt (line 14)) (3.10.0)\n",
+      "Requirement already satisfied: aiofiles<24.0,>=22.0 in /opt/conda/lib/python3.10/site-packages (from gradio->-r gradio_demo/requirements.txt (line 15)) (22.1.0)\n",
+      "Collecting altair<6.0,>=5.0 (from gradio->-r gradio_demo/requirements.txt (line 15))\n",
+      "  Downloading altair-5.3.0-py3-none-any.whl.metadata (9.2 kB)\n",
+      "Requirement already satisfied: fastapi in /opt/conda/lib/python3.10/site-packages (from gradio->-r gradio_demo/requirements.txt (line 15)) (0.111.0)\n",
+      "Collecting ffmpy (from gradio->-r gradio_demo/requirements.txt (line 15))\n",
+      "  Downloading ffmpy-0.3.2.tar.gz (5.5 kB)\n",
+      "  Preparing metadata (setup.py) ... \u001b[?25ldone\n",
+      "\u001b[?25hCollecting gradio-client==1.1.0 (from gradio->-r gradio_demo/requirements.txt (line 15))\n",
+      "  Downloading gradio_client-1.1.0-py3-none-any.whl.metadata (7.1 kB)\n",
+      "Requirement already satisfied: importlib-resources<7.0,>=1.3 in /opt/conda/lib/python3.10/site-packages (from gradio->-r gradio_demo/requirements.txt (line 15)) (6.4.0)\n",
+      "Requirement already satisfied: markupsafe~=2.0 in /opt/conda/lib/python3.10/site-packages (from gradio->-r gradio_demo/requirements.txt (line 15)) (2.1.5)\n",
+      "Requirement already satisfied: orjson~=3.0 in /opt/conda/lib/python3.10/site-packages (from gradio->-r gradio_demo/requirements.txt (line 15)) (3.10.6)\n",
+      "Requirement already satisfied: pandas<3.0,>=1.0 in /opt/conda/lib/python3.10/site-packages (from gradio->-r gradio_demo/requirements.txt (line 15)) (2.0.3)\n",
+      "Collecting pydantic<3,>=1 (from spaces==0.19.4->-r gradio_demo/requirements.txt (line 9))\n",
+      "  Downloading pydantic-2.8.2-py3-none-any.whl.metadata (125 kB)\n",
+      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m125.2/125.2 kB\u001b[0m \u001b[31m16.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
+      "\u001b[?25hCollecting pydub (from gradio->-r gradio_demo/requirements.txt (line 15))\n",
+      "  Downloading pydub-0.25.1-py2.py3-none-any.whl.metadata (1.4 kB)\n",
+      "Requirement already satisfied: python-multipart>=0.0.9 in /opt/conda/lib/python3.10/site-packages (from gradio->-r gradio_demo/requirements.txt (line 15)) (0.0.9)\n",
+      "Collecting ruff>=0.2.2 (from gradio->-r gradio_demo/requirements.txt (line 15))\n",
+      "  Downloading ruff-0.5.4-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata (24 kB)\n",
+      "Collecting semantic-version~=2.0 (from gradio->-r gradio_demo/requirements.txt (line 15))\n",
+      "  Downloading semantic_version-2.10.0-py2.py3-none-any.whl.metadata (9.7 kB)\n",
+      "Collecting tomlkit==0.12.0 (from gradio->-r gradio_demo/requirements.txt (line 15))\n",
+      "  Downloading tomlkit-0.12.0-py3-none-any.whl.metadata (2.7 kB)\n",
+      "Requirement already satisfied: typer<1.0,>=0.12 in /opt/conda/lib/python3.10/site-packages (from gradio->-r gradio_demo/requirements.txt (line 15)) (0.12.3)\n",
+      "Collecting urllib3~=2.0 (from gradio->-r gradio_demo/requirements.txt (line 15))\n",
+      "  Downloading urllib3-2.2.2-py3-none-any.whl.metadata (6.4 kB)\n",
+      "Requirement already satisfied: uvicorn>=0.14.0 in /opt/conda/lib/python3.10/site-packages (from gradio->-r gradio_demo/requirements.txt (line 15)) (0.30.1)\n",
+      "Collecting websockets<12.0,>=10.0 (from gradio-client==1.1.0->gradio->-r gradio_demo/requirements.txt (line 15))\n",
+      "  Downloading websockets-11.0.3-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata (6.6 kB)\n",
+      "Collecting opencv-python-headless (from controlnet_aux->-r gradio_demo/requirements.txt (line 16))\n",
+      "  Downloading opencv_python_headless-4.10.0.84-cp37-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata (20 kB)\n",
+      "Collecting timm<=0.6.7 (from controlnet_aux->-r gradio_demo/requirements.txt (line 16))\n",
+      "  Downloading timm-0.6.7-py3-none-any.whl.metadata (33 kB)\n",
+      "Requirement already satisfied: beautifulsoup4 in /opt/conda/lib/python3.10/site-packages (from gdown->-r gradio_demo/requirements.txt (line 17)) (4.12.3)\n",
+      "Requirement already satisfied: jsonschema>=3.0 in /opt/conda/lib/python3.10/site-packages (from altair<6.0,>=5.0->gradio->-r gradio_demo/requirements.txt (line 15)) (4.22.0)\n",
+      "Requirement already satisfied: toolz in /opt/conda/lib/python3.10/site-packages (from altair<6.0,>=5.0->gradio->-r gradio_demo/requirements.txt (line 15)) (0.12.1)\n",
+      "Requirement already satisfied: anyio in /opt/conda/lib/python3.10/site-packages (from httpx>=0.20->spaces==0.19.4->-r gradio_demo/requirements.txt (line 9)) (4.4.0)\n",
+      "Requirement already satisfied: certifi in /opt/conda/lib/python3.10/site-packages (from httpx>=0.20->spaces==0.19.4->-r gradio_demo/requirements.txt (line 9)) (2024.7.4)\n",
+      "Requirement already satisfied: httpcore==1.* in /opt/conda/lib/python3.10/site-packages (from httpx>=0.20->spaces==0.19.4->-r gradio_demo/requirements.txt (line 9)) (1.0.5)\n",
+      "Requirement already satisfied: idna in /opt/conda/lib/python3.10/site-packages (from httpx>=0.20->spaces==0.19.4->-r gradio_demo/requirements.txt (line 9)) (3.7)\n",
+      "Requirement already satisfied: sniffio in /opt/conda/lib/python3.10/site-packages (from httpx>=0.20->spaces==0.19.4->-r gradio_demo/requirements.txt (line 9)) (1.3.1)\n",
+      "Requirement already satisfied: h11<0.15,>=0.13 in /opt/conda/lib/python3.10/site-packages (from httpcore==1.*->httpx>=0.20->spaces==0.19.4->-r gradio_demo/requirements.txt (line 9)) (0.14.0)\n",
+      "Requirement already satisfied: contourpy>=1.0.1 in /opt/conda/lib/python3.10/site-packages (from matplotlib->insightface->-r gradio_demo/requirements.txt (line 14)) (1.2.1)\n",
+      "Requirement already satisfied: cycler>=0.10 in /opt/conda/lib/python3.10/site-packages (from matplotlib->insightface->-r gradio_demo/requirements.txt (line 14)) (0.12.1)\n",
+      "Requirement already satisfied: fonttools>=4.22.0 in /opt/conda/lib/python3.10/site-packages (from matplotlib->insightface->-r gradio_demo/requirements.txt (line 14)) (4.53.1)\n",
+      "Requirement already satisfied: kiwisolver>=1.0.1 in /opt/conda/lib/python3.10/site-packages (from matplotlib->insightface->-r gradio_demo/requirements.txt (line 14)) (1.4.5)\n",
+      "Requirement already satisfied: pyparsing>=2.3.1 in /opt/conda/lib/python3.10/site-packages (from matplotlib->insightface->-r gradio_demo/requirements.txt (line 14)) (3.1.2)\n",
+      "Requirement already satisfied: python-dateutil>=2.7 in /opt/conda/lib/python3.10/site-packages (from matplotlib->insightface->-r gradio_demo/requirements.txt (line 14)) (2.9.0)\n",
+      "Requirement already satisfied: pytz>=2020.1 in /opt/conda/lib/python3.10/site-packages (from pandas<3.0,>=1.0->gradio->-r gradio_demo/requirements.txt (line 15)) (2024.1)\n",
+      "Requirement already satisfied: tzdata>=2022.1 in /opt/conda/lib/python3.10/site-packages (from pandas<3.0,>=1.0->gradio->-r gradio_demo/requirements.txt (line 15)) (2024.1)\n",
+      "Collecting annotated-types>=0.4.0 (from pydantic<3,>=1->spaces==0.19.4->-r gradio_demo/requirements.txt (line 9))\n",
+      "  Downloading annotated_types-0.7.0-py3-none-any.whl.metadata (15 kB)\n",
+      "Collecting pydantic-core==2.20.1 (from pydantic<3,>=1->spaces==0.19.4->-r gradio_demo/requirements.txt (line 9))\n",
+      "  Downloading pydantic_core-2.20.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata (6.6 kB)\n",
+      "Requirement already satisfied: charset-normalizer<4,>=2 in /opt/conda/lib/python3.10/site-packages (from requests->diffusers==0.25.1->-r gradio_demo/requirements.txt (line 1)) (3.3.2)\n",
+      "Requirement already satisfied: click>=8.0.0 in /opt/conda/lib/python3.10/site-packages (from typer<1.0,>=0.12->gradio->-r gradio_demo/requirements.txt (line 15)) (8.1.7)\n",
+      "Requirement already satisfied: shellingham>=1.3.0 in /opt/conda/lib/python3.10/site-packages (from typer<1.0,>=0.12->gradio->-r gradio_demo/requirements.txt (line 15)) (1.5.4)\n",
+      "Requirement already satisfied: rich>=10.11.0 in /opt/conda/lib/python3.10/site-packages (from typer<1.0,>=0.12->gradio->-r gradio_demo/requirements.txt (line 15)) (13.7.1)\n",
+      "Collecting albucore>=0.0.11 (from albumentations->insightface->-r gradio_demo/requirements.txt (line 14))\n",
+      "  Downloading albucore-0.0.12-py3-none-any.whl.metadata (3.1 kB)\n",
+      "Collecting eval-type-backport (from albumentations->insightface->-r gradio_demo/requirements.txt (line 14))\n",
+      "  Downloading eval_type_backport-0.2.0-py3-none-any.whl.metadata (2.2 kB)\n",
+      "Requirement already satisfied: imageio>=2.33 in /opt/conda/lib/python3.10/site-packages (from scikit-image->insightface->-r gradio_demo/requirements.txt (line 14)) (2.34.2)\n",
+      "Requirement already satisfied: tifffile>=2022.8.12 in /opt/conda/lib/python3.10/site-packages (from scikit-image->insightface->-r gradio_demo/requirements.txt (line 14)) (2024.7.2)\n",
+      "Requirement already satisfied: lazy-loader>=0.4 in /opt/conda/lib/python3.10/site-packages (from scikit-image->insightface->-r gradio_demo/requirements.txt (line 14)) (0.4)\n",
+      "Requirement already satisfied: joblib>=1.2.0 in /opt/conda/lib/python3.10/site-packages (from scikit-learn->insightface->-r gradio_demo/requirements.txt (line 14)) (1.4.2)\n",
+      "Requirement already satisfied: threadpoolctl>=3.1.0 in /opt/conda/lib/python3.10/site-packages (from scikit-learn->insightface->-r gradio_demo/requirements.txt (line 14)) (3.5.0)\n",
+      "Requirement already satisfied: soupsieve>1.2 in /opt/conda/lib/python3.10/site-packages (from beautifulsoup4->gdown->-r gradio_demo/requirements.txt (line 17)) (2.5)\n",
+      "Collecting humanfriendly>=9.1 (from coloredlogs->onnxruntime-gpu->-r gradio_demo/requirements.txt (line 8))\n",
+      "  Downloading humanfriendly-10.0-py2.py3-none-any.whl.metadata (9.2 kB)\n",
+      "Requirement already satisfied: starlette<0.38.0,>=0.37.2 in /opt/conda/lib/python3.10/site-packages (from fastapi->gradio->-r gradio_demo/requirements.txt (line 15)) (0.37.2)\n",
+      "Requirement already satisfied: fastapi-cli>=0.0.2 in /opt/conda/lib/python3.10/site-packages (from fastapi->gradio->-r gradio_demo/requirements.txt (line 15)) (0.0.4)\n",
+      "Requirement already satisfied: ujson!=4.0.2,!=4.1.0,!=4.2.0,!=4.3.0,!=5.0.0,!=5.1.0,>=4.0.1 in /opt/conda/lib/python3.10/site-packages (from fastapi->gradio->-r gradio_demo/requirements.txt (line 15)) (5.10.0)\n",
+      "Requirement already satisfied: email_validator>=2.0.0 in /opt/conda/lib/python3.10/site-packages (from fastapi->gradio->-r gradio_demo/requirements.txt (line 15)) (2.2.0)\n",
+      "Requirement already satisfied: zipp>=0.5 in /opt/conda/lib/python3.10/site-packages (from importlib-metadata->diffusers==0.25.1->-r gradio_demo/requirements.txt (line 1)) (3.19.2)\n",
+      "Requirement already satisfied: wcwidth in /opt/conda/lib/python3.10/site-packages (from prettytable->insightface->-r gradio_demo/requirements.txt (line 14)) (0.2.13)\n",
+      "Requirement already satisfied: PySocks!=1.5.7,>=1.5.6 in /opt/conda/lib/python3.10/site-packages (from requests[socks]->gdown->-r gradio_demo/requirements.txt (line 17)) (1.7.1)\n",
+      "Requirement already satisfied: mpmath<1.4,>=1.1.0 in /opt/conda/lib/python3.10/site-packages (from sympy->torch==2.0.0->-r gradio_demo/requirements.txt (line 2)) (1.3.0)\n",
+      "Requirement already satisfied: tomli>=2.0.1 in /opt/conda/lib/python3.10/site-packages (from albucore>=0.0.11->albumentations->insightface->-r gradio_demo/requirements.txt (line 14)) (2.0.1)\n",
+      "Requirement already satisfied: dnspython>=2.0.0 in /opt/conda/lib/python3.10/site-packages (from email_validator>=2.0.0->fastapi->gradio->-r gradio_demo/requirements.txt (line 15)) (2.6.1)\n",
+      "Requirement already satisfied: attrs>=22.2.0 in /opt/conda/lib/python3.10/site-packages (from jsonschema>=3.0->altair<6.0,>=5.0->gradio->-r gradio_demo/requirements.txt (line 15)) (23.2.0)\n",
+      "Requirement already satisfied: jsonschema-specifications>=2023.03.6 in /opt/conda/lib/python3.10/site-packages (from jsonschema>=3.0->altair<6.0,>=5.0->gradio->-r gradio_demo/requirements.txt (line 15)) (2023.12.1)\n",
+      "Requirement already satisfied: referencing>=0.28.4 in /opt/conda/lib/python3.10/site-packages (from jsonschema>=3.0->altair<6.0,>=5.0->gradio->-r gradio_demo/requirements.txt (line 15)) (0.35.1)\n",
+      "Requirement already satisfied: rpds-py>=0.7.1 in /opt/conda/lib/python3.10/site-packages (from jsonschema>=3.0->altair<6.0,>=5.0->gradio->-r gradio_demo/requirements.txt (line 15)) (0.19.0)\n",
+      "Requirement already satisfied: six>=1.5 in /opt/conda/lib/python3.10/site-packages (from python-dateutil>=2.7->matplotlib->insightface->-r gradio_demo/requirements.txt (line 14)) (1.16.0)\n",
+      "Requirement already satisfied: markdown-it-py>=2.2.0 in /opt/conda/lib/python3.10/site-packages (from rich>=10.11.0->typer<1.0,>=0.12->gradio->-r gradio_demo/requirements.txt (line 15)) (3.0.0)\n",
+      "Requirement already satisfied: pygments<3.0.0,>=2.13.0 in /opt/conda/lib/python3.10/site-packages (from rich>=10.11.0->typer<1.0,>=0.12->gradio->-r gradio_demo/requirements.txt (line 15)) (2.18.0)\n",
+      "Requirement already satisfied: exceptiongroup>=1.0.2 in /opt/conda/lib/python3.10/site-packages (from anyio->httpx>=0.20->spaces==0.19.4->-r gradio_demo/requirements.txt (line 9)) (1.2.0)\n",
+      "Requirement already satisfied: httptools>=0.5.0 in /opt/conda/lib/python3.10/site-packages (from uvicorn[standard]>=0.12.0->fastapi->gradio->-r gradio_demo/requirements.txt (line 15)) (0.6.1)\n",
+      "Requirement already satisfied: python-dotenv>=0.13 in /opt/conda/lib/python3.10/site-packages (from uvicorn[standard]>=0.12.0->fastapi->gradio->-r gradio_demo/requirements.txt (line 15)) (1.0.1)\n",
+      "Requirement already satisfied: uvloop!=0.15.0,!=0.15.1,>=0.14.0 in /opt/conda/lib/python3.10/site-packages (from uvicorn[standard]>=0.12.0->fastapi->gradio->-r gradio_demo/requirements.txt (line 15)) (0.19.0)\n",
+      "Requirement already satisfied: watchfiles>=0.13 in /opt/conda/lib/python3.10/site-packages (from uvicorn[standard]>=0.12.0->fastapi->gradio->-r gradio_demo/requirements.txt (line 15)) (0.22.0)\n",
+      "Requirement already satisfied: mdurl~=0.1 in /opt/conda/lib/python3.10/site-packages (from markdown-it-py>=2.2.0->rich>=10.11.0->typer<1.0,>=0.12->gradio->-r gradio_demo/requirements.txt (line 15)) (0.1.2)\n",
+      "Downloading diffusers-0.25.1-py3-none-any.whl (1.8 MB)\n",
+      "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━���━━━\u001b[0m \u001b[32m1.8/1.8 MB\u001b[0m \u001b[31m33.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0ma \u001b[36m0:00:01\u001b[0m\n",
+      "\u001b[?25hDownloading transformers-4.37.1-py3-none-any.whl (8.4 MB)\n",
+      "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m8.4/8.4 MB\u001b[0m \u001b[31m87.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m:00:01\u001b[0m00:01\u001b[0m\n",
+      "\u001b[?25hDownloading spaces-0.19.4-py3-none-any.whl (15 kB)\n",
+      "Downloading huggingface_hub-0.20.2-py3-none-any.whl (330 kB)\n",
+      "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m330.3/330.3 kB\u001b[0m \u001b[31m37.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
+      "\u001b[?25hDownloading accelerate-0.32.1-py3-none-any.whl (314 kB)\n",
+      "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m314.1/314.1 kB\u001b[0m \u001b[31m36.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
+      "\u001b[?25hDownloading safetensors-0.4.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (1.2 MB)\n",
+      "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.2/1.2 MB\u001b[0m \u001b[31m74.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
+      "\u001b[?25hDownloading einops-0.8.0-py3-none-any.whl (43 kB)\n",
+      "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m43.2/43.2 kB\u001b[0m \u001b[31m6.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
+      "\u001b[?25hDownloading onnxruntime_gpu-1.18.1-cp310-cp310-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl (200.8 MB)\n",
+      "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m200.8/200.8 MB\u001b[0m \u001b[31m5.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m:00:01\u001b[0m00:01\u001b[0m\n",
+      "\u001b[?25hDownloading omegaconf-2.3.0-py3-none-any.whl (79 kB)\n",
+      "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m79.5/79.5 kB\u001b[0m \u001b[31m133.1 kB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0ma \u001b[36m0:00:01\u001b[0m\n",
+      "\u001b[?25hDownloading peft-0.11.1-py3-none-any.whl (251 kB)\n",
+      "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m251.6/251.6 kB\u001b[0m \u001b[31m3.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m:00:01\u001b[0m\n",
+      "\u001b[?25hDownloading opencv_python-4.10.0.84-cp37-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (62.5 MB)\n",
+      "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m62.5/62.5 MB\u001b[0m \u001b[31m21.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m:00:01\u001b[0m00:01\u001b[0m\n",
+      "\u001b[?25hDownloading gradio-4.38.1-py3-none-any.whl (12.4 MB)\n",
+      "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m12.4/12.4 MB\u001b[0m \u001b[31m84.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m:00:01\u001b[0m0:01\u001b[0m\n",
+      "\u001b[?25hDownloading gradio_client-1.1.0-py3-none-any.whl (318 kB)\n",
+      "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m318.1/318.1 kB\u001b[0m \u001b[31m38.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
+      "\u001b[?25hDownloading tomlkit-0.12.0-py3-none-any.whl (37 kB)\n",
+      "Downloading controlnet_aux-0.0.9-py3-none-any.whl (282 kB)\n",
+      "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m282.4/282.4 kB\u001b[0m \u001b[31m31.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
+      "\u001b[?25hDownloading gdown-5.2.0-py3-none-any.whl (18 kB)\n",
+      "Downloading altair-5.3.0-py3-none-any.whl (857 kB)\n",
+      "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m857.8/857.8 kB\u001b[0m \u001b[31m64.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
+      "\u001b[?25hDownloading pydantic-2.8.2-py3-none-any.whl (423 kB)\n",
+      "\u001b[2K   \u001b[90m━━━━━━━━━━━���━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m423.9/423.9 kB\u001b[0m \u001b[31m46.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
+      "\u001b[?25hDownloading pydantic_core-2.20.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (2.1 MB)\n",
+      "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m2.1/2.1 MB\u001b[0m \u001b[31m101.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
+      "\u001b[?25hDownloading regex-2024.5.15-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (775 kB)\n",
+      "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m775.1/775.1 kB\u001b[0m \u001b[31m58.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
+      "\u001b[?25hDownloading ruff-0.5.4-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (10.1 MB)\n",
+      "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m10.1/10.1 MB\u001b[0m \u001b[31m82.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m:00:01\u001b[0m0:01\u001b[0m\n",
+      "\u001b[?25hDownloading semantic_version-2.10.0-py2.py3-none-any.whl (15 kB)\n",
+      "Downloading timm-0.6.7-py3-none-any.whl (509 kB)\n",
+      "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m510.0/510.0 kB\u001b[0m \u001b[31m45.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
+      "\u001b[?25hDownloading tokenizers-0.15.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (3.6 MB)\n",
+      "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m3.6/3.6 MB\u001b[0m \u001b[31m90.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m:00:01\u001b[0m\n",
+      "\u001b[?25hDownloading urllib3-2.2.2-py3-none-any.whl (121 kB)\n",
+      "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m121.4/121.4 kB\u001b[0m \u001b[31m18.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
+      "\u001b[?25hDownloading albumentations-1.4.11-py3-none-any.whl (165 kB)\n",
+      "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m165.3/165.3 kB\u001b[0m \u001b[31m23.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
+      "\u001b[?25hDownloading opencv_python_headless-4.10.0.84-cp37-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (49.9 MB)\n",
+      "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m49.9/49.9 MB\u001b[0m \u001b[31m27.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m:00:01\u001b[0m00:01\u001b[0m\n",
+      "\u001b[?25hDownloading coloredlogs-15.0.1-py2.py3-none-any.whl (46 kB)\n",
+      "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m46.0/46.0 kB\u001b[0m \u001b[31m6.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
+      "\u001b[?25hDownloading easydict-1.13-py3-none-any.whl (6.8 kB)\n",
+      "Downloading flatbuffers-24.3.25-py2.py3-none-any.whl (26 kB)\n",
+      "Downloading onnx-1.16.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (15.9 MB)\n",
+      "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m15.9/15.9 MB\u001b[0m \u001b[31m80.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m:00:01\u001b[0m00:01\u001b[0m\n",
+      "\u001b[?25hDownloading pydub-0.25.1-py2.py3-none-any.whl (32 kB)\n",
+      "Downloading albucore-0.0.12-py3-none-any.whl (8.4 kB)\n",
+      "Downloading annotated_types-0.7.0-py3-none-any.whl (13 kB)\n",
+      "Downloading humanfriendly-10.0-py2.py3-none-any.whl (86 kB)\n",
+      "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m86.8/86.8 kB\u001b[0m \u001b[31m13.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
+      "\u001b[?25hDownloading websockets-11.0.3-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl (129 kB)\n",
+      "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m129.9/129.9 kB\u001b[0m \u001b[31m18.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
+      "\u001b[?25hDownloading eval_type_backport-0.2.0-py3-none-any.whl (5.9 kB)\n",
+      "Building wheels for collected packages: antlr4-python3-runtime, insightface, ffmpy\n",
+      "  Building wheel for antlr4-python3-runtime (setup.py) ... \u001b[?25ldone\n",
+      "\u001b[?25h  Created wheel for antlr4-python3-runtime: filename=antlr4_python3_runtime-4.9.3-py3-none-any.whl size=144552 sha256=e9ae543340a864dee947980bab8fc7d8fc3b8a5a04b28963252f790444a5cd1f\n",
+      "  Stored in directory: /home/jupyter/.cache/pip/wheels/12/93/dd/1f6a127edc45659556564c5730f6d4e300888f4bca2d4c5a88\n",
+      "  Building wheel for insightface (pyproject.toml) ... \u001b[?25ldone\n",
+      "\u001b[?25h  Created wheel for insightface: filename=insightface-0.7.3-cp310-cp310-linux_x86_64.whl size=874168 sha256=f381a87957a87ca37e1795c8ba2b854664428cf5117760cc1f7d58368918523b\n",
+      "  Stored in directory: /home/jupyter/.cache/pip/wheels/e3/d0/80/e3773fb8b6d1cca87ea1d33d9b1f20a223a6493c896da249b5\n",
+      "  Building wheel for ffmpy (setup.py) ... \u001b[?25ldone\n",
+      "\u001b[?25h  Created wheel for ffmpy: filename=ffmpy-0.3.2-py3-none-any.whl size=5581 sha256=d671b217ecfc883cea0aa0408a98e3d187bd0e888ba4e85318ea4b8bfa539786\n",
+      "  Stored in directory: /home/jupyter/.cache/pip/wheels/bd/65/9a/671fc6dcde07d4418df0c592f8df512b26d7a0029c2a23dd81\n",
+      "Successfully built antlr4-python3-runtime insightface ffmpy\n",
+      "Installing collected packages: pydub, flatbuffers, ffmpy, easydict, antlr4-python3-runtime, websockets, urllib3, tomlkit, semantic-version, safetensors, ruff, regex, pydantic-core, opencv-python-headless, opencv-python, onnx, omegaconf, humanfriendly, eval-type-backport, einops, annotated-types, pydantic, coloredlogs, albucore, onnxruntime-gpu, huggingface-hub, albumentations, tokenizers, insightface, gradio-client, gdown, diffusers, altair, transformers, gradio, spaces, timm, accelerate, peft, controlnet_aux\n",
+      "  Attempting uninstall: websockets\n",
+      "    Found existing installation: websockets 12.0\n",
+      "    Uninstalling websockets-12.0:\n",
+      "      Successfully uninstalled websockets-12.0\n",
+      "  Attempting uninstall: urllib3\n",
+      "    Found existing installation: urllib3 1.26.19\n",
+      "    Uninstalling urllib3-1.26.19:\n",
+      "      Successfully uninstalled urllib3-1.26.19\n",
+      "  Attempting uninstall: pydantic\n",
+      "    Found existing installation: pydantic 1.10.17\n",
+      "    Uninstalling pydantic-1.10.17:\n",
+      "      Successfully uninstalled pydantic-1.10.17\n",
+      "\u001b[31mERROR: pip's dependency resolver does not currently take into account all the packages that are installed. This behaviour is the source of the following dependency conflicts.\n",
+      "dataproc-jupyter-plugin 0.1.79 requires pydantic~=1.10.0, but you have pydantic 2.8.2 which is incompatible.\n",
+      "kfp 2.5.0 requires urllib3<2.0.0, but you have urllib3 2.2.2 which is incompatible.\n",
+      "ydata-profiling 4.6.0 requires pydantic<2,>=1.8.1, but you have pydantic 2.8.2 which is incompatible.\u001b[0m\u001b[31m\n",
+      "\u001b[0mSuccessfully installed accelerate-0.32.1 albucore-0.0.12 albumentations-1.4.11 altair-5.3.0 annotated-types-0.7.0 antlr4-python3-runtime-4.9.3 coloredlogs-15.0.1 controlnet_aux-0.0.9 diffusers-0.25.1 easydict-1.13 einops-0.8.0 eval-type-backport-0.2.0 ffmpy-0.3.2 flatbuffers-24.3.25 gdown-5.2.0 gradio-4.38.1 gradio-client-1.1.0 huggingface-hub-0.20.2 humanfriendly-10.0 insightface-0.7.3 omegaconf-2.3.0 onnx-1.16.1 onnxruntime-gpu-1.18.1 opencv-python-4.10.0.84 opencv-python-headless-4.10.0.84 peft-0.11.1 pydantic-2.8.2 pydantic-core-2.20.1 pydub-0.25.1 regex-2024.5.15 ruff-0.5.4 safetensors-0.4.3 semantic-version-2.10.0 spaces-0.19.4 timm-0.6.7 tokenizers-0.15.2 tomlkit-0.12.0 transformers-4.37.1 urllib3-2.2.2 websockets-11.0.3\n"
+     ]
+    }
+   ],
+   "source": [
+    "!pip install -r gradio_demo/requirements.txt"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "id": "dec146ca-0832-4c71-8b31-1586af435d67",
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "ControlNetModel/config.json: 100%|█████████| 1.38k/1.38k [00:00<00:00, 7.07MB/s]\n",
+      "diffusion_pytorch_model.safetensors: 100%|██| 2.50G/2.50G [00:05<00:00, 449MB/s]\n",
+      "ip-adapter.bin: 100%|███████████████████████| 1.69G/1.69G [00:09<00:00, 180MB/s]\n",
+      "pytorch_lora_weights.safetensors: 100%|███████| 394M/394M [00:02<00:00, 169MB/s]\n",
+      "Downloading...\n",
+      "From (original): https://drive.google.com/uc?id=18wEUfMNohBJ4K3Ly5wpTejPfDzp-8fI8\n",
+      "From (redirected): https://drive.google.com/uc?id=18wEUfMNohBJ4K3Ly5wpTejPfDzp-8fI8&confirm=t&uuid=abca1ed1-5c28-423b-a1c7-4fe1fa0d4dbc\n",
+      "To: /home/jupyter/InstantID/models/antelopev2.zip\n",
+      "100%|████████████████████████████████████████| 361M/361M [00:21<00:00, 16.9MB/s]\n",
+      "Archive:  ./models/antelopev2.zip\n",
+      "   creating: ./models/antelopev2/\n",
+      "  inflating: ./models/antelopev2/genderage.onnx  \n",
+      "  inflating: ./models/antelopev2/2d106det.onnx  \n",
+      "  inflating: ./models/antelopev2/1k3d68.onnx  \n",
+      "  inflating: ./models/antelopev2/glintr100.onnx  \n",
+      "  inflating: ./models/antelopev2/scrfd_10g_bnkps.onnx  \n"
+     ]
+    }
+   ],
+   "source": [
+    "!python gradio_demo/download_models.py"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "id": "e4900619-4519-4ec9-bb32-a620128d1727",
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Collecting setuptools==69.5.1\n",
+      "  Downloading setuptools-69.5.1-py3-none-any.whl.metadata (6.2 kB)\n",
+      "Downloading setuptools-69.5.1-py3-none-any.whl (894 kB)\n",
+      "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m894.6/894.6 kB\u001b[0m \u001b[31m7.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m00:01\u001b[0m00:01\u001b[0m\n",
+      "\u001b[?25hInstalling collected packages: setuptools\n",
+      "  Attempting uninstall: setuptools\n",
+      "    Found existing installation: setuptools 70.1.1\n",
+      "    Uninstalling setuptools-70.1.1:\n",
+      "      Successfully uninstalled setuptools-70.1.1\n",
+      "Successfully installed setuptools-69.5.1\n"
+     ]
+    }
+   ],
+   "source": [
+    "!pip install setuptools==69.5.1"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 6,
+   "id": "9584f180-48a2-46bc-968d-9c99bc56f06c",
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Collecting huggingface-hub==0.23.4\n",
+      "  Downloading huggingface_hub-0.23.4-py3-none-any.whl.metadata (12 kB)\n",
+      "Requirement already satisfied: filelock in /opt/conda/lib/python3.10/site-packages (from huggingface-hub==0.23.4) (3.15.4)\n",
+      "Requirement already satisfied: fsspec>=2023.5.0 in /opt/conda/lib/python3.10/site-packages (from huggingface-hub==0.23.4) (2024.6.1)\n",
+      "Requirement already satisfied: packaging>=20.9 in /opt/conda/lib/python3.10/site-packages (from huggingface-hub==0.23.4) (24.1)\n",
+      "Requirement already satisfied: pyyaml>=5.1 in /opt/conda/lib/python3.10/site-packages (from huggingface-hub==0.23.4) (6.0.1)\n",
+      "Requirement already satisfied: requests in /opt/conda/lib/python3.10/site-packages (from huggingface-hub==0.23.4) (2.32.3)\n",
+      "Requirement already satisfied: tqdm>=4.42.1 in /opt/conda/lib/python3.10/site-packages (from huggingface-hub==0.23.4) (4.66.4)\n",
+      "Requirement already satisfied: typing-extensions>=3.7.4.3 in /opt/conda/lib/python3.10/site-packages (from huggingface-hub==0.23.4) (4.12.2)\n",
+      "Requirement already satisfied: charset-normalizer<4,>=2 in /opt/conda/lib/python3.10/site-packages (from requests->huggingface-hub==0.23.4) (3.3.2)\n",
+      "Requirement already satisfied: idna<4,>=2.5 in /opt/conda/lib/python3.10/site-packages (from requests->huggingface-hub==0.23.4) (3.7)\n",
+      "Requirement already satisfied: urllib3<3,>=1.21.1 in /opt/conda/lib/python3.10/site-packages (from requests->huggingface-hub==0.23.4) (2.2.2)\n",
+      "Requirement already satisfied: certifi>=2017.4.17 in /opt/conda/lib/python3.10/site-packages (from requests->huggingface-hub==0.23.4) (2024.7.4)\n",
+      "Downloading huggingface_hub-0.23.4-py3-none-any.whl (402 kB)\n",
+      "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m402.6/402.6 kB\u001b[0m \u001b[31m3.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0ma \u001b[36m0:00:01\u001b[0m\n",
+      "\u001b[?25hInstalling collected packages: huggingface-hub\n",
+      "  Attempting uninstall: huggingface-hub\n",
+      "    Found existing installation: huggingface-hub 0.20.2\n",
+      "    Uninstalling huggingface-hub-0.20.2:\n",
+      "      Successfully uninstalled huggingface-hub-0.20.2\n",
+      "Successfully installed huggingface-hub-0.23.4\n"
+     ]
+    }
+   ],
+   "source": [
+    "!pip install huggingface-hub==0.23.4"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "56683081-5e9f-4378-84df-b957c84b23ad",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  }
+ ],
+ "metadata": {
+  "environment": {
+   "kernel": "python3",
+   "name": ".m123",
+   "type": "gcloud",
+   "uri": "us-docker.pkg.dev/deeplearning-platform-release/gcr.io/:m123"
+  },
+  "kernelspec": {
+   "display_name": "Python 3 (Local)",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.10.14"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}

checkpoints/ControlNetModel/config.json

@@ -0,0 +1,57 @@
+{
+  "_class_name": "ControlNetModel",
+  "_diffusers_version": "0.21.2",
+  "_name_or_path": "/mnt/nj-aigc/usr/guiwan/workspace/diffusion_output/face_xl_ipc_v4_2_XiezhenAnimeForeigner/checkpoint-150000/ControlNetModel",
+  "act_fn": "silu",
+  "addition_embed_type": "text_time",
+  "addition_embed_type_num_heads": 64,
+  "addition_time_embed_dim": 256,
+  "attention_head_dim": [
+    5,
+    10,
+    20
+  ],
+  "block_out_channels": [
+    320,
+    640,
+    1280
+  ],
+  "class_embed_type": null,
+  "conditioning_channels": 3,
+  "conditioning_embedding_out_channels": [
+    16,
+    32,
+    96,
+    256
+  ],
+  "controlnet_conditioning_channel_order": "rgb",
+  "cross_attention_dim": 2048,
+  "down_block_types": [
+    "DownBlock2D",
+    "CrossAttnDownBlock2D",
+    "CrossAttnDownBlock2D"
+  ],
+  "downsample_padding": 1,
+  "encoder_hid_dim": null,
+  "encoder_hid_dim_type": null,
+  "flip_sin_to_cos": true,
+  "freq_shift": 0,
+  "global_pool_conditions": false,
+  "in_channels": 4,
+  "layers_per_block": 2,
+  "mid_block_scale_factor": 1,
+  "norm_eps": 1e-05,
+  "norm_num_groups": 32,
+  "num_attention_heads": null,
+  "num_class_embeds": null,
+  "only_cross_attention": false,
+  "projection_class_embeddings_input_dim": 2816,
+  "resnet_time_scale_shift": "default",
+  "transformer_layers_per_block": [
+    1,
+    2,
+    10
+  ],
+  "upcast_attention": null,
+  "use_linear_projection": true
+}

checkpoints/ControlNetModel/diffusion_pytorch_model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c8127be9f174101ebdafee9964d856b49b634435cf6daa396d3f593cf0bbbb05
+size 2502139136

checkpoints/ip-adapter.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:02b3618e36d803784166660520098089a81388e61a93ef8002aa79a5b1c546e1
+size 1691134141

checkpoints/pytorch_lora_weights.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a764e6859b6e04047cd761c08ff0cee96413a8e004c9f07707530cd776b19141
+size 393855224

cog.yaml

@@ -0,0 +1,40 @@
+# Configuration for Cog ⚙️
+# Reference: https://github.com/replicate/cog/blob/main/docs/yaml.md
+
+build:
+  # set to true if your model requires a GPU
+  gpu: true
+  cuda: "12.1"
+
+  # a list of ubuntu apt packages to install
+  system_packages:
+    - "libgl1-mesa-glx"
+    - "libglib2.0-0"
+
+  # python version in the form '3.11' or '3.11.4'
+  python_version: "3.11"
+
+  # a list of packages in the format <package-name>==<version>
+  python_packages:
+    - "opencv-python==4.9.0.80"
+    - "transformers==4.37.0"
+    - "accelerate==0.26.1"
+    - "insightface==0.7.3"
+    - "diffusers==0.25.1"
+    - "onnxruntime==1.16.3"
+    - "omegaconf==2.3.0"
+    - "gradio==3.50.2"
+    - "peft==0.8.2"
+    - "transformers==4.37.0"
+    - "controlnet-aux==0.0.7"
+    
+    # fix for pydantic issues in cog
+    # https://github.com/replicate/cog/issues/1623
+    - albumentations==1.4.3
+
+  # commands run after the environment is setup
+  run:
+    - curl -o /usr/local/bin/pget -L "https://github.com/replicate/pget/releases/download/v0.6.0/pget_linux_x86_64" && chmod +x /usr/local/bin/pget
+
+# predict.py defines how predictions are run on your model
+predict: "cog/predict.py:Predictor"

cog/README.md

@@ -0,0 +1,60 @@
+# InstantID Cog Model
+
+[![Replicate](https://replicate.com/zsxkib/instant-id/badge)](https://replicate.com/zsxkib/instant-id)
+
+## Overview
+This repository contains the implementation of [InstantID](https://github.com/InstantID/InstantID) as a [Cog](https://github.com/replicate/cog) model. 
+
+Using [Cog](https://github.com/replicate/cog) allows any users with a GPU to run the model locally easily, without the hassle of downloading weights, installing libraries, or managing CUDA versions. Everything just works.
+
+## Development
+To push your own fork of InstantID to [Replicate](https://replicate.com), follow the [Model Pushing Guide](https://replicate.com/docs/guides/push-a-model).
+
+## Basic Usage
+To make predictions using the model, execute the following command from the root of this project:
+
+```bash
+cog predict \
+-i image=@examples/sam_resize.png \
+-i prompt="analog film photo of a man. faded film, desaturated, 35mm photo, grainy, vignette, vintage, Kodachrome, Lomography, stained, highly detailed, found footage, masterpiece, best quality" \
+-i negative_prompt="nsfw" \
+-i width=680 \
+-i height=680 \
+-i ip_adapter_scale=0.8 \
+-i controlnet_conditioning_scale=0.8 \
+-i num_inference_steps=30 \
+-i guidance_scale=5
+```
+
+<table>
+  <tr>
+    <td>
+      <p align="center">Input</p>
+      <img src="https://replicate.delivery/pbxt/KGy0R72cMwriR9EnCLu6hgVkQNd60mY01mDZAQqcUic9rVw4/musk_resize.jpeg" alt="Sample Input Image" width="90%"/>
+    </td>
+    <td>
+      <p align="center">Output</p>
+      <img src="https://replicate.delivery/pbxt/oGOxXELcLcpaMBeIeffwdxKZAkuzwOzzoxKadjhV8YgQWk8IB/result.jpg" alt="Sample Output Image" width="100%"/>
+    </td>
+  </tr>
+</table>
+
+## Input Parameters
+
+The following table provides details about each input parameter for the `predict` function:
+
+| Parameter                       | Description                        | Default Value                                                                                                  | Range       |
+| ------------------------------- | ---------------------------------- | -------------------------------------------------------------------------------------------------------------- | ----------- |
+| `image`                         | Input image                        | A path to the input image file                                                                                 | Path string |
+| `prompt`                        | Input prompt                       | "analog film photo of a man. faded film, desaturated, 35mm photo, grainy, vignette, vintage, Kodachrome, ... " | String      |
+| `negative_prompt`               | Input Negative Prompt              | (empty string)                                                                                                 | String      |
+| `width`                         | Width of output image              | 640                                                                                                            | 512 - 2048  |
+| `height`                        | Height of output image             | 640                                                                                                            | 512 - 2048  |
+| `ip_adapter_scale`              | Scale for IP adapter               | 0.8                                                                                                            | 0.0 - 1.0   |
+| `controlnet_conditioning_scale` | Scale for ControlNet conditioning  | 0.8                                                                                                            | 0.0 - 1.0   |
+| `num_inference_steps`           | Number of denoising steps          | 30                                                                                                             | 1 - 500     |
+| `guidance_scale`                | Scale for classifier-free guidance | 5                                                                                                              | 1 - 50      |
+
+This table provides a quick reference to understand and modify the inputs for generating predictions using the model.
+
+

cog/predict.py

@@ -0,0 +1,756 @@
+# Prediction interface for Cog ⚙️
+# https://github.com/replicate/cog/blob/main/docs/python.md
+
+import os
+import sys
+
+sys.path.append(os.path.join(os.path.dirname(__file__), ".."))
+sys.path.append(os.path.join(os.path.dirname(__file__), "../gradio_demo"))
+
+import cv2
+import time
+import torch
+import mimetypes
+import subprocess
+import numpy as np
+from typing import List
+from cog import BasePredictor, Input, Path
+
+import PIL
+from PIL import Image
+
+import diffusers
+from diffusers import LCMScheduler
+from diffusers.utils import load_image
+from diffusers.models import ControlNetModel
+from diffusers.pipelines.controlnet.multicontrolnet import MultiControlNetModel
+
+from model_util import get_torch_device
+from insightface.app import FaceAnalysis
+from transformers import CLIPImageProcessor
+from controlnet_util import openpose, get_depth_map, get_canny_image
+
+from diffusers.pipelines.stable_diffusion.safety_checker import (
+    StableDiffusionSafetyChecker,
+)
+from pipeline_stable_diffusion_xl_instantid_full import (
+    StableDiffusionXLInstantIDPipeline,
+    draw_kps,
+)
+
+mimetypes.add_type("image/webp", ".webp")
+
+# GPU global variables
+DEVICE = get_torch_device()
+DTYPE = torch.float16 if str(DEVICE).__contains__("cuda") else torch.float32
+
+# for `ip-adapter`, `ControlNetModel`, and `stable-diffusion-xl-base-1.0`
+CHECKPOINTS_CACHE = "./checkpoints"
+CHECKPOINTS_URL = "https://weights.replicate.delivery/default/InstantID/checkpoints.tar"
+
+# for `models/antelopev2`
+MODELS_CACHE = "./models"
+MODELS_URL = "https://weights.replicate.delivery/default/InstantID/models.tar"
+
+# for the safety checker
+SAFETY_CACHE = "./safety-cache"
+FEATURE_EXTRACTOR = "./feature-extractor"
+SAFETY_URL = "https://weights.replicate.delivery/default/playgroundai/safety-cache.tar"
+
+SDXL_NAME_TO_PATHLIKE = {
+    # These are all huggingface models that we host via gcp + pget
+    "stable-diffusion-xl-base-1.0": {
+        "slug": "stabilityai/stable-diffusion-xl-base-1.0",
+        "url": "https://weights.replicate.delivery/default/InstantID/models--stabilityai--stable-diffusion-xl-base-1.0.tar",
+        "path": "checkpoints/models--stabilityai--stable-diffusion-xl-base-1.0",
+    },
+    "afrodite-xl-v2": {
+        "slug": "stablediffusionapi/afrodite-xl-v2",
+        "url": "https://weights.replicate.delivery/default/InstantID/models--stablediffusionapi--afrodite-xl-v2.tar",
+        "path": "checkpoints/models--stablediffusionapi--afrodite-xl-v2",
+    },
+    "albedobase-xl-20": {
+        "slug": "stablediffusionapi/albedobase-xl-20",
+        "url": "https://weights.replicate.delivery/default/InstantID/models--stablediffusionapi--albedobase-xl-20.tar",
+        "path": "checkpoints/models--stablediffusionapi--albedobase-xl-20",
+    },
+    "albedobase-xl-v13": {
+        "slug": "stablediffusionapi/albedobase-xl-v13",
+        "url": "https://weights.replicate.delivery/default/InstantID/models--stablediffusionapi--albedobase-xl-v13.tar",
+        "path": "checkpoints/models--stablediffusionapi--albedobase-xl-v13",
+    },
+    "animagine-xl-30": {
+        "slug": "stablediffusionapi/animagine-xl-30",
+        "url": "https://weights.replicate.delivery/default/InstantID/models--stablediffusionapi--animagine-xl-30.tar",
+        "path": "checkpoints/models--stablediffusionapi--animagine-xl-30",
+    },
+    "anime-art-diffusion-xl": {
+        "slug": "stablediffusionapi/anime-art-diffusion-xl",
+        "url": "https://weights.replicate.delivery/default/InstantID/models--stablediffusionapi--anime-art-diffusion-xl.tar",
+        "path": "checkpoints/models--stablediffusionapi--anime-art-diffusion-xl",
+    },
+    "anime-illust-diffusion-xl": {
+        "slug": "stablediffusionapi/anime-illust-diffusion-xl",
+        "url": "https://weights.replicate.delivery/default/InstantID/models--stablediffusionapi--anime-illust-diffusion-xl.tar",
+        "path": "checkpoints/models--stablediffusionapi--anime-illust-diffusion-xl",
+    },
+    "dreamshaper-xl": {
+        "slug": "stablediffusionapi/dreamshaper-xl",
+        "url": "https://weights.replicate.delivery/default/InstantID/models--stablediffusionapi--dreamshaper-xl.tar",
+        "path": "checkpoints/models--stablediffusionapi--dreamshaper-xl",
+    },
+    "dynavision-xl-v0610": {
+        "slug": "stablediffusionapi/dynavision-xl-v0610",
+        "url": "https://weights.replicate.delivery/default/InstantID/models--stablediffusionapi--dynavision-xl-v0610.tar",
+        "path": "checkpoints/models--stablediffusionapi--dynavision-xl-v0610",
+    },
+    "guofeng4-xl": {
+        "slug": "stablediffusionapi/guofeng4-xl",
+        "url": "https://weights.replicate.delivery/default/InstantID/models--stablediffusionapi--guofeng4-xl.tar",
+        "path": "checkpoints/models--stablediffusionapi--guofeng4-xl",
+    },
+    "juggernaut-xl-v8": {
+        "slug": "stablediffusionapi/juggernaut-xl-v8",
+        "url": "https://weights.replicate.delivery/default/InstantID/models--stablediffusionapi--juggernaut-xl-v8.tar",
+        "path": "checkpoints/models--stablediffusionapi--juggernaut-xl-v8",
+    },
+    "nightvision-xl-0791": {
+        "slug": "stablediffusionapi/nightvision-xl-0791",
+        "url": "https://weights.replicate.delivery/default/InstantID/models--stablediffusionapi--nightvision-xl-0791.tar",
+        "path": "checkpoints/models--stablediffusionapi--nightvision-xl-0791",
+    },
+    "omnigen-xl": {
+        "slug": "stablediffusionapi/omnigen-xl",
+        "url": "https://weights.replicate.delivery/default/InstantID/models--stablediffusionapi--omnigen-xl.tar",
+        "path": "checkpoints/models--stablediffusionapi--omnigen-xl",
+    },
+    "pony-diffusion-v6-xl": {
+        "slug": "stablediffusionapi/pony-diffusion-v6-xl",
+        "url": "https://weights.replicate.delivery/default/InstantID/models--stablediffusionapi--pony-diffusion-v6-xl.tar",
+        "path": "checkpoints/models--stablediffusionapi--pony-diffusion-v6-xl",
+    },
+    "protovision-xl-high-fidel": {
+        "slug": "stablediffusionapi/protovision-xl-high-fidel",
+        "url": "https://weights.replicate.delivery/default/InstantID/models--stablediffusionapi--protovision-xl-high-fidel.tar",
+        "path": "checkpoints/models--stablediffusionapi--protovision-xl-high-fidel",
+    },
+    "RealVisXL_V3.0_Turbo": {
+        "slug": "SG161222/RealVisXL_V3.0_Turbo",
+        "url": "https://weights.replicate.delivery/default/InstantID/models--SG161222--RealVisXL_V3.0_Turbo.tar",
+        "path": "checkpoints/models--SG161222--RealVisXL_V3.0_Turbo",
+    },
+    "RealVisXL_V4.0_Lightning": {
+        "slug": "SG161222/RealVisXL_V4.0_Lightning",
+        "url": "https://weights.replicate.delivery/default/InstantID/models--SG161222--RealVisXL_V4.0_Lightning.tar",
+        "path": "checkpoints/models--SG161222--RealVisXL_V4.0_Lightning",
+    },
+}
+
+
+def convert_from_cv2_to_image(img: np.ndarray) -> Image:
+    return Image.fromarray(cv2.cvtColor(img, cv2.COLOR_BGR2RGB))
+
+
+def convert_from_image_to_cv2(img: Image) -> np.ndarray:
+    return cv2.cvtColor(np.array(img), cv2.COLOR_RGB2BGR)
+
+
+def resize_img(
+    input_image,
+    max_side=1280,
+    min_side=1024,
+    size=None,
+    pad_to_max_side=False,
+    mode=PIL.Image.BILINEAR,
+    base_pixel_number=64,
+):
+    w, h = input_image.size
+    if size is not None:
+        w_resize_new, h_resize_new = size
+    else:
+        ratio = min_side / min(h, w)
+        w, h = round(ratio * w), round(ratio * h)
+        ratio = max_side / max(h, w)
+        input_image = input_image.resize([round(ratio * w), round(ratio * h)], mode)
+        w_resize_new = (round(ratio * w) // base_pixel_number) * base_pixel_number
+        h_resize_new = (round(ratio * h) // base_pixel_number) * base_pixel_number
+    input_image = input_image.resize([w_resize_new, h_resize_new], mode)
+
+    if pad_to_max_side:
+        res = np.ones([max_side, max_side, 3], dtype=np.uint8) * 255
+        offset_x = (max_side - w_resize_new) // 2
+        offset_y = (max_side - h_resize_new) // 2
+        res[offset_y : offset_y + h_resize_new, offset_x : offset_x + w_resize_new] = (
+            np.array(input_image)
+        )
+        input_image = Image.fromarray(res)
+    return input_image
+
+
+def download_weights(url, dest):
+    start = time.time()
+    print("[!] Initiating download from URL: ", url)
+    print("[~] Destination path: ", dest)
+    command = ["pget", "-vf", url, dest]
+    if ".tar" in url:
+        command.append("-x")
+    try:
+        subprocess.check_call(command, close_fds=False)
+    except subprocess.CalledProcessError as e:
+        print(
+            f"[ERROR] Failed to download weights. Command '{' '.join(e.cmd)}' returned non-zero exit status {e.returncode}."
+        )
+        raise
+    print("[+] Download completed in: ", time.time() - start, "seconds")
+
+
+class Predictor(BasePredictor):
+    def setup(self) -> None:
+        """Load the model into memory to make running multiple predictions efficient"""
+
+        if not os.path.exists(CHECKPOINTS_CACHE):
+            download_weights(CHECKPOINTS_URL, CHECKPOINTS_CACHE)
+
+        if not os.path.exists(MODELS_CACHE):
+            download_weights(MODELS_URL, MODELS_CACHE)
+
+        self.face_detection_input_width, self.face_detection_input_height = 640, 640
+        self.app = FaceAnalysis(
+            name="antelopev2",
+            root="./",
+            providers=["CUDAExecutionProvider", "CPUExecutionProvider"],
+        )
+        self.app.prepare(ctx_id=0, det_size=(self.face_detection_input_width, self.face_detection_input_height))
+
+        # Path to InstantID models
+        self.face_adapter = f"./checkpoints/ip-adapter.bin"
+        controlnet_path = f"./checkpoints/ControlNetModel"
+
+        # Load pipeline face ControlNetModel
+        self.controlnet_identitynet = ControlNetModel.from_pretrained(
+            controlnet_path,
+            torch_dtype=DTYPE,
+            cache_dir=CHECKPOINTS_CACHE,
+            local_files_only=True,
+        )
+        self.setup_extra_controlnets()
+
+        self.load_weights("stable-diffusion-xl-base-1.0")
+        self.setup_safety_checker()
+
+    def setup_safety_checker(self):
+        print(f"[~] Seting up safety checker")
+
+        if not os.path.exists(SAFETY_CACHE):
+            download_weights(SAFETY_URL, SAFETY_CACHE)
+
+        self.safety_checker = StableDiffusionSafetyChecker.from_pretrained(
+            SAFETY_CACHE,
+            torch_dtype=DTYPE,
+            local_files_only=True,
+        )
+        self.safety_checker.to(DEVICE)
+        self.feature_extractor = CLIPImageProcessor.from_pretrained(FEATURE_EXTRACTOR)
+
+    def run_safety_checker(self, image):
+        safety_checker_input = self.feature_extractor(image, return_tensors="pt").to(
+            DEVICE
+        )
+        np_image = np.array(image)
+        image, has_nsfw_concept = self.safety_checker(
+            images=[np_image],
+            clip_input=safety_checker_input.pixel_values.to(DTYPE),
+        )
+        return image, has_nsfw_concept
+
+    def load_weights(self, sdxl_weights):
+        self.base_weights = sdxl_weights
+        weights_info = SDXL_NAME_TO_PATHLIKE[self.base_weights]
+
+        download_url = weights_info["url"]
+        path_to_weights_dir = weights_info["path"]
+        if not os.path.exists(path_to_weights_dir):
+            download_weights(download_url, path_to_weights_dir)
+
+        is_hugging_face_model = "slug" in weights_info.keys()
+        path_to_weights_file = os.path.join(
+            path_to_weights_dir,
+            weights_info.get("file", ""),
+        )
+
+        print(f"[~] Loading new SDXL weights: {path_to_weights_file}")
+        if is_hugging_face_model:
+            self.pipe = StableDiffusionXLInstantIDPipeline.from_pretrained(
+                weights_info["slug"],
+                controlnet=[self.controlnet_identitynet],
+                torch_dtype=DTYPE,
+                cache_dir=CHECKPOINTS_CACHE,
+                local_files_only=True,
+                safety_checker=None,
+                feature_extractor=None,
+            )
+            self.pipe.scheduler = diffusers.EulerDiscreteScheduler.from_config(
+                self.pipe.scheduler.config
+            )
+        else:  # e.g. .safetensors, NOTE: This functionality is not being used right now
+            self.pipe.from_single_file(
+                path_to_weights_file,
+                controlnet=self.controlnet_identitynet,
+                torch_dtype=DTYPE,
+                cache_dir=CHECKPOINTS_CACHE,
+            )
+
+        self.pipe.load_ip_adapter_instantid(self.face_adapter)
+        self.setup_lcm_lora()
+        self.pipe.cuda()
+
+    def setup_lcm_lora(self):
+        print(f"[~] Seting up LCM (just in case)")
+
+        lcm_lora_key = "models--latent-consistency--lcm-lora-sdxl"
+        lcm_lora_path = f"checkpoints/{lcm_lora_key}"
+        if not os.path.exists(lcm_lora_path):
+            download_weights(
+                f"https://weights.replicate.delivery/default/InstantID/{lcm_lora_key}.tar",
+                lcm_lora_path,
+            )
+        self.pipe.load_lora_weights(
+            "latent-consistency/lcm-lora-sdxl",
+            cache_dir=CHECKPOINTS_CACHE,
+            local_files_only=True,
+            weight_name="pytorch_lora_weights.safetensors",
+        )
+        self.pipe.disable_lora()
+
+    def setup_extra_controlnets(self):
+        print(f"[~] Seting up pose, canny, depth ControlNets")
+
+        controlnet_pose_model = "thibaud/controlnet-openpose-sdxl-1.0"
+        controlnet_canny_model = "diffusers/controlnet-canny-sdxl-1.0"
+        controlnet_depth_model = "diffusers/controlnet-depth-sdxl-1.0-small"
+
+        for controlnet_key in [
+            "models--diffusers--controlnet-canny-sdxl-1.0",
+            "models--diffusers--controlnet-depth-sdxl-1.0-small",
+            "models--thibaud--controlnet-openpose-sdxl-1.0",
+        ]:
+            controlnet_path = f"checkpoints/{controlnet_key}"
+            if not os.path.exists(controlnet_path):
+                download_weights(
+                    f"https://weights.replicate.delivery/default/InstantID/{controlnet_key}.tar",
+                    controlnet_path,
+                )
+
+        controlnet_pose = ControlNetModel.from_pretrained(
+            controlnet_pose_model,
+            torch_dtype=DTYPE,
+            cache_dir=CHECKPOINTS_CACHE,
+            local_files_only=True,
+        ).to(DEVICE)
+        controlnet_canny = ControlNetModel.from_pretrained(
+            controlnet_canny_model,
+            torch_dtype=DTYPE,
+            cache_dir=CHECKPOINTS_CACHE,
+            local_files_only=True,
+        ).to(DEVICE)
+        controlnet_depth = ControlNetModel.from_pretrained(
+            controlnet_depth_model,
+            torch_dtype=DTYPE,
+            cache_dir=CHECKPOINTS_CACHE,
+            local_files_only=True,
+        ).to(DEVICE)
+
+        self.controlnet_map = {
+            "pose": controlnet_pose,
+            "canny": controlnet_canny,
+            "depth": controlnet_depth,
+        }
+        self.controlnet_map_fn = {
+            "pose": openpose,
+            "canny": get_canny_image,
+            "depth": get_depth_map,
+        }
+
+    def generate_image(
+        self,
+        face_image_path,
+        pose_image_path,
+        prompt,
+        negative_prompt,
+        num_steps,
+        identitynet_strength_ratio,
+        adapter_strength_ratio,
+        pose_strength,
+        canny_strength,
+        depth_strength,
+        controlnet_selection,
+        guidance_scale,
+        seed,
+        scheduler,
+        enable_LCM,
+        enhance_face_region,
+        num_images_per_prompt,
+    ):
+        if enable_LCM:
+            self.pipe.enable_lora()
+            self.pipe.scheduler = LCMScheduler.from_config(self.pipe.scheduler.config)
+        else:
+            self.pipe.disable_lora()
+            scheduler_class_name = scheduler.split("-")[0]
+
+            add_kwargs = {}
+            if len(scheduler.split("-")) > 1:
+                add_kwargs["use_karras_sigmas"] = True
+            if len(scheduler.split("-")) > 2:
+                add_kwargs["algorithm_type"] = "sde-dpmsolver++"
+            scheduler = getattr(diffusers, scheduler_class_name)
+            self.pipe.scheduler = scheduler.from_config(
+                self.pipe.scheduler.config,
+                **add_kwargs,
+            )
+
+        if face_image_path is None:
+            raise Exception(
+                f"Cannot find any input face `image`! Please upload the face `image`"
+            )
+
+        face_image = load_image(face_image_path)
+        face_image = resize_img(face_image)
+        face_image_cv2 = convert_from_image_to_cv2(face_image)
+        height, width, _ = face_image_cv2.shape
+
+        # Extract face features
+        face_info = self.app.get(face_image_cv2)
+
+        if len(face_info) == 0:
+            raise Exception(
+                "Face detector could not find a face in the `image`. Please use a different `image` as input."
+            )
+
+        face_info = sorted(
+            face_info,
+            key=lambda x: (x["bbox"][2] - x["bbox"][0]) * x["bbox"][3] - x["bbox"][1],
+        )[
+            -1
+        ]  # only use the maximum face
+        face_emb = face_info["embedding"]
+        face_kps = draw_kps(convert_from_cv2_to_image(face_image_cv2), face_info["kps"])
+
+        img_controlnet = face_image
+        if pose_image_path is not None:
+            pose_image = load_image(pose_image_path)
+            pose_image = resize_img(pose_image, max_side=1024)
+            img_controlnet = pose_image
+            pose_image_cv2 = convert_from_image_to_cv2(pose_image)
+
+            face_info = self.app.get(pose_image_cv2)
+
+            if len(face_info) == 0:
+                raise Exception(
+                    "Face detector could not find a face in the `pose_image`. Please use a different `pose_image` as input."
+                )
+
+            face_info = face_info[-1]
+            face_kps = draw_kps(pose_image, face_info["kps"])
+
+            width, height = face_kps.size
+
+        if enhance_face_region:
+            control_mask = np.zeros([height, width, 3])
+            x1, y1, x2, y2 = face_info["bbox"]
+            x1, y1, x2, y2 = int(x1), int(y1), int(x2), int(y2)
+            control_mask[y1:y2, x1:x2] = 255
+            control_mask = Image.fromarray(control_mask.astype(np.uint8))
+        else:
+            control_mask = None
+
+        if len(controlnet_selection) > 0:
+            controlnet_scales = {
+                "pose": pose_strength,
+                "canny": canny_strength,
+                "depth": depth_strength,
+            }
+            self.pipe.controlnet = MultiControlNetModel(
+                [self.controlnet_identitynet]
+                + [self.controlnet_map[s] for s in controlnet_selection]
+            )
+            control_scales = [float(identitynet_strength_ratio)] + [
+                controlnet_scales[s] for s in controlnet_selection
+            ]
+            control_images = [face_kps] + [
+                self.controlnet_map_fn[s](img_controlnet).resize((width, height))
+                for s in controlnet_selection
+            ]
+        else:
+            self.pipe.controlnet = self.controlnet_identitynet
+            control_scales = float(identitynet_strength_ratio)
+            control_images = face_kps
+
+        generator = torch.Generator(device=DEVICE).manual_seed(seed)
+
+        print("Start inference...")
+        print(f"[Debug] Prompt: {prompt}, \n[Debug] Neg Prompt: {negative_prompt}")
+
+        self.pipe.set_ip_adapter_scale(adapter_strength_ratio)
+        images = self.pipe(
+            prompt=prompt,
+            negative_prompt=negative_prompt,
+            image_embeds=face_emb,
+            image=control_images,
+            control_mask=control_mask,
+            controlnet_conditioning_scale=control_scales,
+            num_inference_steps=num_steps,
+            guidance_scale=guidance_scale,
+            height=height,
+            width=width,
+            generator=generator,
+            num_images_per_prompt=num_images_per_prompt,
+        ).images
+
+        return images
+
+    def predict(
+        self,
+        image: Path = Input(
+            description="Input face image",
+        ),
+        pose_image: Path = Input(
+            description="(Optional) reference pose image",
+            default=None,
+        ),
+        prompt: str = Input(
+            description="Input prompt",
+            default="a person",
+        ),
+        negative_prompt: str = Input(
+            description="Input Negative Prompt",
+            default="",
+        ),
+        sdxl_weights: str = Input(
+            description="Pick which base weights you want to use",
+            default="stable-diffusion-xl-base-1.0",
+            choices=[
+                "stable-diffusion-xl-base-1.0",
+                "juggernaut-xl-v8",
+                "afrodite-xl-v2",
+                "albedobase-xl-20",
+                "albedobase-xl-v13",
+                "animagine-xl-30",
+                "anime-art-diffusion-xl",
+                "anime-illust-diffusion-xl",
+                "dreamshaper-xl",
+                "dynavision-xl-v0610",
+                "guofeng4-xl",
+                "nightvision-xl-0791",
+                "omnigen-xl",
+                "pony-diffusion-v6-xl",
+                "protovision-xl-high-fidel",
+                "RealVisXL_V3.0_Turbo",
+                "RealVisXL_V4.0_Lightning",
+            ],
+        ),
+        face_detection_input_width: int = Input(
+            description="Width of the input image for face detection",
+            default=640,
+            ge=640,
+            le=4096,
+        ),
+        face_detection_input_height: int = Input(
+            description="Height of the input image for face detection",
+            default=640,
+            ge=640,
+            le=4096,
+        ),
+        scheduler: str = Input(
+            description="Scheduler",
+            choices=[
+                "DEISMultistepScheduler",
+                "HeunDiscreteScheduler",
+                "EulerDiscreteScheduler",
+                "DPMSolverMultistepScheduler",
+                "DPMSolverMultistepScheduler-Karras",
+                "DPMSolverMultistepScheduler-Karras-SDE",
+            ],
+            default="EulerDiscreteScheduler",
+        ),
+        num_inference_steps: int = Input(
+            description="Number of denoising steps",
+            default=30,
+            ge=1,
+            le=500,
+        ),
+        guidance_scale: float = Input(
+            description="Scale for classifier-free guidance",
+            default=7.5,
+            ge=1,
+            le=50,
+        ),
+        ip_adapter_scale: float = Input(
+            description="Scale for image adapter strength (for detail)",  # adapter_strength_ratio
+            default=0.8,
+            ge=0,
+            le=1.5,
+        ),
+        controlnet_conditioning_scale: float = Input(
+            description="Scale for IdentityNet strength (for fidelity)",  # identitynet_strength_ratio
+            default=0.8,
+            ge=0,
+            le=1.5,
+        ),
+        enable_pose_controlnet: bool = Input(
+            description="Enable Openpose ControlNet, overrides strength if set to false",
+            default=True,
+        ),
+        pose_strength: float = Input(
+            description="Openpose ControlNet strength, effective only if `enable_pose_controlnet` is true",
+            default=0.4,
+            ge=0,
+            le=1,
+        ),
+        enable_canny_controlnet: bool = Input(
+            description="Enable Canny ControlNet, overrides strength if set to false",
+            default=False,
+        ),
+        canny_strength: float = Input(
+            description="Canny ControlNet strength, effective only if `enable_canny_controlnet` is true",
+            default=0.3,
+            ge=0,
+            le=1,
+        ),
+        enable_depth_controlnet: bool = Input(
+            description="Enable Depth ControlNet, overrides strength if set to false",
+            default=False,
+        ),
+        depth_strength: float = Input(
+            description="Depth ControlNet strength, effective only if `enable_depth_controlnet` is true",
+            default=0.5,
+            ge=0,
+            le=1,
+        ),
+        enable_lcm: bool = Input(
+            description="Enable Fast Inference with LCM (Latent Consistency Models) - speeds up inference steps, trade-off is the quality of the generated image. Performs better with close-up portrait face images",
+            default=False,
+        ),
+        lcm_num_inference_steps: int = Input(
+            description="Only used when `enable_lcm` is set to True, Number of denoising steps when using LCM",
+            default=5,
+            ge=1,
+            le=10,
+        ),
+        lcm_guidance_scale: float = Input(
+            description="Only used when `enable_lcm` is set to True, Scale for classifier-free guidance when using LCM",
+            default=1.5,
+            ge=1,
+            le=20,
+        ),
+        enhance_nonface_region: bool = Input(
+            description="Enhance non-face region", default=True
+        ),
+        output_format: str = Input(
+            description="Format of the output images",
+            choices=["webp", "jpg", "png"],
+            default="webp",
+        ),
+        output_quality: int = Input(
+            description="Quality of the output images, from 0 to 100. 100 is best quality, 0 is lowest quality.",
+            default=80,
+            ge=0,
+            le=100,
+        ),
+        seed: int = Input(
+            description="Random seed. Leave blank to randomize the seed",
+            default=None,
+        ),
+        num_outputs: int = Input(
+            description="Number of images to output",
+            default=1,
+            ge=1,
+            le=8,
+        ),
+        disable_safety_checker: bool = Input(
+            description="Disable safety checker for generated images",
+            default=False,
+        ),
+    ) -> List[Path]:
+        """Run a single prediction on the model"""
+
+        # If no seed is provided, generate a random seed
+        if seed is None:
+            seed = int.from_bytes(os.urandom(2), "big")
+        print(f"Using seed: {seed}")
+
+        # Load the weights if they are different from the base weights
+        if sdxl_weights != self.base_weights:
+            self.load_weights(sdxl_weights)
+
+        # Resize the output if the provided dimensions are different from the current ones
+        if self.face_detection_input_width != face_detection_input_width or self.face_detection_input_height != face_detection_input_height:
+            print(f"[!] Resizing output to {face_detection_input_width}x{face_detection_input_height}")
+            self.face_detection_input_width = face_detection_input_width
+            self.face_detection_input_height = face_detection_input_height
+            self.app.prepare(ctx_id=0, det_size=(self.face_detection_input_width, self.face_detection_input_height))
+
+        # Set up ControlNet selection and their respective strength values (if any)
+        controlnet_selection = []
+        if pose_strength > 0 and enable_pose_controlnet:
+            controlnet_selection.append("pose")
+        if canny_strength > 0 and enable_canny_controlnet:
+            controlnet_selection.append("canny")
+        if depth_strength > 0 and enable_depth_controlnet:
+            controlnet_selection.append("depth")
+
+        # Switch to LCM inference steps and guidance scale if LCM is enabled
+        if enable_lcm:
+            num_inference_steps = lcm_num_inference_steps
+            guidance_scale = lcm_guidance_scale
+
+        # Generate
+        images = self.generate_image(
+            face_image_path=str(image),
+            pose_image_path=str(pose_image) if pose_image else None,
+            prompt=prompt,
+            negative_prompt=negative_prompt,
+            num_steps=num_inference_steps,
+            identitynet_strength_ratio=controlnet_conditioning_scale,
+            adapter_strength_ratio=ip_adapter_scale,
+            pose_strength=pose_strength,
+            canny_strength=canny_strength,
+            depth_strength=depth_strength,
+            controlnet_selection=controlnet_selection,
+            scheduler=scheduler,
+            guidance_scale=guidance_scale,
+            seed=seed,
+            enable_LCM=enable_lcm,
+            enhance_face_region=enhance_nonface_region,
+            num_images_per_prompt=num_outputs,
+        )
+
+        # Save the generated images and check for NSFW content
+        output_paths = []
+        for i, output_image in enumerate(images):
+            if not disable_safety_checker:
+                _, has_nsfw_content_list = self.run_safety_checker(output_image)
+                has_nsfw_content = any(has_nsfw_content_list)
+                print(f"NSFW content detected: {has_nsfw_content}")
+                if has_nsfw_content:
+                    raise Exception(
+                        "NSFW content detected. Try running it again, or try a different prompt."
+                    )
+
+            extension = output_format.lower()
+            extension = "jpeg" if extension == "jpg" else extension
+            output_path = f"/tmp/out_{i}.{extension}"
+
+            print(f"[~] Saving to {output_path}...")
+            print(f"[~] Output format: {extension.upper()}")
+            if output_format != "png":
+                print(f"[~] Output quality: {output_quality}")
+
+            save_params = {"format": extension.upper()}
+            if output_format != "png":
+                save_params["quality"] = output_quality
+                save_params["optimize"] = True
+
+            output_image.save(output_path, **save_params)
+            output_paths.append(Path(output_path))
+        return output_paths

feature-extractor/preprocessor_config.json

@@ -0,0 +1,27 @@
+{
+  "crop_size": {
+    "height": 224,
+    "width": 224
+  },
+  "do_center_crop": true,
+  "do_convert_rgb": true,
+  "do_normalize": true,
+  "do_rescale": true,
+  "do_resize": true,
+  "image_mean": [
+    0.48145466,
+    0.4578275,
+    0.40821073
+  ],
+  "image_processor_type": "CLIPImageProcessor",
+  "image_std": [
+    0.26862954,
+    0.26130258,
+    0.27577711
+  ],
+  "resample": 3,
+  "rescale_factor": 0.00392156862745098,
+  "size": {
+    "shortest_edge": 224
+  }
+}

generation_log.csv

@@ -0,0 +1,7 @@
+image_name,new_file_name,identitynet_strength_ratio,adapter_strength_ratio,num_inference_steps,guidance_scale,seed,success,error_message,style_name,prompt,negative_prompt,time_taken,current_timestamp
+musk_resize.jpeg,20240723_053704_668578_0.png,1.1491785966677859,0.8654292835406997,50,10.881974934041711,4170043132,True,,(No style),"human, sharp focus","(blurry, blur, text, abstract, glitch, lowres, low quality, worst quality:1.2), (text:1.2), watermark, painting, drawing, illustration, glitch, deformed, mutated, cross-eyed, ugly, disfigured",53.160874,2024-07-23 05:37:05
+sam_resize.png,20240723_053801_148984_0.png,1.0277924316289087,0.9683019180411349,53,11.111615489229361,1039000092,True,,(No style),"human, sharp focus","(blurry, blur, text, abstract, glitch, lowres, low quality, worst quality:1.2), (text:1.2), watermark, painting, drawing, illustration, glitch, deformed, mutated, cross-eyed, ugly, disfigured",56.287061,2024-07-23 05:38:01
+schmidhuber_resize.png,20240723_053853_022841_0.png,1.4917970061395218,0.7393876001187043,48,11.679426057392323,3752244045,True,,(No style),"human, sharp focus","(blurry, blur, text, abstract, glitch, lowres, low quality, worst quality:1.2), (text:1.2), watermark, painting, drawing, illustration, glitch, deformed, mutated, cross-eyed, ugly, disfigured",51.661633,2024-07-23 05:38:53
+kaifu_resize.png,20240723_053948_468290_0.png,1.4485948536834086,0.8122224472625851,52,9.984434112216853,2295950491,True,,(No style),"human, sharp focus","(blurry, blur, text, abstract, glitch, lowres, low quality, worst quality:1.2), (text:1.2), watermark, painting, drawing, illustration, glitch, deformed, mutated, cross-eyed, ugly, disfigured",55.366897,2024-07-23 05:39:49
+pp_0.jpg,20240723_054025_692605_0.png,1.1794069160183727,0.9857350785784462,51,8.76420747179281,2648835109,True,,(No style),"human, sharp focus","(blurry, blur, text, abstract, glitch, lowres, low quality, worst quality:1.2), (text:1.2), watermark, painting, drawing, illustration, glitch, deformed, mutated, cross-eyed, ugly, disfigured",36.771416,2024-07-23 05:40:26
+yann-lecun_resize.jpg,20240723_054124_697176_0.png,1.2770220875965888,0.8245108249424827,56,9.372671733967127,3933691473,True,,(No style),"human, sharp focus","(blurry, blur, text, abstract, glitch, lowres, low quality, worst quality:1.2), (text:1.2), watermark, painting, drawing, illustration, glitch, deformed, mutated, cross-eyed, ugly, disfigured",59.069521,2024-07-23 05:41:25

gradio_demo/aaa.py

@@ -0,0 +1,957 @@
+import sys
+sys.path.append('./')
+
+from typing import Tuple
+
+import os
+import cv2
+import math
+import torch
+import random
+import numpy as np
+import argparse
+
+import PIL
+from PIL import Image
+
+import diffusers
+from diffusers.utils import load_image
+from diffusers.models import ControlNetModel
+from diffusers import LCMScheduler
+
+from huggingface_hub import hf_hub_download
+
+import insightface
+from insightface.app import FaceAnalysis
+
+from style_template import styles
+from pipeline_stable_diffusion_xl_instantid_full import StableDiffusionXLInstantIDPipeline
+from model_util import load_models_xl, get_torch_device, torch_gc
+
+
+# global variable
+MAX_SEED = np.iinfo(np.int32).max
+device = get_torch_device()
+dtype = torch.float16 if str(device).__contains__("cuda") else torch.float32
+STYLE_NAMES = list(styles.keys())
+DEFAULT_STYLE_NAME = "Watercolor"
+
+# Load face encoder
+app = FaceAnalysis(name='antelopev2', root='./', providers=['CUDAExecutionProvider', 'CPUExecutionProvider'])
+app.prepare(ctx_id=0, det_size=(320, 320))
+
+# Path to InstantID models
+face_adapter = f'./checkpoints/ip-adapter.bin'
+controlnet_path = f'./checkpoints/ControlNetModel'
+
+# Load pipeline
+controlnet = ControlNetModel.from_pretrained(controlnet_path, torch_dtype=dtype)
+
+logo = Image.open("./gradio_demo/logo.png")
+
+from cv2 import imencode
+import base64
+
+# def encode_pil_to_base64_new(pil_image):
+#     print("AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA")
+#     image_arr = np.asarray(pil_image)[:,:,::-1]
+#     _, byte_data = imencode('.png', image_arr)        
+#     base64_data = base64.b64encode(byte_data)
+#     base64_string_opencv = base64_data.decode("utf-8")
+#     return "data:image/png;base64," + base64_string_opencv
+
+import gradio as gr
+
+# gr.processing_utils.encode_pil_to_base64 = encode_pil_to_base64_new
+
+def main(pretrained_model_name_or_path="wangqixun/YamerMIX_v8", enable_lcm_arg=False):
+
+    if pretrained_model_name_or_path.endswith(
+            ".ckpt"
+        ) or pretrained_model_name_or_path.endswith(".safetensors"):
+            scheduler_kwargs = hf_hub_download(
+                repo_id="wangqixun/YamerMIX_v8",
+                subfolder="scheduler",
+                filename="scheduler_config.json",
+            )
+
+            (tokenizers, text_encoders, unet, _, vae) = load_models_xl(
+                pretrained_model_name_or_path=pretrained_model_name_or_path,
+                scheduler_name=None,
+                weight_dtype=dtype,
+            )
+
+            scheduler = diffusers.EulerDiscreteScheduler.from_config(scheduler_kwargs)
+            pipe = StableDiffusionXLInstantIDPipeline(
+                vae=vae,
+                text_encoder=text_encoders[0],
+                text_encoder_2=text_encoders[1],
+                tokenizer=tokenizers[0],
+                tokenizer_2=tokenizers[1],
+                unet=unet,
+                scheduler=scheduler,
+                controlnet=controlnet,
+            ).to(device)
+
+    else:
+        pipe = StableDiffusionXLInstantIDPipeline.from_pretrained(
+            pretrained_model_name_or_path,
+            controlnet=controlnet,
+            torch_dtype=dtype,
+            safety_checker=None,
+            feature_extractor=None,
+        ).to(device)
+
+        pipe.scheduler = diffusers.EulerDiscreteScheduler.from_config(pipe.scheduler.config)
+
+    pipe.load_ip_adapter_instantid(face_adapter)
+    # load and disable LCM
+    pipe.load_lora_weights("latent-consistency/lcm-lora-sdxl")
+    pipe.disable_lora()
+    
+    def remove_tips():
+        print("GG")
+        return gr.update(visible=False)
+
+    
+    # prompts = [
+    #     ["superman","Vibrant Color"], ["japanese anime character with white/neon hair","Watercolor"], 
+    #     # ["Suited professional","(No style)"], 
+    #     ["Scooba diver","Line art"], ["eskimo","Snow"]
+    # ]
+
+    def convert_from_cv2_to_image(img: np.ndarray) -> Image:
+        return Image.fromarray(cv2.cvtColor(img, cv2.COLOR_BGR2RGB))
+
+    def convert_from_image_to_cv2(img: Image) -> np.ndarray:
+        return cv2.cvtColor(np.array(img), cv2.COLOR_RGB2BGR)
+
+    def run_for_prompts1(face_file,style,progress=gr.Progress(track_tqdm=True)):
+        # if email != "":
+        p,n = styles.get(style, styles.get(STYLE_NAMES[1]))
+        return generate_image(face_file, p[0], n)
+        # else:
+            # raise gr.Error("Email ID is compulsory")
+    def run_for_prompts2(face_file,style,progress=gr.Progress(track_tqdm=True)):
+        # if email != "":
+        p,n = styles.get(style, styles.get(STYLE_NAMES[1]))
+        return generate_image(face_file, p[1], n)
+    def run_for_prompts3(face_file,style,progress=gr.Progress(track_tqdm=True)):
+        # if email != "":
+        p,n = styles.get(style, styles.get(STYLE_NAMES[1]))
+        return generate_image(face_file, p[2], n)
+    def run_for_prompts4(face_file,style,progress=gr.Progress(track_tqdm=True)):
+        # if email != "":
+        p,n = styles.get(style, styles.get(STYLE_NAMES[1]))
+        return generate_image(face_file, p[3], n)
+    
+#     def validate_and_process(face_file, style, email):
+    
+#     # Your processing logic here
+#     gallery1, gallery2, gallery3, gallery4 = run_for_prompts1(face_file, style), run_for_prompts2(face_file, style), run_for_prompts3(face_file, style), run_for_prompts4(face_file, style)
+#     return gallery1, gallery2, gallery3, gallery4
+
+    def draw_kps(image_pil, kps, color_list=[(255,0,0), (0,255,0), (0,0,255), (255,255,0), (255,0,255)]):
+        stickwidth = 4
+        limbSeq = np.array([[0, 2], [1, 2], [3, 2], [4, 2]])
+        kps = np.array(kps)
+
+        w, h = image_pil.size
+        out_img = np.zeros([h, w, 3])
+
+        for i in range(len(limbSeq)):
+            index = limbSeq[i]
+            color = color_list[index[0]]
+
+            x = kps[index][:, 0]
+            y = kps[index][:, 1]
+            length = ((x[0] - x[1]) ** 2 + (y[0] - y[1]) ** 2) ** 0.5
+            angle = math.degrees(math.atan2(y[0] - y[1], x[0] - x[1]))
+            polygon = cv2.ellipse2Poly((int(np.mean(x)), int(np.mean(y))), (int(length / 2), stickwidth), int(angle), 0, 360, 1)
+            out_img = cv2.fillConvexPoly(out_img.copy(), polygon, color)
+        out_img = (out_img * 0.6).astype(np.uint8)
+
+        for idx_kp, kp in enumerate(kps):
+            color = color_list[idx_kp]
+            x, y = kp
+            out_img = cv2.circle(out_img.copy(), (int(x), int(y)), 10, color, -1)
+
+        out_img_pil = Image.fromarray(out_img.astype(np.uint8))
+        return out_img_pil
+
+    def resize_img(input_image, max_side=640, min_side=640, size=None, 
+                pad_to_max_side=True, mode=PIL.Image.BILINEAR, base_pixel_number=64):
+
+            w, h = input_image.size
+            print(w)
+            print(h)
+            if size is not None:
+                w_resize_new, h_resize_new = size
+            else:
+                ratio = min_side / min(h, w)
+                w, h = round(ratio*w), round(ratio*h)
+                ratio = max_side / max(h, w)
+                input_image = input_image.resize([round(ratio*w), round(ratio*h)], mode)
+                w_resize_new = (round(ratio * w) // base_pixel_number) * base_pixel_number
+                h_resize_new = (round(ratio * h) // base_pixel_number) * base_pixel_number
+            input_image = input_image.resize([w_resize_new, h_resize_new], mode)
+
+            if pad_to_max_side:
+                res = np.ones([max_side, max_side, 3], dtype=np.uint8) * 255
+                offset_x = (max_side - w_resize_new) // 2
+                offset_y = (max_side - h_resize_new) // 2
+                res[offset_y:offset_y+h_resize_new, offset_x:offset_x+w_resize_new] = np.array(input_image)
+                input_image = Image.fromarray(res)
+            return input_image
+
+    # def apply_style(style_name: str, positive: str, negative: str = "") -> Tuple[str, str]:
+    #     p, n = styles.get(style_name, styles[DEFAULT_STYLE_NAME])
+    #     return p.replace("{prompt}", positive), n + ' ' + negative
+
+    def generate_image(face_image,prompt,negative_prompt):
+        pose_image_path = None
+        # prompt = "superman"
+        enable_LCM = False
+        identitynet_strength_ratio = 0.95
+        adapter_strength_ratio = 0.60
+        num_steps = 15
+        guidance_scale = 8.5
+        seed = random.randint(0, MAX_SEED)
+        # negative_prompt = ""
+        # negative_prompt += neg
+        enhance_face_region = True
+        if enable_LCM:
+            pipe.enable_lora()
+            pipe.scheduler = LCMScheduler.from_config(pipe.scheduler.config)
+        else:
+            pipe.disable_lora()
+            pipe.scheduler = diffusers.EulerDiscreteScheduler.from_config(pipe.scheduler.config)
+    
+        if face_image is None:
+            raise gr.Error(f"Cannot find any input face image! Please upload the face image")
+        
+        # if prompt is None:
+        #     prompt = "a person"
+        
+        # apply the style template
+        # prompt, negative_prompt = apply_style(style_name, prompt, negative_prompt)
+        
+        # face_image = load_image(face_image_path)
+        face_image = resize_img(face_image)
+        face_image_cv2 = convert_from_image_to_cv2(face_image)
+        height, width, _ = face_image_cv2.shape
+        
+        # Extract face features
+        face_info = app.get(face_image_cv2)
+        
+        if len(face_info) == 0:
+            raise gr.Error(f"Cannot find any face in the image! Please upload another person image")
+        
+        face_info = sorted(face_info, key=lambda x:(x['bbox'][2]-x['bbox'][0])*(x['bbox'][3]-x['bbox'][1]))[-1]  # only use the maximum face
+        face_emb = face_info['embedding']
+        face_kps = draw_kps(convert_from_cv2_to_image(face_image_cv2), face_info['kps'])
+        
+        if pose_image_path is not None:
+            pose_image = load_image(pose_image_path)
+            pose_image = resize_img(pose_image)
+            pose_image_cv2 = convert_from_image_to_cv2(pose_image)
+            
+            face_info = app.get(pose_image_cv2)
+            
+            if len(face_info) == 0:
+                raise gr.Error(f"Cannot find any face in the reference image! Please upload another person image")
+            
+            face_info = face_info[-1]
+            face_kps = draw_kps(pose_image, face_info['kps'])
+            
+            width, height = face_kps.size
+
+        if enhance_face_region:
+            control_mask = np.zeros([height, width, 3])
+            x1, y1, x2, y2 = face_info["bbox"]
+            x1, y1, x2, y2 = int(x1), int(y1), int(x2), int(y2)
+            control_mask[y1:y2, x1:x2] = 255
+            control_mask = Image.fromarray(control_mask.astype(np.uint8))
+        else:
+            control_mask = None
+                        
+        generator = torch.Generator(device=device).manual_seed(seed)
+        
+        print("Start inference...")
+        print(f"[Debug] Prompt: {prompt}, \n[Debug] Neg Prompt: {negative_prompt}")
+        
+        pipe.set_ip_adapter_scale(adapter_strength_ratio)
+        images = pipe(
+            prompt=prompt,
+            negative_prompt=negative_prompt,
+            image_embeds=face_emb,
+            image=face_kps,
+            control_mask=control_mask,
+            controlnet_conditioning_scale=float(identitynet_strength_ratio),
+            num_inference_steps=num_steps,
+            guidance_scale=guidance_scale,
+            height=height,
+            width=width,
+            generator=generator,
+            # num_images_per_prompt = 4
+        ).images
+
+        return images[0]
+
+    ### Description
+    title = r"""
+    <h1 align="center">Choose your AVATAR</h1>
+    """
+
+    description = r"""
+    <h2> Powered by IDfy </h2>"""
+
+    article = r""""""
+
+    tips = r""""""
+    
+    # js = '''  '''
+    
+    css = '''
+    .gradio-container {width: 95% !important; background-color: #E6F3FF;} 
+    .image-gallery {height: 100vh !important; overflow: auto;}
+    .gradio-row .gradio-element { margin: 0 !important; }
+    '''
+#     with gr.Blocks(css=css, js=js) as demo:
+
+#         # description
+#         gr.Markdown(title)
+#         with gr.Row():
+#             gr.Image("./gradio_demo/logo.png",scale=0,min_width=50,show_label=False,show_download_button=False)
+#             gr.Markdown(description)
+#         with gr.Row():
+#             with gr.Column():
+#                 style = gr.Dropdown(label="Choose your STYLE", choices=STYLE_NAMES)
+#                 face_file = gr.Image(label="Upload a photo of your face", type="pil",sources="webcam")
+#                 submit = gr.Button("Submit", variant="primary")
+#             with gr.Column():
+#                 with gr.Row():
+#                     gallery1 = gr.Image(label="Generated Images")
+#                     gallery2 = gr.Image(label="Generated Images")
+#                 with gr.Row():
+#                     gallery3 = gr.Image(label="Generated Images")
+#                     gallery4 = gr.Image(label="Generated Images")
+#                 email = gr.Textbox(label="Email",
+#                         info="Enter your email address",
+#                         value="")
+#                 # submit1 = gr.Button("Store")
+            
+#                 usage_tips = gr.Markdown(label="Usage tips of InstantID", value=tips ,visible=False)
+                
+#             face_file.upload(
+#                 fn=remove_tips,
+#                 outputs=usage_tips,
+#                 queue=True,
+#                 api_name=False,
+#                 show_progress = "full"
+#             ).then(
+#                 fn=run_for_prompts1,
+#                 inputs=[face_file,style],
+#                 outputs=[gallery1]
+#             ).then(
+#                 fn=run_for_prompts2,
+#                 inputs=[face_file,style],
+#                 outputs=[gallery2]
+#             ).then(
+#                 fn=run_for_prompts3,
+#                 inputs=[face_file,style],
+#                 outputs=[gallery3]
+#             ).then(
+#                 fn=run_for_prompts4,
+#                 inputs=[face_file,style],
+#                 outputs=[gallery4]
+#             )
+#             submit.click(
+#                 fn=remove_tips,
+#                 outputs=usage_tips,
+#                 queue=True,
+#                 api_name=False,
+#                 show_progress = "full"
+#             ).then(
+#                 fn=run_for_prompts1,
+#                 inputs=[face_file,style],
+#                 outputs=[gallery1]
+#             ).then(
+#                 fn=run_for_prompts2,
+#                 inputs=[face_file,style],
+#                 outputs=[gallery2]
+#             ).then(
+#                 fn=run_for_prompts3,
+#                 inputs=[face_file,style],
+#                 outputs=[gallery3]
+#             ).then(
+#                 fn=run_for_prompts4,
+#                 inputs=[face_file,style],
+#                 outputs=[gallery4]
+#             )
+            
+#             # submit1.click(
+#             #     fn=store_images,
+#             #     inputs=[email,gallery1,gallery2,gallery3,gallery4],
+#             #     outputs=None)
+            
+        
+        
+#         gr.Markdown(article)
+
+#     demo.launch(share=True)
+
+    with gr.Blocks(css=css) as demo:
+
+        # description
+        gr.Markdown(title)
+        with gr.Row():
+            gr.Image("./gradio_demo/logo.png",scale=0,min_width=50,show_label=False,show_download_button=False)
+            gr.Markdown(description)
+        with gr.Row():
+            with gr.Column():
+                style = gr.Dropdown(label="Choose your STYLE", choices=STYLE_NAMES)
+                face_file = gr.Image(label="Upload a photo of your face", type="pil",sources="webcam")
+                submit = gr.Button("Submit", variant="primary")
+            with gr.Column():
+                with gr.Row():
+                    gallery1 = gr.Image(label="Generated Images")
+                    gallery2 = gr.Image(label="Generated Images")
+                with gr.Row():
+                    gallery3 = gr.Image(label="Generated Images")
+                    gallery4 = gr.Image(label="Generated Images")
+                email = gr.Textbox(label="Email",
+                        info="Enter your email address",
+                        value="")
+            
+            usage_tips = gr.Markdown(label="Usage tips of InstantID", value=tips ,visible=False)
+            # identitynet_strength_ratio = gr.Slider(
+            #     label="IdentityNet strength (for fidelity)",
+            #     minimum=0,
+            #     maximum=1.5,
+            #     step=0.05,
+            #     value=0.95,
+            # )
+            # adapter_strength_ratio = gr.Slider(
+            #     label="Image adapter strength (for detail)",
+            #     minimum=0,
+            #     maximum=1.5,
+            #     step=0.05,
+            #     value=0.60,
+            # )
+            # negative_prompt = gr.Textbox(
+            #     label="Negative Prompt", 
+            #     placeholder="low quality",
+            #     value="(lowres, low quality, worst quality:1.2), (text:1.2), watermark, (frame:1.2), deformed, ugly, deformed eyes, blur, out of focus, blurry, deformed cat, deformed, photo, anthropomorphic cat, monochrome, pet collar, gun, weapon, blue, 3d, drones, drone, buildings in background, green",
+            # )
+            # num_steps = gr.Slider( 
+            #     label="Number of sample steps",
+            #     minimum=15,
+            #     maximum=100,
+            #     step=1,
+            #     value=5 if enable_lcm_arg else 15,
+            # )
+            # guidance_scale = gr.Slider(
+            #     label="Guidance scale",
+            #     minimum=0.1,
+            #     maximum=10.0,
+            #     step=0.1,
+            #     value=0 if enable_lcm_arg else 8.5,
+            # )
+            # if email is None:
+            #     print("STOPPPP")
+            #     raise gr.Error("Email ID is compulsory")
+            face_file.upload(
+                fn=remove_tips,
+                outputs=usage_tips,
+                queue=True,
+                api_name=False,
+                show_progress = "full"
+            ).then(
+                fn=run_for_prompts1,
+                inputs=[face_file,style],
+                outputs=[gallery1]
+            ).then(
+                fn=run_for_prompts2,
+                inputs=[face_file,style],
+                outputs=[gallery2]
+            ).then(
+                fn=run_for_prompts3,
+                inputs=[face_file,style],
+                outputs=[gallery3]
+            ).then(
+                fn=run_for_prompts4,
+                inputs=[face_file,style],
+                outputs=[gallery4]
+            )
+            submit.click(
+                fn=remove_tips,
+                outputs=usage_tips,
+                queue=True,
+                api_name=False,
+                show_progress = "full"
+            ).then(
+                fn=run_for_prompts1,
+                inputs=[face_file,style],
+                outputs=[gallery1]
+            ).then(
+                fn=run_for_prompts2,
+                inputs=[face_file,style],
+                outputs=[gallery2]
+            ).then(
+                fn=run_for_prompts3,
+                inputs=[face_file,style],
+                outputs=[gallery3]
+            ).then(
+                fn=run_for_prompts4,
+                inputs=[face_file,style],
+                outputs=[gallery4]
+            )
+        
+        
+        gr.Markdown(article)
+
+    demo.launch(share=True)
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--pretrained_model_name_or_path", type=str, default="wangqixun/YamerMIX_v8")
+    args = parser.parse_args()
+
+    main(args.pretrained_model_name_or_path, False)
+
+
+# import sys
+# sys.path.append('./')
+
+# from typing import Tuple
+
+# import os
+# import cv2
+# import math
+# import torch
+# import random
+# import numpy as np
+# import argparse
+
+# import PIL
+# from PIL import Image
+
+# import diffusers
+# from diffusers.utils import load_image
+# from diffusers.models import ControlNetModel
+# from diffusers import LCMScheduler
+
+# from huggingface_hub import hf_hub_download
+
+# import insightface
+# from insightface.app import FaceAnalysis
+
+# from style_template import styles
+# from pipeline_stable_diffusion_xl_instantid_full import StableDiffusionXLInstantIDPipeline
+# from model_util import load_models_xl, get_torch_device, torch_gc
+
+
+# # global variable
+# MAX_SEED = np.iinfo(np.int32).max
+# device = get_torch_device()
+# dtype = torch.float16 if str(device).__contains__("cuda") else torch.float32
+# STYLE_NAMES = list(styles.keys())
+# DEFAULT_STYLE_NAME = "Watercolor"
+
+# # Load face encoder
+# app = FaceAnalysis(name='antelopev2', root='./', providers=['CUDAExecutionProvider', 'CPUExecutionProvider'])
+# app.prepare(ctx_id=0, det_size=(320, 320))
+
+# # Path to InstantID models
+# face_adapter = f'./checkpoints/ip-adapter.bin'
+# controlnet_path = f'./checkpoints/ControlNetModel'
+
+# # Load pipeline
+# controlnet = ControlNetModel.from_pretrained(controlnet_path, torch_dtype=dtype)
+
+# logo = Image.open("./gradio_demo/logo.png")
+
+# from cv2 import imencode
+# import base64
+
+# # def encode_pil_to_base64_new(pil_image):
+# #     print("AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA")
+# #     image_arr = np.asarray(pil_image)[:,:,::-1]
+# #     _, byte_data = imencode('.png', image_arr)        
+# #     base64_data = base64.b64encode(byte_data)
+# #     base64_string_opencv = base64_data.decode("utf-8")
+# #     return "data:image/png;base64," + base64_string_opencv
+
+# import gradio as gr
+
+# # gr.processing_utils.encode_pil_to_base64 = encode_pil_to_base64_new
+
+# def main(pretrained_model_name_or_path="wangqixun/YamerMIX_v8", enable_lcm_arg=False):
+
+#     if pretrained_model_name_or_path.endswith(
+#             ".ckpt"
+#         ) or pretrained_model_name_or_path.endswith(".safetensors"):
+#             scheduler_kwargs = hf_hub_download(
+#                 repo_id="wangqixun/YamerMIX_v8",
+#                 subfolder="scheduler",
+#                 filename="scheduler_config.json",
+#             )
+
+#             (tokenizers, text_encoders, unet, _, vae) = load_models_xl(
+#                 pretrained_model_name_or_path=pretrained_model_name_or_path,
+#                 scheduler_name=None,
+#                 weight_dtype=dtype,
+#             )
+
+#             scheduler = diffusers.EulerDiscreteScheduler.from_config(scheduler_kwargs)
+#             pipe = StableDiffusionXLInstantIDPipeline(
+#                 vae=vae,
+#                 text_encoder=text_encoders[0],
+#                 text_encoder_2=text_encoders[1],
+#                 tokenizer=tokenizers[0],
+#                 tokenizer_2=tokenizers[1],
+#                 unet=unet,
+#                 scheduler=scheduler,
+#                 controlnet=controlnet,
+#             ).to(device)
+
+#     else:
+#         pipe = StableDiffusionXLInstantIDPipeline.from_pretrained(
+#             pretrained_model_name_or_path,
+#             controlnet=controlnet,
+#             torch_dtype=dtype,
+#             safety_checker=None,
+#             feature_extractor=None,
+#         ).to(device)
+
+#         pipe.scheduler = diffusers.EulerDiscreteScheduler.from_config(pipe.scheduler.config)
+
+#     pipe.load_ip_adapter_instantid(face_adapter)
+#     # load and disable LCM
+#     pipe.load_lora_weights("latent-consistency/lcm-lora-sdxl")
+#     pipe.disable_lora()
+    
+#     def remove_tips():
+#         return gr.update(visible=False)
+
+    
+#     # prompts = [
+#     #     ["superman","Vibrant Color"], ["japanese anime character with white/neon hair","Watercolor"], 
+#     #     # ["Suited professional","(No style)"], 
+#     #     ["Scooba diver","Line art"], ["eskimo","Snow"]
+#     # ]
+
+#     def convert_from_cv2_to_image(img: np.ndarray) -> Image:
+#         return Image.fromarray(cv2.cvtColor(img, cv2.COLOR_BGR2RGB))
+
+#     def convert_from_image_to_cv2(img: Image) -> np.ndarray:
+#         return cv2.cvtColor(np.array(img), cv2.COLOR_RGB2BGR)
+
+#     def run_for_prompts1(face_file,style,progress=gr.Progress(track_tqdm=True)):
+#         # if email != "":
+#         p,n = styles.get(style, styles.get(STYLE_NAMES[1]))
+#         return generate_image(face_file, p[0], n)
+#         # else:
+#             # raise gr.Error("Email ID is compulsory")
+#     def run_for_prompts2(face_file,style,progress=gr.Progress(track_tqdm=True)):
+#         # if email != "":
+#         p,n = styles.get(style, styles.get(STYLE_NAMES[1]))
+#         return generate_image(face_file, p[1], n)
+#     def run_for_prompts3(face_file,style,progress=gr.Progress(track_tqdm=True)):
+#         # if email != "":
+#         p,n = styles.get(style, styles.get(STYLE_NAMES[1]))
+#         return generate_image(face_file, p[2], n)
+#     def run_for_prompts4(face_file,style,progress=gr.Progress(track_tqdm=True)):
+#         # if email != "":
+#         p,n = styles.get(style, styles.get(STYLE_NAMES[1]))
+#         return generate_image(face_file, p[3], n)
+    
+# #     def validate_and_process(face_file, style, email):
+    
+# #     # Your processing logic here
+# #     gallery1, gallery2, gallery3, gallery4 = run_for_prompts1(face_file, style), run_for_prompts2(face_file, style), run_for_prompts3(face_file, style), run_for_prompts4(face_file, style)
+# #     return gallery1, gallery2, gallery3, gallery4
+
+#     def draw_kps(image_pil, kps, color_list=[(255,0,0), (0,255,0), (0,0,255), (255,255,0), (255,0,255)]):
+#         stickwidth = 4
+#         limbSeq = np.array([[0, 2], [1, 2], [3, 2], [4, 2]])
+#         kps = np.array(kps)
+
+#         w, h = image_pil.size
+#         out_img = np.zeros([h, w, 3])
+
+#         for i in range(len(limbSeq)):
+#             index = limbSeq[i]
+#             color = color_list[index[0]]
+
+#             x = kps[index][:, 0]
+#             y = kps[index][:, 1]
+#             length = ((x[0] - x[1]) ** 2 + (y[0] - y[1]) ** 2) ** 0.5
+#             angle = math.degrees(math.atan2(y[0] - y[1], x[0] - x[1]))
+#             polygon = cv2.ellipse2Poly((int(np.mean(x)), int(np.mean(y))), (int(length / 2), stickwidth), int(angle), 0, 360, 1)
+#             out_img = cv2.fillConvexPoly(out_img.copy(), polygon, color)
+#         out_img = (out_img * 0.6).astype(np.uint8)
+
+#         for idx_kp, kp in enumerate(kps):
+#             color = color_list[idx_kp]
+#             x, y = kp
+#             out_img = cv2.circle(out_img.copy(), (int(x), int(y)), 10, color, -1)
+
+#         out_img_pil = Image.fromarray(out_img.astype(np.uint8))
+#         return out_img_pil
+
+#     def resize_img(input_image, max_side=640, min_side=640, size=None, 
+#                 pad_to_max_side=True, mode=PIL.Image.BILINEAR, base_pixel_number=64):
+
+#             w, h = input_image.size
+#             print(w)
+#             print(h)
+#             if size is not None:
+#                 w_resize_new, h_resize_new = size
+#             else:
+#                 ratio = min_side / min(h, w)
+#                 w, h = round(ratio*w), round(ratio*h)
+#                 ratio = max_side / max(h, w)
+#                 input_image = input_image.resize([round(ratio*w), round(ratio*h)], mode)
+#                 w_resize_new = (round(ratio * w) // base_pixel_number) * base_pixel_number
+#                 h_resize_new = (round(ratio * h) // base_pixel_number) * base_pixel_number
+#             input_image = input_image.resize([w_resize_new, h_resize_new], mode)
+
+#             if pad_to_max_side:
+#                 res = np.ones([max_side, max_side, 3], dtype=np.uint8) * 255
+#                 offset_x = (max_side - w_resize_new) // 2
+#                 offset_y = (max_side - h_resize_new) // 2
+#                 res[offset_y:offset_y+h_resize_new, offset_x:offset_x+w_resize_new] = np.array(input_image)
+#                 input_image = Image.fromarray(res)
+#             return input_image
+
+#     # def apply_style(style_name: str, positive: str, negative: str = "") -> Tuple[str, str]:
+#     #     p, n = styles.get(style_name, styles[DEFAULT_STYLE_NAME])
+#     #     return p.replace("{prompt}", positive), n + ' ' + negative
+
+#     def generate_image(face_image,prompt,negative_prompt):
+#         pose_image_path = None
+#         # prompt = "superman"
+#         enable_LCM = False
+#         identitynet_strength_ratio = 0.95
+#         adapter_strength_ratio = 0.60
+#         num_steps = 15
+#         guidance_scale = 8.5
+#         seed = random.randint(0, MAX_SEED)
+#         # negative_prompt = ""
+#         # negative_prompt += neg
+#         enhance_face_region = True
+#         if enable_LCM:
+#             pipe.enable_lora()
+#             pipe.scheduler = LCMScheduler.from_config(pipe.scheduler.config)
+#         else:
+#             pipe.disable_lora()
+#             pipe.scheduler = diffusers.EulerDiscreteScheduler.from_config(pipe.scheduler.config)
+    
+#         if face_image is None:
+#             raise gr.Error(f"Cannot find any input face image! Please upload the face image")
+        
+#         # if prompt is None:
+#         #     prompt = "a person"
+        
+#         # apply the style template
+#         # prompt, negative_prompt = apply_style(style_name, prompt, negative_prompt)
+        
+#         # face_image = load_image(face_image_path)
+#         face_image = resize_img(face_image)
+#         face_image_cv2 = convert_from_image_to_cv2(face_image)
+#         height, width, _ = face_image_cv2.shape
+        
+#         # Extract face features
+#         face_info = app.get(face_image_cv2)
+        
+#         if len(face_info) == 0:
+#             raise gr.Error(f"Cannot find any face in the image! Please upload another person image")
+        
+#         face_info = sorted(face_info, key=lambda x:(x['bbox'][2]-x['bbox'][0])*(x['bbox'][3]-x['bbox'][1]))[-1]  # only use the maximum face
+#         face_emb = face_info['embedding']
+#         face_kps = draw_kps(convert_from_cv2_to_image(face_image_cv2), face_info['kps'])
+        
+#         if pose_image_path is not None:
+#             pose_image = load_image(pose_image_path)
+#             pose_image = resize_img(pose_image)
+#             pose_image_cv2 = convert_from_image_to_cv2(pose_image)
+            
+#             face_info = app.get(pose_image_cv2)
+            
+#             if len(face_info) == 0:
+#                 raise gr.Error(f"Cannot find any face in the reference image! Please upload another person image")
+            
+#             face_info = face_info[-1]
+#             face_kps = draw_kps(pose_image, face_info['kps'])
+            
+#             width, height = face_kps.size
+
+#         if enhance_face_region:
+#             control_mask = np.zeros([height, width, 3])
+#             x1, y1, x2, y2 = face_info["bbox"]
+#             x1, y1, x2, y2 = int(x1), int(y1), int(x2), int(y2)
+#             control_mask[y1:y2, x1:x2] = 255
+#             control_mask = Image.fromarray(control_mask.astype(np.uint8))
+#         else:
+#             control_mask = None
+                        
+#         generator = torch.Generator(device=device).manual_seed(seed)
+        
+#         print("Start inference...")
+#         print(f"[Debug] Prompt: {prompt}, \n[Debug] Neg Prompt: {negative_prompt}")
+        
+#         pipe.set_ip_adapter_scale(adapter_strength_ratio)
+#         images = pipe(
+#             prompt=prompt,
+#             negative_prompt=negative_prompt,
+#             image_embeds=face_emb,
+#             image=face_kps,
+#             control_mask=control_mask,
+#             controlnet_conditioning_scale=float(identitynet_strength_ratio),
+#             num_inference_steps=num_steps,
+#             guidance_scale=guidance_scale,
+#             height=height,
+#             width=width,
+#             generator=generator,
+#             # num_images_per_prompt = 4
+#         ).images
+
+#         return images[0]
+
+#     ### Description
+#     title = r"""
+#     <h1 align="center">Choose your AVATAR</h1>
+#     """
+
+#     description = r"""
+#     <h2> Powered by IDfy </h2>"""
+
+#     article = r""""""
+
+#     tips = r""""""
+
+#     css = '''
+#     .gradio-container {width: 95% !important; background-color: #E6F3FF;} 
+#     .image-gallery {height: 100vh !important; overflow: auto;}
+#     .gradio-row .gradio-element { margin: 0 !important; }
+#     '''
+#     with gr.Blocks(css=css) as demo:
+
+#         # description
+#         gr.Markdown(title)
+#         with gr.Row():
+#             gr.Image("./gradio_demo/logo.png",scale=0,min_width=50,show_label=False,show_download_button=False)
+#             gr.Markdown(description)
+#         with gr.Row():
+#             with gr.Column():
+#                 style = gr.Dropdown(label="Choose your STYLE", choices=STYLE_NAMES)
+#                 face_file = gr.Image(label="Upload a photo of your face", type="pil",sources="webcam")
+#                 submit = gr.Button("Submit", variant="primary")
+#             with gr.Column():
+#                 with gr.Row():
+#                     gallery1 = gr.Image(label="Generated Images")
+#                     gallery2 = gr.Image(label="Generated Images")
+#                 with gr.Row():
+#                     gallery3 = gr.Image(label="Generated Images")
+#                     gallery4 = gr.Image(label="Generated Images")
+#                 email = gr.Textbox(label="Email",
+#                         info="Enter your email address",
+#                         value="")
+            
+#             usage_tips = gr.Markdown(label="Usage tips of InstantID", value=tips ,visible=False)
+#             # identitynet_strength_ratio = gr.Slider(
+#             #     label="IdentityNet strength (for fidelity)",
+#             #     minimum=0,
+#             #     maximum=1.5,
+#             #     step=0.05,
+#             #     value=0.95,
+#             # )
+#             # adapter_strength_ratio = gr.Slider(
+#             #     label="Image adapter strength (for detail)",
+#             #     minimum=0,
+#             #     maximum=1.5,
+#             #     step=0.05,
+#             #     value=0.60,
+#             # )
+#             # negative_prompt = gr.Textbox(
+#             #     label="Negative Prompt", 
+#             #     placeholder="low quality",
+#             #     value="(lowres, low quality, worst quality:1.2), (text:1.2), watermark, (frame:1.2), deformed, ugly, deformed eyes, blur, out of focus, blurry, deformed cat, deformed, photo, anthropomorphic cat, monochrome, pet collar, gun, weapon, blue, 3d, drones, drone, buildings in background, green",
+#             # )
+#             # num_steps = gr.Slider( 
+#             #     label="Number of sample steps",
+#             #     minimum=15,
+#             #     maximum=100,
+#             #     step=1,
+#             #     value=5 if enable_lcm_arg else 15,
+#             # )
+#             # guidance_scale = gr.Slider(
+#             #     label="Guidance scale",
+#             #     minimum=0.1,
+#             #     maximum=10.0,
+#             #     step=0.1,
+#             #     value=0 if enable_lcm_arg else 8.5,
+#             # )
+#             # if email is None:
+#             #     print("STOPPPP")
+#             #     raise gr.Error("Email ID is compulsory")
+#             face_file.upload(
+#                 fn=remove_tips,
+#                 outputs=usage_tips,
+#                 queue=True,
+#                 api_name=False,
+#                 show_progress = "full"
+#             ).then(
+#                 fn=run_for_prompts1,
+#                 inputs=[face_file,style],
+#                 outputs=[gallery1]
+#             ).then(
+#                 fn=run_for_prompts2,
+#                 inputs=[face_file,style],
+#                 outputs=[gallery2]
+#             ).then(
+#                 fn=run_for_prompts3,
+#                 inputs=[face_file,style],
+#                 outputs=[gallery3]
+#             ).then(
+#                 fn=run_for_prompts4,
+#                 inputs=[face_file,style],
+#                 outputs=[gallery4]
+#             )
+#             submit.click(
+#                 fn=remove_tips,
+#                 outputs=usage_tips,
+#                 queue=True,
+#                 api_name=False,
+#                 show_progress = "full"
+#             ).then(
+#                 fn=run_for_prompts1,
+#                 inputs=[face_file,style],
+#                 outputs=[gallery1]
+#             ).then(
+#                 fn=run_for_prompts2,
+#                 inputs=[face_file,style],
+#                 outputs=[gallery2]
+#             ).then(
+#                 fn=run_for_prompts3,
+#                 inputs=[face_file,style],
+#                 outputs=[gallery3]
+#             ).then(
+#                 fn=run_for_prompts4,
+#                 inputs=[face_file,style],
+#                 outputs=[gallery4]
+#             )
+        
+        
+#         gr.Markdown(article)
+
+#     demo.launch(share=True)
+
+# if __name__ == "__main__":
+#     parser = argparse.ArgumentParser()
+#     parser.add_argument("--pretrained_model_name_or_path", type=str, default="wangqixun/YamerMIX_v8")
+#     args = parser.parse_args()
+
+#     main(args.pretrained_model_name_or_path, False)

gradio_demo/app-multicontrolnet.py

@@ -0,0 +1,670 @@
+import sys
+sys.path.append("./")
+
+from typing import Tuple
+
+import os
+import cv2
+import math
+import torch
+import random
+import numpy as np
+import argparse
+
+import PIL
+from PIL import Image
+
+import diffusers
+from diffusers.utils import load_image
+from diffusers.models import ControlNetModel
+from diffusers.pipelines.controlnet.multicontrolnet import MultiControlNetModel
+
+from huggingface_hub import hf_hub_download
+
+from insightface.app import FaceAnalysis
+
+from style_template import styles
+from pipeline_stable_diffusion_xl_instantid_full import StableDiffusionXLInstantIDPipeline
+from model_util import load_models_xl, get_torch_device, torch_gc
+from controlnet_util import openpose, get_depth_map, get_canny_image
+
+import gradio as gr
+
+
+# global variable
+MAX_SEED = np.iinfo(np.int32).max
+device = get_torch_device()
+dtype = torch.float16 if str(device).__contains__("cuda") else torch.float32
+STYLE_NAMES = list(styles.keys())
+DEFAULT_STYLE_NAME = "Watercolor"
+
+# Load face encoder
+app = FaceAnalysis(
+    name="antelopev2",
+    root="./",
+    providers=["CUDAExecutionProvider", "CPUExecutionProvider"],
+)
+app.prepare(ctx_id=0, det_size=(640, 640))
+
+# Path to InstantID models
+face_adapter = f"./checkpoints/ip-adapter.bin"
+controlnet_path = f"./checkpoints/ControlNetModel"
+
+# Load pipeline face ControlNetModel
+controlnet_identitynet = ControlNetModel.from_pretrained(
+    controlnet_path, torch_dtype=dtype
+)
+
+# controlnet-pose
+controlnet_pose_model = "thibaud/controlnet-openpose-sdxl-1.0"
+controlnet_canny_model = "diffusers/controlnet-canny-sdxl-1.0"
+controlnet_depth_model = "diffusers/controlnet-depth-sdxl-1.0-small"
+
+controlnet_pose = ControlNetModel.from_pretrained(
+    controlnet_pose_model, torch_dtype=dtype
+).to(device)
+controlnet_canny = ControlNetModel.from_pretrained(
+    controlnet_canny_model, torch_dtype=dtype
+).to(device)
+controlnet_depth = ControlNetModel.from_pretrained(
+    controlnet_depth_model, torch_dtype=dtype
+).to(device)
+
+controlnet_map = {
+    "pose": controlnet_pose,
+    "canny": controlnet_canny,
+    "depth": controlnet_depth,
+}
+controlnet_map_fn = {
+    "pose": openpose,
+    "canny": get_canny_image,
+    "depth": get_depth_map,
+}
+
+
+def main(pretrained_model_name_or_path="wangqixun/YamerMIX_v8", enable_lcm_arg=False):
+    if pretrained_model_name_or_path.endswith(
+        ".ckpt"
+    ) or pretrained_model_name_or_path.endswith(".safetensors"):
+        scheduler_kwargs = hf_hub_download(
+            repo_id="wangqixun/YamerMIX_v8",
+            subfolder="scheduler",
+            filename="scheduler_config.json",
+        )
+
+        (tokenizers, text_encoders, unet, _, vae) = load_models_xl(
+            pretrained_model_name_or_path=pretrained_model_name_or_path,
+            scheduler_name=None,
+            weight_dtype=dtype,
+        )
+
+        scheduler = diffusers.EulerDiscreteScheduler.from_config(scheduler_kwargs)
+        pipe = StableDiffusionXLInstantIDPipeline(
+            vae=vae,
+            text_encoder=text_encoders[0],
+            text_encoder_2=text_encoders[1],
+            tokenizer=tokenizers[0],
+            tokenizer_2=tokenizers[1],
+            unet=unet,
+            scheduler=scheduler,
+            controlnet=[controlnet_identitynet],
+        ).to(device)
+
+    else:
+        pipe = StableDiffusionXLInstantIDPipeline.from_pretrained(
+            pretrained_model_name_or_path,
+            controlnet=[controlnet_identitynet],
+            torch_dtype=dtype,
+            safety_checker=None,
+            feature_extractor=None,
+        ).to(device)
+
+        pipe.scheduler = diffusers.EulerDiscreteScheduler.from_config(
+            pipe.scheduler.config
+        )
+
+    pipe.load_ip_adapter_instantid(face_adapter)
+    # load and disable LCM
+    pipe.load_lora_weights("latent-consistency/lcm-lora-sdxl")
+    pipe.disable_lora()
+
+    def toggle_lcm_ui(value):
+        if value:
+            return (
+                gr.update(minimum=0, maximum=100, step=1, value=5),
+                gr.update(minimum=0.1, maximum=20.0, step=0.1, value=1.5),
+            )
+        else:
+            return (
+                gr.update(minimum=5, maximum=100, step=1, value=30),
+                gr.update(minimum=0.1, maximum=20.0, step=0.1, value=5),
+            )
+
+    def randomize_seed_fn(seed: int, randomize_seed: bool) -> int:
+        if randomize_seed:
+            seed = random.randint(0, MAX_SEED)
+        return seed
+
+    def remove_tips():
+        return gr.update(visible=False)
+
+    def get_example():
+        case = [
+            [
+                "./examples/yann-lecun_resize.jpg",
+                None,
+                "a man",
+                "Snow",
+                "(lowres, low quality, worst quality:1.2), (text:1.2), watermark, (frame:1.2), deformed, ugly, deformed eyes, blur, out of focus, blurry, deformed cat, deformed, photo, anthropomorphic cat, monochrome, photo, pet collar, gun, weapon, blue, 3d, drones, drone, buildings in background, green",
+            ],
+            [
+                "./examples/musk_resize.jpeg",
+                "./examples/poses/pose2.jpg",
+                "a man flying in the sky in Mars",
+                "Mars",
+                "(lowres, low quality, worst quality:1.2), (text:1.2), watermark, (frame:1.2), deformed, ugly, deformed eyes, blur, out of focus, blurry, deformed cat, deformed, photo, anthropomorphic cat, monochrome, photo, pet collar, gun, weapon, blue, 3d, drones, drone, buildings in background, green",
+            ],
+            [
+                "./examples/sam_resize.png",
+                "./examples/poses/pose4.jpg",
+                "a man doing a silly pose wearing a suite",
+                "Jungle",
+                "(lowres, low quality, worst quality:1.2), (text:1.2), watermark, (frame:1.2), deformed, ugly, deformed eyes, blur, out of focus, blurry, deformed cat, deformed, photo, anthropomorphic cat, monochrome, photo, pet collar, gun, weapon, blue, 3d, drones, drone, buildings in background, gree",
+            ],
+            [
+                "./examples/schmidhuber_resize.png",
+                "./examples/poses/pose3.jpg",
+                "a man sit on a chair",
+                "Neon",
+                "(lowres, low quality, worst quality:1.2), (text:1.2), watermark, (frame:1.2), deformed, ugly, deformed eyes, blur, out of focus, blurry, deformed cat, deformed, photo, anthropomorphic cat, monochrome, photo, pet collar, gun, weapon, blue, 3d, drones, drone, buildings in background, green",
+            ],
+            [
+                "./examples/kaifu_resize.png",
+                "./examples/poses/pose.jpg",
+                "a man",
+                "Vibrant Color",
+                "(lowres, low quality, worst quality:1.2), (text:1.2), watermark, (frame:1.2), deformed, ugly, deformed eyes, blur, out of focus, blurry, deformed cat, deformed, photo, anthropomorphic cat, monochrome, photo, pet collar, gun, weapon, blue, 3d, drones, drone, buildings in background, green",
+            ],
+        ]
+        return case
+
+    def run_for_examples(face_file, pose_file, prompt, style, negative_prompt):
+        return generate_image(
+            face_file,
+            pose_file,
+            prompt,
+            negative_prompt,
+            style,
+            20,  # num_steps
+            0.8,  # identitynet_strength_ratio
+            0.8,  # adapter_strength_ratio
+            0.4,  # pose_strength
+            0.3,  # canny_strength
+            0.5,  # depth_strength
+            ["pose", "canny"],  # controlnet_selection
+            5.0,  # guidance_scale
+            42,  # seed
+            "EulerDiscreteScheduler",  # scheduler
+            False,  # enable_LCM
+            True,  # enable_Face_Region
+        )
+
+    def convert_from_cv2_to_image(img: np.ndarray) -> Image:
+        return Image.fromarray(cv2.cvtColor(img, cv2.COLOR_BGR2RGB))
+
+    def convert_from_image_to_cv2(img: Image) -> np.ndarray:
+        return cv2.cvtColor(np.array(img), cv2.COLOR_RGB2BGR)
+
+    def draw_kps(
+        image_pil,
+        kps,
+        color_list=[
+            (255, 0, 0),
+            (0, 255, 0),
+            (0, 0, 255),
+            (255, 255, 0),
+            (255, 0, 255),
+        ],
+    ):
+        stickwidth = 4
+        limbSeq = np.array([[0, 2], [1, 2], [3, 2], [4, 2]])
+        kps = np.array(kps)
+
+        w, h = image_pil.size
+        out_img = np.zeros([h, w, 3])
+
+        for i in range(len(limbSeq)):
+            index = limbSeq[i]
+            color = color_list[index[0]]
+
+            x = kps[index][:, 0]
+            y = kps[index][:, 1]
+            length = ((x[0] - x[1]) ** 2 + (y[0] - y[1]) ** 2) ** 0.5
+            angle = math.degrees(math.atan2(y[0] - y[1], x[0] - x[1]))
+            polygon = cv2.ellipse2Poly(
+                (int(np.mean(x)), int(np.mean(y))),
+                (int(length / 2), stickwidth),
+                int(angle),
+                0,
+                360,
+                1,
+            )
+            out_img = cv2.fillConvexPoly(out_img.copy(), polygon, color)
+        out_img = (out_img * 0.6).astype(np.uint8)
+
+        for idx_kp, kp in enumerate(kps):
+            color = color_list[idx_kp]
+            x, y = kp
+            out_img = cv2.circle(out_img.copy(), (int(x), int(y)), 10, color, -1)
+
+        out_img_pil = Image.fromarray(out_img.astype(np.uint8))
+        return out_img_pil
+
+    def resize_img(
+        input_image,
+        max_side=1280,
+        min_side=1024,
+        size=None,
+        pad_to_max_side=False,
+        mode=PIL.Image.BILINEAR,
+        base_pixel_number=64,
+    ):
+        w, h = input_image.size
+        if size is not None:
+            w_resize_new, h_resize_new = size
+        else:
+            ratio = min_side / min(h, w)
+            w, h = round(ratio * w), round(ratio * h)
+            ratio = max_side / max(h, w)
+            input_image = input_image.resize([round(ratio * w), round(ratio * h)], mode)
+            w_resize_new = (round(ratio * w) // base_pixel_number) * base_pixel_number
+            h_resize_new = (round(ratio * h) // base_pixel_number) * base_pixel_number
+        input_image = input_image.resize([w_resize_new, h_resize_new], mode)
+
+        if pad_to_max_side:
+            res = np.ones([max_side, max_side, 3], dtype=np.uint8) * 255
+            offset_x = (max_side - w_resize_new) // 2
+            offset_y = (max_side - h_resize_new) // 2
+            res[
+                offset_y : offset_y + h_resize_new, offset_x : offset_x + w_resize_new
+            ] = np.array(input_image)
+            input_image = Image.fromarray(res)
+        return input_image
+
+    def apply_style(
+        style_name: str, positive: str, negative: str = ""
+    ) -> Tuple[str, str]:
+        p, n = styles.get(style_name, styles[DEFAULT_STYLE_NAME])
+        return p.replace("{prompt}", positive), n + " " + negative
+
+    def generate_image(
+        face_image_path,
+        pose_image_path,
+        prompt,
+        negative_prompt,
+        style_name,
+        num_steps,
+        identitynet_strength_ratio,
+        adapter_strength_ratio,
+        pose_strength,
+        canny_strength,
+        depth_strength,
+        controlnet_selection,
+        guidance_scale,
+        seed,
+        scheduler,
+        enable_LCM,
+        enhance_face_region,
+        progress=gr.Progress(track_tqdm=True),
+    ):
+
+        if enable_LCM:
+            pipe.scheduler = diffusers.LCMScheduler.from_config(pipe.scheduler.config)
+            pipe.enable_lora()
+        else:
+            pipe.disable_lora()
+            scheduler_class_name = scheduler.split("-")[0]
+
+            add_kwargs = {}
+            if len(scheduler.split("-")) > 1:
+                add_kwargs["use_karras_sigmas"] = True
+            if len(scheduler.split("-")) > 2:
+                add_kwargs["algorithm_type"] = "sde-dpmsolver++"
+            scheduler = getattr(diffusers, scheduler_class_name)
+            pipe.scheduler = scheduler.from_config(pipe.scheduler.config, **add_kwargs)
+
+        if face_image_path is None:
+            raise gr.Error(
+                f"Cannot find any input face image! Please upload the face image"
+            )
+
+        if prompt is None:
+            prompt = "a person"
+
+        # apply the style template
+        prompt, negative_prompt = apply_style(style_name, prompt, negative_prompt)
+
+        face_image = load_image(face_image_path)
+        face_image = resize_img(face_image, max_side=1024)
+        face_image_cv2 = convert_from_image_to_cv2(face_image)
+        height, width, _ = face_image_cv2.shape
+
+        # Extract face features
+        face_info = app.get(face_image_cv2)
+
+        if len(face_info) == 0:
+            raise gr.Error(
+                f"Unable to detect a face in the image. Please upload a different photo with a clear face."
+            )
+
+        face_info = sorted(face_info, key=lambda x:(x['bbox'][2]-x['bbox'][0])*(x['bbox'][3]-x['bbox'][1]))[-1]  # only use the maximum face
+        face_emb = face_info["embedding"]
+        face_kps = draw_kps(convert_from_cv2_to_image(face_image_cv2), face_info["kps"])
+        img_controlnet = face_image
+        if pose_image_path is not None:
+            pose_image = load_image(pose_image_path)
+            pose_image = resize_img(pose_image, max_side=1024)
+            img_controlnet = pose_image
+            pose_image_cv2 = convert_from_image_to_cv2(pose_image)
+
+            face_info = app.get(pose_image_cv2)
+
+            if len(face_info) == 0:
+                raise gr.Error(
+                    f"Cannot find any face in the reference image! Please upload another person image"
+                )
+
+            face_info = face_info[-1]
+            face_kps = draw_kps(pose_image, face_info["kps"])
+
+            width, height = face_kps.size
+
+        if enhance_face_region:
+            control_mask = np.zeros([height, width, 3])
+            x1, y1, x2, y2 = face_info["bbox"]
+            x1, y1, x2, y2 = int(x1), int(y1), int(x2), int(y2)
+            control_mask[y1:y2, x1:x2] = 255
+            control_mask = Image.fromarray(control_mask.astype(np.uint8))
+        else:
+            control_mask = None
+
+        if len(controlnet_selection) > 0:
+            controlnet_scales = {
+                "pose": pose_strength,
+                "canny": canny_strength,
+                "depth": depth_strength,
+            }
+            pipe.controlnet = MultiControlNetModel(
+                [controlnet_identitynet]
+                + [controlnet_map[s] for s in controlnet_selection]
+            )
+            control_scales = [float(identitynet_strength_ratio)] + [
+                controlnet_scales[s] for s in controlnet_selection
+            ]
+            control_images = [face_kps] + [
+                controlnet_map_fn[s](img_controlnet).resize((width, height))
+                for s in controlnet_selection
+            ]
+        else:
+            pipe.controlnet = controlnet_identitynet
+            control_scales = float(identitynet_strength_ratio)
+            control_images = face_kps
+
+        generator = torch.Generator(device=device).manual_seed(seed)
+
+        print("Start inference...")
+        print(f"[Debug] Prompt: {prompt}, \n[Debug] Neg Prompt: {negative_prompt}")
+
+        pipe.set_ip_adapter_scale(adapter_strength_ratio)
+        images = pipe(
+            prompt=prompt,
+            negative_prompt=negative_prompt,
+            image_embeds=face_emb,
+            image=control_images,
+            control_mask=control_mask,
+            controlnet_conditioning_scale=control_scales,
+            num_inference_steps=num_steps,
+            guidance_scale=guidance_scale,
+            height=height,
+            width=width,
+            generator=generator,
+        ).images
+
+        return images[0], gr.update(visible=True)
+
+    # Description
+    title = r"""
+    <h1 align="center">InstantID: Zero-shot Identity-Preserving Generation in Seconds</h1>
+    """
+
+    description = r"""
+    <b>Official 🤗 Gradio demo</b> for <a href='https://github.com/InstantID/InstantID' target='_blank'><b>InstantID: Zero-shot Identity-Preserving Generation in Seconds</b></a>.<br>
+
+    How to use:<br>
+    1. Upload an image with a face. For images with multiple faces, we will only detect the largest face. Ensure the face is not too small and is clearly visible without significant obstructions or blurring.
+    2. (Optional) You can upload another image as a reference for the face pose. If you don't, we will use the first detected face image to extract facial landmarks. If you use a cropped face at step 1, it is recommended to upload it to define a new face pose.
+    3. (Optional) You can select multiple ControlNet models to control the generation process. The default is to use the IdentityNet only. The ControlNet models include pose skeleton, canny, and depth. You can adjust the strength of each ControlNet model to control the generation process.
+    4. Enter a text prompt, as done in normal text-to-image models.
+    5. Click the <b>Submit</b> button to begin customization.
+    6. Share your customized photo with your friends and enjoy! 😊"""
+
+    article = r"""
+    ---
+    📝 **Citation**
+    <br>
+    If our work is helpful for your research or applications, please cite us via:
+    ```bibtex
+    @article{wang2024instantid,
+    title={InstantID: Zero-shot Identity-Preserving Generation in Seconds},
+    author={Wang, Qixun and Bai, Xu and Wang, Haofan and Qin, Zekui and Chen, Anthony},
+    journal={arXiv preprint arXiv:2401.07519},
+    year={2024}
+    }
+    ```
+    📧 **Contact**
+    <br>
+    If you have any questions, please feel free to open an issue or directly reach us out at <b>[email protected]</b>.
+    """
+
+    tips = r"""
+    ### Usage tips of InstantID
+    1. If you're not satisfied with the similarity, try increasing the weight of "IdentityNet Strength" and "Adapter Strength."    
+    2. If you feel that the saturation is too high, first decrease the Adapter strength. If it remains too high, then decrease the IdentityNet strength.
+    3. If you find that text control is not as expected, decrease Adapter strength.
+    4. If you find that realistic style is not good enough, go for our Github repo and use a more realistic base model.
+    """
+
+    css = """
+    .gradio-container {width: 85% !important}
+    """
+    with gr.Blocks(css=css) as demo:
+        # description
+        gr.Markdown(title)
+        gr.Markdown(description)
+
+        with gr.Row():
+            with gr.Column():
+                with gr.Row(equal_height=True):
+                    # upload face image
+                    face_file = gr.Image(
+                        label="Upload a photo of your face", type="filepath"
+                    )
+                    # optional: upload a reference pose image
+                    pose_file = gr.Image(
+                        label="Upload a reference pose image (Optional)",
+                        type="filepath",
+                    )
+
+                # prompt
+                prompt = gr.Textbox(
+                    label="Prompt",
+                    info="Give simple prompt is enough to achieve good face fidelity",
+                    placeholder="A photo of a person",
+                    value="",
+                )
+
+                submit = gr.Button("Submit", variant="primary")
+                enable_LCM = gr.Checkbox(
+                    label="Enable Fast Inference with LCM", value=enable_lcm_arg,
+                    info="LCM speeds up the inference step, the trade-off is the quality of the generated image. It performs better with portrait face images rather than distant faces",
+                )
+                style = gr.Dropdown(
+                    label="Style template",
+                    choices=STYLE_NAMES,
+                    value=DEFAULT_STYLE_NAME,
+                )
+
+                # strength
+                identitynet_strength_ratio = gr.Slider(
+                    label="IdentityNet strength (for fidelity)",
+                    minimum=0,
+                    maximum=1.5,
+                    step=0.05,
+                    value=0.80,
+                )
+                adapter_strength_ratio = gr.Slider(
+                    label="Image adapter strength (for detail)",
+                    minimum=0,
+                    maximum=1.5,
+                    step=0.05,
+                    value=0.80,
+                )
+                with gr.Accordion("Controlnet"):
+                    controlnet_selection = gr.CheckboxGroup(
+                        ["pose", "canny", "depth"], label="Controlnet", value=["pose"],
+                        info="Use pose for skeleton inference, canny for edge detection, and depth for depth map estimation. You can try all three to control the generation process"
+                    )
+                    pose_strength = gr.Slider(
+                        label="Pose strength",
+                        minimum=0,
+                        maximum=1.5,
+                        step=0.05,
+                        value=0.40,
+                    )
+                    canny_strength = gr.Slider(
+                        label="Canny strength",
+                        minimum=0,
+                        maximum=1.5,
+                        step=0.05,
+                        value=0.40,
+                    )
+                    depth_strength = gr.Slider(
+                        label="Depth strength",
+                        minimum=0,
+                        maximum=1.5,
+                        step=0.05,
+                        value=0.40,
+                    )
+                with gr.Accordion(open=False, label="Advanced Options"):
+                    negative_prompt = gr.Textbox(
+                        label="Negative Prompt",
+                        placeholder="low quality",
+                        value="(lowres, low quality, worst quality:1.2), (text:1.2), watermark, (frame:1.2), deformed, ugly, deformed eyes, blur, out of focus, blurry, deformed cat, deformed, photo, anthropomorphic cat, monochrome, pet collar, gun, weapon, blue, 3d, drones, drone, buildings in background, green",
+                    )
+                    num_steps = gr.Slider(
+                        label="Number of sample steps",
+                        minimum=1,
+                        maximum=100,
+                        step=1,
+                        value=5 if enable_lcm_arg else 30,
+                    )
+                    guidance_scale = gr.Slider(
+                        label="Guidance scale",
+                        minimum=0.1,
+                        maximum=20.0,
+                        step=0.1,
+                        value=0.0 if enable_lcm_arg else 5.0,
+                    )
+                    seed = gr.Slider(
+                        label="Seed",
+                        minimum=0,
+                        maximum=MAX_SEED,
+                        step=1,
+                        value=42,
+                    )
+                    schedulers = [
+                        "DEISMultistepScheduler",
+                        "HeunDiscreteScheduler",
+                        "EulerDiscreteScheduler",
+                        "DPMSolverMultistepScheduler",
+                        "DPMSolverMultistepScheduler-Karras",
+                        "DPMSolverMultistepScheduler-Karras-SDE",
+                    ]
+                    scheduler = gr.Dropdown(
+                        label="Schedulers",
+                        choices=schedulers,
+                        value="EulerDiscreteScheduler",
+                    )
+                    randomize_seed = gr.Checkbox(label="Randomize seed", value=True)
+                    enhance_face_region = gr.Checkbox(label="Enhance non-face region", value=True)
+
+            with gr.Column(scale=1):
+                gallery = gr.Image(label="Generated Images")
+                usage_tips = gr.Markdown(
+                    label="InstantID Usage Tips", value=tips, visible=False
+                )
+
+            submit.click(
+                fn=remove_tips,
+                outputs=usage_tips,
+            ).then(
+                fn=randomize_seed_fn,
+                inputs=[seed, randomize_seed],
+                outputs=seed,
+                queue=False,
+                api_name=False,
+            ).then(
+                fn=generate_image,
+                inputs=[
+                    face_file,
+                    pose_file,
+                    prompt,
+                    negative_prompt,
+                    style,
+                    num_steps,
+                    identitynet_strength_ratio,
+                    adapter_strength_ratio,
+                    pose_strength,
+                    canny_strength,
+                    depth_strength,
+                    controlnet_selection,
+                    guidance_scale,
+                    seed,
+                    scheduler,
+                    enable_LCM,
+                    enhance_face_region,
+                ],
+                outputs=[gallery, usage_tips],
+            )
+
+            enable_LCM.input(
+                fn=toggle_lcm_ui,
+                inputs=[enable_LCM],
+                outputs=[num_steps, guidance_scale],
+                queue=False,
+            )
+
+        gr.Examples(
+            examples=get_example(),
+            inputs=[face_file, pose_file, prompt, style, negative_prompt],
+            fn=run_for_examples,
+            outputs=[gallery, usage_tips],
+            cache_examples=True,
+        )
+
+        gr.Markdown(article)
+
+    demo.launch()
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        "--pretrained_model_name_or_path", type=str, default="wangqixun/YamerMIX_v8"
+    )
+    parser.add_argument(
+        "--enable_LCM", type=bool, default=os.environ.get("ENABLE_LCM", False)
+    )
+    args = parser.parse_args()
+
+    main(args.pretrained_model_name_or_path, args.enable_LCM)

gradio_demo/app.py

@@ -0,0 +1,524 @@
+import sys
+sys.path.append('./')
+
+from typing import Tuple
+
+import os
+import cv2
+import math
+import torch
+import random
+import numpy as np
+import argparse
+
+import PIL
+from PIL import Image
+
+import diffusers
+from diffusers.utils import load_image
+from diffusers.models import ControlNetModel
+from diffusers import LCMScheduler
+
+from huggingface_hub import hf_hub_download
+
+import insightface
+from insightface.app import FaceAnalysis
+
+from style_template import styles
+from pipeline_stable_diffusion_xl_instantid_full import StableDiffusionXLInstantIDPipeline
+from model_util import load_models_xl, get_torch_device, torch_gc
+
+
+# global variable
+MAX_SEED = np.iinfo(np.int32).max
+device = get_torch_device()
+dtype = torch.float16 if str(device).__contains__("cuda") else torch.float32
+STYLE_NAMES = list(styles.keys())
+DEFAULT_STYLE_NAME = "Watercolor"
+
+# Load face encoder
+app = FaceAnalysis(name='antelopev2', root='./', providers=['CUDAExecutionProvider', 'CPUExecutionProvider'])
+app.prepare(ctx_id=0, det_size=(640, 640))
+
+# Path to InstantID models
+face_adapter = f'./checkpoints/ip-adapter.bin'
+controlnet_path = f'./checkpoints/ControlNetModel'
+
+# Load pipeline
+controlnet = ControlNetModel.from_pretrained(controlnet_path, torch_dtype=dtype)
+
+logo = Image.open("./gradio_demo/logo.png")
+
+from cv2 import imencode
+import base64
+
+# def encode_pil_to_base64_new(pil_image):
+#     print("AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA")
+#     image_arr = np.asarray(pil_image)[:,:,::-1]
+#     _, byte_data = imencode('.png', image_arr)        
+#     base64_data = base64.b64encode(byte_data)
+#     base64_string_opencv = base64_data.decode("utf-8")
+#     return "data:image/png;base64," + base64_string_opencv
+
+import gradio as gr
+
+# gr.processing_utils.encode_pil_to_base64 = encode_pil_to_base64_new
+
+def main(pretrained_model_name_or_path="wangqixun/YamerMIX_v8", enable_lcm_arg=False):
+
+    if pretrained_model_name_or_path.endswith(
+            ".ckpt"
+        ) or pretrained_model_name_or_path.endswith(".safetensors"):
+            scheduler_kwargs = hf_hub_download(
+                repo_id="wangqixun/YamerMIX_v8",
+                subfolder="scheduler",
+                filename="scheduler_config.json",
+            )
+
+            (tokenizers, text_encoders, unet, _, vae) = load_models_xl(
+                pretrained_model_name_or_path=pretrained_model_name_or_path,
+                scheduler_name=None,
+                weight_dtype=dtype,
+            )
+
+            scheduler = diffusers.EulerDiscreteScheduler.from_config(scheduler_kwargs)
+            pipe = StableDiffusionXLInstantIDPipeline(
+                vae=vae,
+                text_encoder=text_encoders[0],
+                text_encoder_2=text_encoders[1],
+                tokenizer=tokenizers[0],
+                tokenizer_2=tokenizers[1],
+                unet=unet,
+                scheduler=scheduler,
+                controlnet=controlnet,
+            ).to(device)
+
+    else:
+        pipe = StableDiffusionXLInstantIDPipeline.from_pretrained(
+            pretrained_model_name_or_path,
+            controlnet=controlnet,
+            torch_dtype=dtype,
+            safety_checker=None,
+            feature_extractor=None,
+        ).to(device)
+
+        pipe.scheduler = diffusers.EulerDiscreteScheduler.from_config(pipe.scheduler.config)
+
+    pipe.load_ip_adapter_instantid(face_adapter)
+    # load and disable LCM
+    pipe.load_lora_weights("latent-consistency/lcm-lora-sdxl")
+    pipe.disable_lora()
+    
+    def remove_tips():
+        print("GG")
+        return gr.update(visible=False)
+
+    
+    # prompts = [
+    #     ["superman","Vibrant Color"], ["japanese anime character with white/neon hair","Watercolor"], 
+    #     # ["Suited professional","(No style)"], 
+    #     ["Scooba diver","Line art"], ["eskimo","Snow"]
+    # ]
+
+    def convert_from_cv2_to_image(img: np.ndarray) -> Image:
+        return Image.fromarray(cv2.cvtColor(img, cv2.COLOR_BGR2RGB))
+
+    def convert_from_image_to_cv2(img: Image) -> np.ndarray:
+        return cv2.cvtColor(np.array(img), cv2.COLOR_RGB2BGR)
+
+    def run_for_prompts1(face_file,style,progress=gr.Progress(track_tqdm=True)):
+        # if email != "":
+        p,n = styles.get(style, styles.get(STYLE_NAMES[1]))
+        return generate_image(face_file, p[0], n)
+        # else:
+            # raise gr.Error("Email ID is compulsory")
+    def run_for_prompts2(face_file,style,progress=gr.Progress(track_tqdm=True)):
+        # if email != "":
+        p,n = styles.get(style, styles.get(STYLE_NAMES[1]))
+        return generate_image(face_file, p[1], n)
+    def run_for_prompts3(face_file,style,progress=gr.Progress(track_tqdm=True)):
+        # if email != "":
+        p,n = styles.get(style, styles.get(STYLE_NAMES[1]))
+        return generate_image(face_file, p[2], n)
+    def run_for_prompts4(face_file,style,progress=gr.Progress(track_tqdm=True)):
+        # if email != "":
+        p,n = styles.get(style, styles.get(STYLE_NAMES[1]))
+        return generate_image(face_file, p[3], n)
+    
+#     def validate_and_process(face_file, style, email):
+    
+#     # Your processing logic here
+#     gallery1, gallery2, gallery3, gallery4 = run_for_prompts1(face_file, style), run_for_prompts2(face_file, style), run_for_prompts3(face_file, style), run_for_prompts4(face_file, style)
+#     return gallery1, gallery2, gallery3, gallery4
+
+    def draw_kps(image_pil, kps, color_list=[(255,0,0), (0,255,0), (0,0,255), (255,255,0), (255,0,255)]):
+        stickwidth = 4
+        limbSeq = np.array([[0, 2], [1, 2], [3, 2], [4, 2]])
+        kps = np.array(kps)
+
+        w, h = image_pil.size
+        out_img = np.zeros([h, w, 3])
+
+        for i in range(len(limbSeq)):
+            index = limbSeq[i]
+            color = color_list[index[0]]
+
+            x = kps[index][:, 0]
+            y = kps[index][:, 1]
+            length = ((x[0] - x[1]) ** 2 + (y[0] - y[1]) ** 2) ** 0.5
+            angle = math.degrees(math.atan2(y[0] - y[1], x[0] - x[1]))
+            polygon = cv2.ellipse2Poly((int(np.mean(x)), int(np.mean(y))), (int(length / 2), stickwidth), int(angle), 0, 360, 1)
+            out_img = cv2.fillConvexPoly(out_img.copy(), polygon, color)
+        out_img = (out_img * 0.6).astype(np.uint8)
+
+        for idx_kp, kp in enumerate(kps):
+            color = color_list[idx_kp]
+            x, y = kp
+            out_img = cv2.circle(out_img.copy(), (int(x), int(y)), 10, color, -1)
+
+        out_img_pil = Image.fromarray(out_img.astype(np.uint8))
+        return out_img_pil
+
+    def resize_img(input_image, max_side=1280, min_side=1280, size=None, 
+                pad_to_max_side=True, mode=PIL.Image.BILINEAR, base_pixel_number=64):
+
+            w, h = input_image.size
+            print(f"Original Size --> {input_image.size}")
+            if size is not None:
+                w_resize_new, h_resize_new = size
+            else:
+                ratio = min_side / min(h, w)
+                w, h = round(ratio*w), round(ratio*h)
+                ratio = max_side / max(h, w)
+                input_image = input_image.resize([round(ratio*w), round(ratio*h)], mode)
+                w_resize_new = (round(ratio * w) // base_pixel_number) * base_pixel_number
+                h_resize_new = (round(ratio * h) // base_pixel_number) * base_pixel_number
+            input_image = input_image.resize([w_resize_new, h_resize_new], mode)
+
+            if pad_to_max_side:
+                res = np.ones([max_side, max_side, 3], dtype=np.uint8) * 255
+                offset_x = (max_side - w_resize_new) // 2
+                offset_y = (max_side - h_resize_new) // 2
+                res[offset_y:offset_y+h_resize_new, offset_x:offset_x+w_resize_new] = np.array(input_image)
+                input_image = Image.fromarray(res)
+            
+            print(f"Final modified image size --> {input_image.size}")
+            return input_image
+
+    # def apply_style(style_name: str, positive: str, negative: str = "") -> Tuple[str, str]:
+    #     p, n = styles.get(style_name, styles[DEFAULT_STYLE_NAME])
+    #     return p.replace("{prompt}", positive), n + ' ' + negative
+
+    def generate_image(face_image,prompt,negative_prompt):
+        pose_image_path = None
+        # prompt = "superman"
+        enable_LCM = False
+        identitynet_strength_ratio = 0.90
+        adapter_strength_ratio = 0.60
+        num_steps = 25
+        guidance_scale = 5
+        seed = random.randint(0, MAX_SEED)
+        print(f"Seed --> {seed}")
+        
+        # negative_prompt = ""
+        # negative_prompt += neg
+        enhance_face_region = True
+        if enable_LCM:
+            pipe.enable_lora()
+            pipe.scheduler = LCMScheduler.from_config(pipe.scheduler.config)
+        else:
+            pipe.disable_lora()
+            pipe.scheduler = diffusers.EulerDiscreteScheduler.from_config(pipe.scheduler.config)
+    
+        if face_image is None:
+            raise gr.Error(f"Cannot find any input face image! Please upload the face image")
+        
+        # if prompt is None:
+        #     prompt = "a person"
+        
+        # apply the style template
+        # prompt, negative_prompt = apply_style(style_name, prompt, negative_prompt)
+        
+        # face_image = load_image(face_image_path)
+        face_image = resize_img(face_image)
+        face_image_cv2 = convert_from_image_to_cv2(face_image)
+        height, width, _ = face_image_cv2.shape
+        
+        # Extract face features
+        face_info = app.get(face_image_cv2)
+        
+        if len(face_info) == 0:
+            raise gr.Error(f"Cannot find any face in the image! Please upload another person image")
+        
+        face_info = sorted(face_info, key=lambda x:(x['bbox'][2]-x['bbox'][0])*(x['bbox'][3]-x['bbox'][1]))[-1]  # only use the maximum face
+        face_emb = face_info['embedding']
+        face_kps = draw_kps(convert_from_cv2_to_image(face_image_cv2), face_info['kps'])
+        
+        if pose_image_path is not None:
+            pose_image = load_image(pose_image_path)
+            pose_image = resize_img(pose_image)
+            pose_image_cv2 = convert_from_image_to_cv2(pose_image)
+            
+            face_info = app.get(pose_image_cv2)
+            
+            if len(face_info) == 0:
+                raise gr.Error(f"Cannot find any face in the reference image! Please upload another person image")
+            
+            face_info = face_info[-1]
+            face_kps = draw_kps(pose_image, face_info['kps'])
+            
+            width, height = face_kps.size
+
+        if enhance_face_region:
+            control_mask = np.zeros([height, width, 3])
+            x1, y1, x2, y2 = face_info["bbox"]
+            x1, y1, x2, y2 = int(x1), int(y1), int(x2), int(y2)
+            control_mask[y1:y2, x1:x2] = 255
+            control_mask = Image.fromarray(control_mask.astype(np.uint8))
+        else:
+            control_mask = None
+                        
+        generator = torch.Generator(device=device).manual_seed(seed)
+        
+        print("Start inference...")
+        print(f"[Debug] Prompt: {prompt}, \n[Debug] Neg Prompt: {negative_prompt}")
+        
+        pipe.set_ip_adapter_scale(adapter_strength_ratio)
+        images = pipe(
+            prompt=prompt,
+            negative_prompt=negative_prompt,
+            image_embeds=face_emb,
+            image=face_kps,
+            control_mask=control_mask,
+            controlnet_conditioning_scale=float(identitynet_strength_ratio),
+            num_inference_steps=num_steps,
+            guidance_scale=guidance_scale,
+            height=height,
+            width=width,
+            generator=generator,
+            # num_images_per_prompt = 4
+        ).images
+
+        return images[0]
+
+    ### Description
+    title = r"""
+    <h1 align="center">Choose your AVATAR</h1>
+    """
+
+    description = r"""
+    <h2> Powered by IDfy </h2>"""
+
+    article = r""""""
+
+    tips = r""""""
+    
+    css = '''
+    .gradio-container {width: 95% !important; background-color: #E6F3FF;} 
+    .image-gallery {height: 100vh !important; overflow: auto;}
+    .gradio-row .gradio-element { margin: 0 !important; }
+    '''
+    with gr.Blocks(css=css) as demo:
+
+        # description
+        gr.Markdown(title)
+        with gr.Row():
+            gr.Image("./gradio_demo/logo.png",scale=0,min_width=50,show_label=False,show_download_button=False)
+            gr.Markdown(description)
+        with gr.Row():
+            with gr.Column():
+                style = gr.Dropdown(label="Choose your STYLE", choices=STYLE_NAMES)
+                face_file = gr.Image(label="Upload a photo of your face", type="pil",sources="webcam")
+                submit = gr.Button("Submit", variant="primary")
+            with gr.Column():
+                with gr.Row():
+                    gallery1 = gr.Image(label="Generated Images")
+                    gallery2 = gr.Image(label="Generated Images")
+                with gr.Row():
+                    gallery3 = gr.Image(label="Generated Images")
+                    gallery4 = gr.Image(label="Generated Images")
+                email = gr.Textbox(label="Email",
+                        info="Enter your email address",
+                        value="")
+                # submit1 = gr.Button("Store")
+            
+                usage_tips = gr.Markdown(label="Usage tips of InstantID", value=tips ,visible=False)
+                
+            face_file.upload(
+                fn=remove_tips,
+                outputs=usage_tips,
+                queue=True,
+                api_name=False,
+                show_progress = "full"
+            ).then(
+                fn=run_for_prompts1,
+                inputs=[face_file,style],
+                outputs=[gallery1]
+            ).then(
+                fn=run_for_prompts2,
+                inputs=[face_file,style],
+                outputs=[gallery2]
+            ).then(
+                fn=run_for_prompts3,
+                inputs=[face_file,style],
+                outputs=[gallery3]
+            ).then(
+                fn=run_for_prompts4,
+                inputs=[face_file,style],
+                outputs=[gallery4]
+            )
+            submit.click(
+                fn=remove_tips,
+                outputs=usage_tips,
+                queue=True,
+                api_name=False,
+                show_progress = "full"
+            ).then(
+                fn=run_for_prompts1,
+                inputs=[face_file,style],
+                outputs=[gallery1]
+            ).then(
+                fn=run_for_prompts2,
+                inputs=[face_file,style],
+                outputs=[gallery2]
+            ).then(
+                fn=run_for_prompts3,
+                inputs=[face_file,style],
+                outputs=[gallery3]
+            ).then(
+                fn=run_for_prompts4,
+                inputs=[face_file,style],
+                outputs=[gallery4]
+            )
+            
+            # submit1.click(
+            #     fn=store_images,
+            #     inputs=[email,gallery1,gallery2,gallery3,gallery4],
+            #     outputs=None)
+            
+        
+        
+        gr.Markdown(article)
+
+    demo.launch(share=True)
+
+#     with gr.Blocks(css=css, js=js) as demo:
+
+#         # description
+#         gr.Markdown(title)
+#         with gr.Row():
+#             gr.Image("./gradio_demo/logo.png",scale=0,min_width=50,show_label=False,show_download_button=False)
+#             gr.Markdown(description)
+#         with gr.Row():
+#             with gr.Column():
+#                 style = gr.Dropdown(label="Choose your STYLE", choices=STYLE_NAMES)
+#                 face_file = gr.Image(label="Upload a photo of your face", type="pil",sources="webcam")
+#                 submit = gr.Button("Submit", variant="primary")
+#             with gr.Column():
+#                 with gr.Row():
+#                     gallery1 = gr.Image(label="Generated Images")
+#                     gallery2 = gr.Image(label="Generated Images")
+#                 with gr.Row():
+#                     gallery3 = gr.Image(label="Generated Images")
+#                     gallery4 = gr.Image(label="Generated Images")
+#                 email = gr.Textbox(label="Email",
+#                         info="Enter your email address",
+#                         value="")
+            
+#             usage_tips = gr.Markdown(label="Usage tips of InstantID", value=tips ,visible=False)
+#             # identitynet_strength_ratio = gr.Slider(
+#             #     label="IdentityNet strength (for fidelity)",
+#             #     minimum=0,
+#             #     maximum=1.5,
+#             #     step=0.05,
+#             #     value=0.95,
+#             # )
+#             # adapter_strength_ratio = gr.Slider(
+#             #     label="Image adapter strength (for detail)",
+#             #     minimum=0,
+#             #     maximum=1.5,
+#             #     step=0.05,
+#             #     value=0.60,
+#             # )
+#             # negative_prompt = gr.Textbox(
+#             #     label="Negative Prompt", 
+#             #     placeholder="low quality",
+#             #     value="(lowres, low quality, worst quality:1.2), (text:1.2), watermark, (frame:1.2), deformed, ugly, deformed eyes, blur, out of focus, blurry, deformed cat, deformed, photo, anthropomorphic cat, monochrome, pet collar, gun, weapon, blue, 3d, drones, drone, buildings in background, green",
+#             # )
+#             # num_steps = gr.Slider( 
+#             #     label="Number of sample steps",
+#             #     minimum=15,
+#             #     maximum=100,
+#             #     step=1,
+#             #     value=5 if enable_lcm_arg else 15,
+#             # )
+#             # guidance_scale = gr.Slider(
+#             #     label="Guidance scale",
+#             #     minimum=0.1,
+#             #     maximum=10.0,
+#             #     step=0.1,
+#             #     value=0 if enable_lcm_arg else 8.5,
+#             # )
+#             # if email is None:
+#             #     print("STOPPPP")
+#             #     raise gr.Error("Email ID is compulsory")
+#             face_file.upload(
+#                 fn=remove_tips,
+#                 outputs=usage_tips,
+#                 queue=True,
+#                 api_name=False,
+#                 show_progress = "full"
+#             ).then(
+#                 fn=run_for_prompts1,
+#                 inputs=[face_file,style],
+#                 outputs=[gallery1]
+#             ).then(
+#                 fn=run_for_prompts2,
+#                 inputs=[face_file,style],
+#                 outputs=[gallery2]
+#             ).then(
+#                 fn=run_for_prompts3,
+#                 inputs=[face_file,style],
+#                 outputs=[gallery3]
+#             ).then(
+#                 fn=run_for_prompts4,
+#                 inputs=[face_file,style],
+#                 outputs=[gallery4]
+#             )
+#             submit.click(
+#                 fn=remove_tips,
+#                 outputs=usage_tips,
+#                 queue=True,
+#                 api_name=False,
+#                 show_progress = "full"
+#             ).then(
+#                 fn=run_for_prompts1,
+#                 inputs=[face_file,style],
+#                 outputs=[gallery1]
+#             ).then(
+#                 fn=run_for_prompts2,
+#                 inputs=[face_file,style],
+#                 outputs=[gallery2]
+#             ).then(
+#                 fn=run_for_prompts3,
+#                 inputs=[face_file,style],
+#                 outputs=[gallery3]
+#             ).then(
+#                 fn=run_for_prompts4,
+#                 inputs=[face_file,style],
+#                 outputs=[gallery4]
+#             )
+        
+        
+#         gr.Markdown(article)
+
+#     demo.launch(share=True)
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--pretrained_model_name_or_path", type=str, default="wangqixun/YamerMIX_v8")
+    args = parser.parse_args()
+
+    main(args.pretrained_model_name_or_path, False)
+
+

gradio_demo/app1.py

@@ -0,0 +1,434 @@
+import sys
+sys.path.append('./')
+
+from typing import Tuple
+
+import os
+import cv2
+import math
+import torch
+import random
+import numpy as np
+import argparse
+
+import PIL
+from PIL import Image
+
+import diffusers
+from diffusers.utils import load_image
+from diffusers.models import ControlNetModel
+from diffusers import LCMScheduler
+
+from huggingface_hub import hf_hub_download
+
+import insightface
+from insightface.app import FaceAnalysis
+
+from style_template import styles
+from pipeline_stable_diffusion_xl_instantid_full import StableDiffusionXLInstantIDPipeline
+from model_util import load_models_xl, get_torch_device, torch_gc
+
+
+# global variable
+MAX_SEED = np.iinfo(np.int32).max
+device = get_torch_device()
+dtype = torch.float16 if str(device).__contains__("cuda") else torch.float32
+STYLE_NAMES = list(styles.keys())
+DEFAULT_STYLE_NAME = "Watercolor"
+
+# Load face encoder
+app = FaceAnalysis(name='antelopev2', root='./', providers=['CUDAExecutionProvider', 'CPUExecutionProvider'])
+app.prepare(ctx_id=0, det_size=(320, 320))
+
+# Path to InstantID models
+face_adapter = f'./checkpoints/ip-adapter.bin'
+controlnet_path = f'./checkpoints/ControlNetModel'
+
+# Load pipeline
+controlnet = ControlNetModel.from_pretrained(controlnet_path, torch_dtype=dtype)
+
+logo = Image.open("./gradio_demo/logo.png")
+
+from cv2 import imencode
+import base64
+
+# def encode_pil_to_base64_new(pil_image):
+#     print("AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA")
+#     image_arr = np.asarray(pil_image)[:,:,::-1]
+#     _, byte_data = imencode('.png', image_arr)        
+#     base64_data = base64.b64encode(byte_data)
+#     base64_string_opencv = base64_data.decode("utf-8")
+#     return "data:image/png;base64," + base64_string_opencv
+
+import gradio as gr
+
+# gr.processing_utils.encode_pil_to_base64 = encode_pil_to_base64_new
+
+def main(pretrained_model_name_or_path="wangqixun/YamerMIX_v8", enable_lcm_arg=False):
+
+    if pretrained_model_name_or_path.endswith(
+            ".ckpt"
+        ) or pretrained_model_name_or_path.endswith(".safetensors"):
+            scheduler_kwargs = hf_hub_download(
+                repo_id="wangqixun/YamerMIX_v8",
+                subfolder="scheduler",
+                filename="scheduler_config.json",
+            )
+
+            (tokenizers, text_encoders, unet, _, vae) = load_models_xl(
+                pretrained_model_name_or_path=pretrained_model_name_or_path,
+                scheduler_name=None,
+                weight_dtype=dtype,
+            )
+
+            scheduler = diffusers.EulerDiscreteScheduler.from_config(scheduler_kwargs)
+            pipe = StableDiffusionXLInstantIDPipeline(
+                vae=vae,
+                text_encoder=text_encoders[0],
+                text_encoder_2=text_encoders[1],
+                tokenizer=tokenizers[0],
+                tokenizer_2=tokenizers[1],
+                unet=unet,
+                scheduler=scheduler,
+                controlnet=controlnet,
+            ).to(device)
+
+    else:
+        pipe = StableDiffusionXLInstantIDPipeline.from_pretrained(
+            pretrained_model_name_or_path,
+            controlnet=controlnet,
+            torch_dtype=dtype,
+            safety_checker=None,
+            feature_extractor=None,
+        ).to(device)
+
+        pipe.scheduler = diffusers.EulerDiscreteScheduler.from_config(pipe.scheduler.config)
+
+    pipe.load_ip_adapter_instantid(face_adapter)
+    # load and disable LCM
+    pipe.load_lora_weights("latent-consistency/lcm-lora-sdxl")
+    pipe.disable_lora()
+    
+    def remove_tips():
+        return gr.update(visible=False)
+
+    
+    # prompts = [
+    #     ["superman","Vibrant Color"], ["japanese anime character with white/neon hair","Watercolor"], 
+    #     # ["Suited professional","(No style)"], 
+    #     ["Scooba diver","Line art"], ["eskimo","Snow"]
+    # ]
+
+    def convert_from_cv2_to_image(img: np.ndarray) -> Image:
+        return Image.fromarray(cv2.cvtColor(img, cv2.COLOR_BGR2RGB))
+
+    def convert_from_image_to_cv2(img: Image) -> np.ndarray:
+        return cv2.cvtColor(np.array(img), cv2.COLOR_RGB2BGR)
+
+    def run_for_prompts1(face_file,style,progress=gr.Progress(track_tqdm=True)):
+        # if email != "":
+        p,n = styles.get(style, styles.get(STYLE_NAMES[1]))
+        return generate_image(face_file, p[0], n)
+        # else:
+            # raise gr.Error("Email ID is compulsory")
+    def run_for_prompts2(face_file,style,progress=gr.Progress(track_tqdm=True)):
+        # if email != "":
+        p,n = styles.get(style, styles.get(STYLE_NAMES[1]))
+        return generate_image(face_file, p[1], n)
+    def run_for_prompts3(face_file,style,progress=gr.Progress(track_tqdm=True)):
+        # if email != "":
+        p,n = styles.get(style, styles.get(STYLE_NAMES[1]))
+        return generate_image(face_file, p[2], n)
+    def run_for_prompts4(face_file,style,progress=gr.Progress(track_tqdm=True)):
+        # if email != "":
+        p,n = styles.get(style, styles.get(STYLE_NAMES[1]))
+        return generate_image(face_file, p[3], n)
+    
+#     def validate_and_process(face_file, style, email):
+    
+#     # Your processing logic here
+#     gallery1, gallery2, gallery3, gallery4 = run_for_prompts1(face_file, style), run_for_prompts2(face_file, style), run_for_prompts3(face_file, style), run_for_prompts4(face_file, style)
+#     return gallery1, gallery2, gallery3, gallery4
+
+    def draw_kps(image_pil, kps, color_list=[(255,0,0), (0,255,0), (0,0,255), (255,255,0), (255,0,255)]):
+        stickwidth = 4
+        limbSeq = np.array([[0, 2], [1, 2], [3, 2], [4, 2]])
+        kps = np.array(kps)
+
+        w, h = image_pil.size
+        out_img = np.zeros([h, w, 3])
+
+        for i in range(len(limbSeq)):
+            index = limbSeq[i]
+            color = color_list[index[0]]
+
+            x = kps[index][:, 0]
+            y = kps[index][:, 1]
+            length = ((x[0] - x[1]) ** 2 + (y[0] - y[1]) ** 2) ** 0.5
+            angle = math.degrees(math.atan2(y[0] - y[1], x[0] - x[1]))
+            polygon = cv2.ellipse2Poly((int(np.mean(x)), int(np.mean(y))), (int(length / 2), stickwidth), int(angle), 0, 360, 1)
+            out_img = cv2.fillConvexPoly(out_img.copy(), polygon, color)
+        out_img = (out_img * 0.6).astype(np.uint8)
+
+        for idx_kp, kp in enumerate(kps):
+            color = color_list[idx_kp]
+            x, y = kp
+            out_img = cv2.circle(out_img.copy(), (int(x), int(y)), 10, color, -1)
+
+        out_img_pil = Image.fromarray(out_img.astype(np.uint8))
+        return out_img_pil
+
+    def resize_img(input_image, max_side=640, min_side=640, size=None, 
+                pad_to_max_side=True, mode=PIL.Image.BILINEAR, base_pixel_number=64):
+
+            w, h = input_image.size
+            print(w)
+            print(h)
+            if size is not None:
+                w_resize_new, h_resize_new = size
+            else:
+                ratio = min_side / min(h, w)
+                w, h = round(ratio*w), round(ratio*h)
+                ratio = max_side / max(h, w)
+                input_image = input_image.resize([round(ratio*w), round(ratio*h)], mode)
+                w_resize_new = (round(ratio * w) // base_pixel_number) * base_pixel_number
+                h_resize_new = (round(ratio * h) // base_pixel_number) * base_pixel_number
+            input_image = input_image.resize([w_resize_new, h_resize_new], mode)
+
+            if pad_to_max_side:
+                res = np.ones([max_side, max_side, 3], dtype=np.uint8) * 255
+                offset_x = (max_side - w_resize_new) // 2
+                offset_y = (max_side - h_resize_new) // 2
+                res[offset_y:offset_y+h_resize_new, offset_x:offset_x+w_resize_new] = np.array(input_image)
+                input_image = Image.fromarray(res)
+            return input_image
+
+    # def apply_style(style_name: str, positive: str, negative: str = "") -> Tuple[str, str]:
+    #     p, n = styles.get(style_name, styles[DEFAULT_STYLE_NAME])
+    #     return p.replace("{prompt}", positive), n + ' ' + negative
+
+    def generate_image(face_image,prompt,negative_prompt):
+        pose_image_path = None
+        # prompt = "superman"
+        enable_LCM = False
+        identitynet_strength_ratio = 0.95
+        adapter_strength_ratio = 0.60
+        num_steps = 15
+        guidance_scale = 8.5
+        seed = random.randint(0, MAX_SEED)
+        # negative_prompt = ""
+        # negative_prompt += neg
+        enhance_face_region = True
+        if enable_LCM:
+            pipe.enable_lora()
+            pipe.scheduler = LCMScheduler.from_config(pipe.scheduler.config)
+        else:
+            pipe.disable_lora()
+            pipe.scheduler = diffusers.EulerDiscreteScheduler.from_config(pipe.scheduler.config)
+    
+        if face_image is None:
+            raise gr.Error(f"Cannot find any input face image! Please upload the face image")
+        
+        # if prompt is None:
+        #     prompt = "a person"
+        
+        # apply the style template
+        # prompt, negative_prompt = apply_style(style_name, prompt, negative_prompt)
+        
+        # face_image = load_image(face_image_path)
+        face_image = resize_img(face_image)
+        face_image_cv2 = convert_from_image_to_cv2(face_image)
+        height, width, _ = face_image_cv2.shape
+        
+        # Extract face features
+        face_info = app.get(face_image_cv2)
+        
+        if len(face_info) == 0:
+            raise gr.Error(f"Cannot find any face in the image! Please upload another person image")
+        
+        face_info = sorted(face_info, key=lambda x:(x['bbox'][2]-x['bbox'][0])*(x['bbox'][3]-x['bbox'][1]))[-1]  # only use the maximum face
+        face_emb = face_info['embedding']
+        face_kps = draw_kps(convert_from_cv2_to_image(face_image_cv2), face_info['kps'])
+        
+        if pose_image_path is not None:
+            pose_image = load_image(pose_image_path)
+            pose_image = resize_img(pose_image)
+            pose_image_cv2 = convert_from_image_to_cv2(pose_image)
+            
+            face_info = app.get(pose_image_cv2)
+            
+            if len(face_info) == 0:
+                raise gr.Error(f"Cannot find any face in the reference image! Please upload another person image")
+            
+            face_info = face_info[-1]
+            face_kps = draw_kps(pose_image, face_info['kps'])
+            
+            width, height = face_kps.size
+
+        if enhance_face_region:
+            control_mask = np.zeros([height, width, 3])
+            x1, y1, x2, y2 = face_info["bbox"]
+            x1, y1, x2, y2 = int(x1), int(y1), int(x2), int(y2)
+            control_mask[y1:y2, x1:x2] = 255
+            control_mask = Image.fromarray(control_mask.astype(np.uint8))
+        else:
+            control_mask = None
+                        
+        generator = torch.Generator(device=device).manual_seed(seed)
+        
+        print("Start inference...")
+        print(f"[Debug] Prompt: {prompt}, \n[Debug] Neg Prompt: {negative_prompt}")
+        
+        pipe.set_ip_adapter_scale(adapter_strength_ratio)
+        images = pipe(
+            prompt=prompt,
+            negative_prompt=negative_prompt,
+            image_embeds=face_emb,
+            image=face_kps,
+            control_mask=control_mask,
+            controlnet_conditioning_scale=float(identitynet_strength_ratio),
+            num_inference_steps=num_steps,
+            guidance_scale=guidance_scale,
+            height=height,
+            width=width,
+            generator=generator,
+            # num_images_per_prompt = 4
+        ).images
+
+        return images[0]
+
+    ### Description
+    title = r"""
+    <h1 align="center">Choose your AVATAR</h1>
+    """
+
+    description = r"""
+    <h2> Powered by IDfy </h2>"""
+
+    article = r""""""
+
+    tips = r""""""
+
+    css = '''
+    .gradio-container {width: 95% !important; background-color: #E6F3FF;} 
+    .image-gallery {height: 100vh !important; overflow: auto;}
+    .gradio-row .gradio-element { margin: 0 !important; }
+    '''
+    with gr.Blocks(css=css) as demo:
+
+        # description
+        gr.Markdown(title)
+        with gr.Row():
+            gr.Image("./gradio_demo/logo.png",scale=0,min_width=50,show_label=False,show_download_button=False)
+            gr.Markdown(description)
+        with gr.Row():
+            with gr.Column():
+                style = gr.Dropdown(label="Choose your STYLE", choices=STYLE_NAMES)
+                face_file = gr.Image(label="Upload a photo of your face", type="pil",sources="webcam")
+                submit = gr.Button("Submit", variant="primary")
+            with gr.Column():
+                with gr.Row():
+                    gallery1 = gr.Image(label="Generated Images")
+                    gallery2 = gr.Image(label="Generated Images")
+                with gr.Row():
+                    gallery3 = gr.Image(label="Generated Images")
+                    gallery4 = gr.Image(label="Generated Images")
+                email = gr.Textbox(label="Email",
+                        info="Enter your email address",
+                        value="")
+            
+            usage_tips = gr.Markdown(label="Usage tips of InstantID", value=tips ,visible=False)
+            # identitynet_strength_ratio = gr.Slider(
+            #     label="IdentityNet strength (for fidelity)",
+            #     minimum=0,
+            #     maximum=1.5,
+            #     step=0.05,
+            #     value=0.95,
+            # )
+            # adapter_strength_ratio = gr.Slider(
+            #     label="Image adapter strength (for detail)",
+            #     minimum=0,
+            #     maximum=1.5,
+            #     step=0.05,
+            #     value=0.60,
+            # )
+            # negative_prompt = gr.Textbox(
+            #     label="Negative Prompt", 
+            #     placeholder="low quality",
+            #     value="(lowres, low quality, worst quality:1.2), (text:1.2), watermark, (frame:1.2), deformed, ugly, deformed eyes, blur, out of focus, blurry, deformed cat, deformed, photo, anthropomorphic cat, monochrome, pet collar, gun, weapon, blue, 3d, drones, drone, buildings in background, green",
+            # )
+            # num_steps = gr.Slider( 
+            #     label="Number of sample steps",
+            #     minimum=15,
+            #     maximum=100,
+            #     step=1,
+            #     value=5 if enable_lcm_arg else 15,
+            # )
+            # guidance_scale = gr.Slider(
+            #     label="Guidance scale",
+            #     minimum=0.1,
+            #     maximum=10.0,
+            #     step=0.1,
+            #     value=0 if enable_lcm_arg else 8.5,
+            # )
+            # if email is None:
+            #     print("STOPPPP")
+            #     raise gr.Error("Email ID is compulsory")
+            face_file.upload(
+                fn=remove_tips,
+                outputs=usage_tips,
+                queue=True,
+                api_name=False,
+                show_progress = "full"
+            ).then(
+                fn=run_for_prompts1,
+                inputs=[face_file,style],
+                outputs=[gallery1]
+            ).then(
+                fn=run_for_prompts2,
+                inputs=[face_file,style],
+                outputs=[gallery2]
+            ).then(
+                fn=run_for_prompts3,
+                inputs=[face_file,style],
+                outputs=[gallery3]
+            ).then(
+                fn=run_for_prompts4,
+                inputs=[face_file,style],
+                outputs=[gallery4]
+            )
+            submit.click(
+                fn=remove_tips,
+                outputs=usage_tips,
+                queue=True,
+                api_name=False,
+                show_progress = "full"
+            ).then(
+                fn=run_for_prompts1,
+                inputs=[face_file,style],
+                outputs=[gallery1]
+            ).then(
+                fn=run_for_prompts2,
+                inputs=[face_file,style],
+                outputs=[gallery2]
+            ).then(
+                fn=run_for_prompts3,
+                inputs=[face_file,style],
+                outputs=[gallery3]
+            ).then(
+                fn=run_for_prompts4,
+                inputs=[face_file,style],
+                outputs=[gallery4]
+            )
+        
+        
+        gr.Markdown(article)
+
+    demo.launch(share=True)
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--pretrained_model_name_or_path", type=str, default="wangqixun/YamerMIX_v8")
+    args = parser.parse_args()
+
+    main(args.pretrained_model_name_or_path, False)

gradio_demo/controlnet_util.py

@@ -0,0 +1,39 @@
+import torch
+import numpy as np
+from PIL import Image
+from controlnet_aux import OpenposeDetector
+from model_util import get_torch_device
+import cv2
+
+
+from transformers import DPTImageProcessor, DPTForDepthEstimation
+
+device = get_torch_device()
+depth_estimator = DPTForDepthEstimation.from_pretrained("Intel/dpt-hybrid-midas").to(device)
+feature_extractor = DPTImageProcessor.from_pretrained("Intel/dpt-hybrid-midas")
+openpose = OpenposeDetector.from_pretrained("lllyasviel/ControlNet")
+
+def get_depth_map(image):
+    image = feature_extractor(images=image, return_tensors="pt").pixel_values.to("cuda")
+    with torch.no_grad(), torch.autocast("cuda"):
+        depth_map = depth_estimator(image).predicted_depth
+
+    depth_map = torch.nn.functional.interpolate(
+        depth_map.unsqueeze(1),
+        size=(1024, 1024),
+        mode="bicubic",
+        align_corners=False,
+    )
+    depth_min = torch.amin(depth_map, dim=[1, 2, 3], keepdim=True)
+    depth_max = torch.amax(depth_map, dim=[1, 2, 3], keepdim=True)
+    depth_map = (depth_map - depth_min) / (depth_max - depth_min)
+    image = torch.cat([depth_map] * 3, dim=1)
+
+    image = image.permute(0, 2, 3, 1).cpu().numpy()[0]
+    image = Image.fromarray((image * 255.0).clip(0, 255).astype(np.uint8))
+    return image
+
+def get_canny_image(image, t1=100, t2=200):
+    image = cv2.cvtColor(np.array(image), cv2.COLOR_RGB2BGR)
+    edges = cv2.Canny(image, t1, t2)
+    return Image.fromarray(edges, "L")

gradio_demo/demo.py

@@ -0,0 +1,369 @@
+import sys
+sys.path.append('./')
+
+from typing import Tuple
+
+import os
+import cv2
+import math
+import torch
+import random
+import numpy as np
+import argparse
+
+import PIL
+from PIL import Image
+
+import diffusers
+from diffusers.utils import load_image
+from diffusers.models import ControlNetModel
+from diffusers import LCMScheduler
+
+from huggingface_hub import hf_hub_download
+
+import insightface
+from insightface.app import FaceAnalysis
+
+from style_template import styles
+from pipeline_stable_diffusion_xl_instantid_full import StableDiffusionXLInstantIDPipeline
+from model_util import load_models_xl, get_torch_device, torch_gc
+
+from cv2 import imencode
+import base64
+
+# def encode_pil_to_base64_new(pil_image):
+#     print("AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA")
+#     image_arr = np.asarray(pil_image)[:,:,::-1]
+#     _, byte_data = imencode('.png', image_arr)        
+#     base64_data = base64.b64encode(byte_data)
+#     base64_string_opencv = base64_data.decode("utf-8")
+#     return "data:image/png;base64," + base64_string_opencv
+
+import gradio as gr
+
+
+# global variable
+MAX_SEED = np.iinfo(np.int32).max
+device = get_torch_device()
+dtype = torch.float16 if str(device).__contains__("cuda") else torch.float32
+STYLE_NAMES = list(styles.keys())
+DEFAULT_STYLE_NAME = "Watercolor"
+
+# Load face encoder
+app = FaceAnalysis(name='antelopev2', root='./', providers=['CUDAExecutionProvider', 'CPUExecutionProvider'])
+app.prepare(ctx_id=0, det_size=(320, 320))
+
+# Path to InstantID models
+face_adapter = f'./checkpoints/ip-adapter.bin'
+controlnet_path = f'./checkpoints/ControlNetModel'
+
+# Load pipeline
+controlnet = ControlNetModel.from_pretrained(controlnet_path, torch_dtype=dtype)
+
+logo = Image.open("./gradio_demo/logo.png")
+
+pretrained_model_name_or_path="wangqixun/YamerMIX_v8"
+
+
+if pretrained_model_name_or_path.endswith(
+            ".ckpt"
+        ) or pretrained_model_name_or_path.endswith(".safetensors"):
+            scheduler_kwargs = hf_hub_download(
+                repo_id="wangqixun/YamerMIX_v8",
+                subfolder="scheduler",
+                filename="scheduler_config.json",
+            )
+
+            (tokenizers, text_encoders, unet, _, vae) = load_models_xl(
+                pretrained_model_name_or_path=pretrained_model_name_or_path,
+                scheduler_name=None,
+                weight_dtype=dtype,
+            )
+
+            scheduler = diffusers.EulerDiscreteScheduler.from_config(scheduler_kwargs)
+            pipe = StableDiffusionXLInstantIDPipeline(
+                vae=vae,
+                text_encoder=text_encoders[0],
+                text_encoder_2=text_encoders[1],
+                tokenizer=tokenizers[0],
+                tokenizer_2=tokenizers[1],
+                unet=unet,
+                scheduler=scheduler,
+                controlnet=controlnet,
+            ).to(device)
+
+else:
+    pipe = StableDiffusionXLInstantIDPipeline.from_pretrained(
+        pretrained_model_name_or_path,
+        controlnet=controlnet,
+        torch_dtype=dtype,
+        safety_checker=None,
+        feature_extractor=None,
+    ).to(device)
+
+    pipe.scheduler = diffusers.EulerDiscreteScheduler.from_config(pipe.scheduler.config)
+
+pipe.load_ip_adapter_instantid(face_adapter)
+# load and disable LCM
+pipe.load_lora_weights("latent-consistency/lcm-lora-sdxl")
+pipe.disable_lora()
+
+# gr.processing_utils.encode_pil_to_base64 = encode_pil_to_base64_new
+def remove_tips():
+    print("GG")
+    return gr.update(visible=False)
+
+def convert_from_cv2_to_image(img: np.ndarray) -> Image:
+    return Image.fromarray(cv2.cvtColor(img, cv2.COLOR_BGR2RGB))
+
+def convert_from_image_to_cv2(img: Image) -> np.ndarray:
+    return cv2.cvtColor(np.array(img), cv2.COLOR_RGB2BGR)
+
+def run_for_prompts1(face_file,style,progress=gr.Progress(track_tqdm=True)):
+    # if email != "":
+    p,n = styles.get(style, styles.get(STYLE_NAMES[1]))
+    return generate_image(face_file, p[0], n)
+    # else:
+        # raise gr.Error("Email ID is compulsory")
+def run_for_prompts2(face_file,style,progress=gr.Progress(track_tqdm=True)):
+    # if email != "":
+    p,n = styles.get(style, styles.get(STYLE_NAMES[1]))
+    return generate_image(face_file, p[1], n)
+
+def run_for_prompts3(face_file,style,progress=gr.Progress(track_tqdm=True)):
+    # if email != "":
+    p,n = styles.get(style, styles.get(STYLE_NAMES[1]))
+    return generate_image(face_file, p[2], n)
+
+def run_for_prompts4(face_file,style,progress=gr.Progress(track_tqdm=True)):
+    # if email != "":
+    p,n = styles.get(style, styles.get(STYLE_NAMES[1]))
+    return generate_image(face_file, p[3], n)
+
+
+def draw_kps(image_pil, kps, color_list=[(255,0,0), (0,255,0), (0,0,255), (255,255,0), (255,0,255)]):
+    stickwidth = 4
+    limbSeq = np.array([[0, 2], [1, 2], [3, 2], [4, 2]])
+    kps = np.array(kps)
+
+    w, h = image_pil.size
+    out_img = np.zeros([h, w, 3])
+
+    for i in range(len(limbSeq)):
+        index = limbSeq[i]
+        color = color_list[index[0]]
+
+        x = kps[index][:, 0]
+        y = kps[index][:, 1]
+        length = ((x[0] - x[1]) ** 2 + (y[0] - y[1]) ** 2) ** 0.5
+        angle = math.degrees(math.atan2(y[0] - y[1], x[0] - x[1]))
+        polygon = cv2.ellipse2Poly((int(np.mean(x)), int(np.mean(y))), (int(length / 2), stickwidth), int(angle), 0, 360, 1)
+        out_img = cv2.fillConvexPoly(out_img.copy(), polygon, color)
+    out_img = (out_img * 0.6).astype(np.uint8)
+
+    for idx_kp, kp in enumerate(kps):
+        color = color_list[idx_kp]
+        x, y = kp
+        out_img = cv2.circle(out_img.copy(), (int(x), int(y)), 10, color, -1)
+
+    out_img_pil = Image.fromarray(out_img.astype(np.uint8))
+    return out_img_pil
+
+def resize_img(input_image, max_side=640, min_side=640, size=None, 
+            pad_to_max_side=True, mode=PIL.Image.BILINEAR, base_pixel_number=64):
+
+        w, h = input_image.size
+        print(w)
+        print(h)
+        if size is not None:
+            w_resize_new, h_resize_new = size
+        else:
+            ratio = min_side / min(h, w)
+            w, h = round(ratio*w), round(ratio*h)
+            ratio = max_side / max(h, w)
+            input_image = input_image.resize([round(ratio*w), round(ratio*h)], mode)
+            w_resize_new = (round(ratio * w) // base_pixel_number) * base_pixel_number
+            h_resize_new = (round(ratio * h) // base_pixel_number) * base_pixel_number
+        input_image = input_image.resize([w_resize_new, h_resize_new], mode)
+
+        if pad_to_max_side:
+            res = np.ones([max_side, max_side, 3], dtype=np.uint8) * 255
+            offset_x = (max_side - w_resize_new) // 2
+            offset_y = (max_side - h_resize_new) // 2
+            res[offset_y:offset_y+h_resize_new, offset_x:offset_x+w_resize_new] = np.array(input_image)
+            input_image = Image.fromarray(res)
+        return input_image
+
+
+def generate_image(face_image,prompt,negative_prompt):
+    pose_image_path = None
+    # prompt = "superman"
+    enable_LCM = False
+    identitynet_strength_ratio = 0.95
+    adapter_strength_ratio = 0.60
+    num_steps = 15
+    guidance_scale = 8.5
+    seed = random.randint(0, MAX_SEED)
+    # negative_prompt = ""
+    # negative_prompt += neg
+    enhance_face_region = True
+    if enable_LCM:
+        pipe.enable_lora()
+        pipe.scheduler = LCMScheduler.from_config(pipe.scheduler.config)
+    else:
+        pipe.disable_lora()
+        pipe.scheduler = diffusers.EulerDiscreteScheduler.from_config(pipe.scheduler.config)
+
+    if face_image is None:
+        raise gr.Error(f"Cannot find any input face image! Please upload the face image")
+    
+    # if prompt is None:
+    #     prompt = "a person"
+    
+    # apply the style template
+    # prompt, negative_prompt = apply_style(style_name, prompt, negative_prompt)
+    
+    # face_image = load_image(face_image_path)
+    face_image = resize_img(face_image)
+    face_image_cv2 = convert_from_image_to_cv2(face_image)
+    height, width, _ = face_image_cv2.shape
+    
+    # Extract face features
+    face_info = app.get(face_image_cv2)
+    
+    if len(face_info) == 0:
+        raise gr.Error(f"Cannot find any face in the image! Please upload another person image")
+    
+    face_info = sorted(face_info, key=lambda x:(x['bbox'][2]-x['bbox'][0])*(x['bbox'][3]-x['bbox'][1]))[-1]  # only use the maximum face
+    face_emb = face_info['embedding']
+    face_kps = draw_kps(convert_from_cv2_to_image(face_image_cv2), face_info['kps'])
+    
+    if pose_image_path is not None:
+        pose_image = load_image(pose_image_path)
+        pose_image = resize_img(pose_image)
+        pose_image_cv2 = convert_from_image_to_cv2(pose_image)
+        
+        face_info = app.get(pose_image_cv2)
+        
+        if len(face_info) == 0:
+            raise gr.Error(f"Cannot find any face in the reference image! Please upload another person image")
+        
+        face_info = face_info[-1]
+        face_kps = draw_kps(pose_image, face_info['kps'])
+        
+        width, height = face_kps.size
+
+    if enhance_face_region:
+        control_mask = np.zeros([height, width, 3])
+        x1, y1, x2, y2 = face_info["bbox"]
+        x1, y1, x2, y2 = int(x1), int(y1), int(x2), int(y2)
+        control_mask[y1:y2, x1:x2] = 255
+        control_mask = Image.fromarray(control_mask.astype(np.uint8))
+    else:
+        control_mask = None
+                    
+    generator = torch.Generator(device=device).manual_seed(seed)
+    
+    print("Start inference...")
+    print(f"[Debug] Prompt: {prompt}, \n[Debug] Neg Prompt: {negative_prompt}")
+    
+    pipe.set_ip_adapter_scale(adapter_strength_ratio)
+    images = pipe(
+        prompt=prompt,
+        negative_prompt=negative_prompt,
+        image_embeds=face_emb,
+        image=face_kps,
+        control_mask=control_mask,
+        controlnet_conditioning_scale=float(identitynet_strength_ratio),
+        num_inference_steps=num_steps,
+        guidance_scale=guidance_scale,
+        height=height,
+        width=width,
+        generator=generator,
+        # num_images_per_prompt = 4
+    ).images
+
+    return images[0]
+
+def main(pretrained_model_name_or_path="wangqixun/YamerMIX_v8", enable_lcm_arg=False):
+
+    
+
+
+    ### Description
+    title = r"""
+    <h1 align="center">Choose your AVATAR</h1>
+    """
+
+    description = r"""
+    <h2> Powered by IDfy </h2>"""
+
+    article = r""""""
+
+    tips = r""""""
+    
+    js = '''  '''
+    
+    css = '''
+    .gradio-container {width: 95% !important; background-color: #E6F3FF;} 
+    .image-gallery {height: 100vh !important; overflow: auto;}
+    .gradio-row .gradio-element { margin: 0 !important; }
+    '''
+
+
+    with gr.Blocks(css=css, js=js) as demo:
+
+        # description
+        gr.Markdown(title)
+        with gr.Row():
+            gr.Image("./gradio_demo/logo.png",scale=0,min_width=50,show_label=False,show_download_button=False)
+            gr.Markdown(description)
+        with gr.Row():
+            with gr.Column():
+                style = gr.Dropdown(label="Choose your STYLE", choices=STYLE_NAMES)
+                face_file = gr.Image(label="Upload a photo of your face", type="pil",sources="webcam")
+                submit = gr.Button("Submit", variant="primary")
+            with gr.Column():
+                with gr.Row():
+                    gallery1 = gr.Image(label="Generated Images")
+                    gallery2 = gr.Image(label="Generated Images")
+                with gr.Row():
+                    gallery3 = gr.Image(label="Generated Images")
+                    gallery4 = gr.Image(label="Generated Images")
+                email = gr.Textbox(label="Email",
+                        info="Enter your email address",
+                        value="")
+            
+            usage_tips = gr.Markdown(label="Usage tips of InstantID", value=tips ,visible=False)
+
+            face_file.upload(
+                fn=remove_tips,
+                outputs=usage_tips,
+                queue=True,
+                api_name=False,
+                show_progress = "full"
+            )
+
+            submit.click(
+                fn=remove_tips,
+                outputs=usage_tips,
+                queue=True,
+                api_name=False,
+                show_progress = "full"
+            ).then(
+                fn=run_for_prompts1,
+                inputs=[face_file,style],
+                outputs=[gallery1]
+            )
+        
+        
+        gr.Markdown(article)
+
+    demo.launch(share=True)
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--pretrained_model_name_or_path", type=str, default="wangqixun/YamerMIX_v8")
+    args = parser.parse_args()
+
+    main(args.pretrained_model_name_or_path, False)

gradio_demo/download_models.py

@@ -0,0 +1,27 @@
+from huggingface_hub import hf_hub_download
+import gdown
+import os
+
+# download models
+hf_hub_download(
+    repo_id="InstantX/InstantID",
+    filename="ControlNetModel/config.json",
+    local_dir="./checkpoints",
+)
+hf_hub_download(
+    repo_id="InstantX/InstantID",
+    filename="ControlNetModel/diffusion_pytorch_model.safetensors",
+    local_dir="./checkpoints",
+)
+hf_hub_download(
+    repo_id="InstantX/InstantID", filename="ip-adapter.bin", local_dir="./checkpoints"
+)
+hf_hub_download(
+    repo_id="latent-consistency/lcm-lora-sdxl",
+    filename="pytorch_lora_weights.safetensors",
+    local_dir="./checkpoints",
+)
+# download antelopev2
+gdown.download(url="https://drive.google.com/file/d/18wEUfMNohBJ4K3Ly5wpTejPfDzp-8fI8/view?usp=sharing", output="./models/", quiet=False, fuzzy=True)
+# unzip antelopev2.zip
+os.system("unzip ./models/antelopev2.zip -d ./models/")

gradio_demo/model_util.py

@@ -0,0 +1,472 @@
+from typing import Literal, Union, Optional, Tuple, List
+
+import torch
+from transformers import CLIPTextModel, CLIPTokenizer, CLIPTextModelWithProjection
+from diffusers import (
+    UNet2DConditionModel,
+    SchedulerMixin,
+    StableDiffusionPipeline,
+    StableDiffusionXLPipeline,
+    AutoencoderKL,
+)
+from diffusers.pipelines.stable_diffusion.convert_from_ckpt import (
+    convert_ldm_unet_checkpoint,
+)
+from safetensors.torch import load_file
+from diffusers.schedulers import (
+    DDIMScheduler,
+    DDPMScheduler,
+    LMSDiscreteScheduler,
+    EulerDiscreteScheduler,
+    EulerAncestralDiscreteScheduler,
+    UniPCMultistepScheduler,
+)
+
+from omegaconf import OmegaConf
+
+# DiffUsers版StableDiffusionのモデルパラメータ
+NUM_TRAIN_TIMESTEPS = 1000
+BETA_START = 0.00085
+BETA_END = 0.0120
+
+UNET_PARAMS_MODEL_CHANNELS = 320
+UNET_PARAMS_CHANNEL_MULT = [1, 2, 4, 4]
+UNET_PARAMS_ATTENTION_RESOLUTIONS = [4, 2, 1]
+UNET_PARAMS_IMAGE_SIZE = 64  # fixed from old invalid value `32`
+UNET_PARAMS_IN_CHANNELS = 4
+UNET_PARAMS_OUT_CHANNELS = 4
+UNET_PARAMS_NUM_RES_BLOCKS = 2
+UNET_PARAMS_CONTEXT_DIM = 768
+UNET_PARAMS_NUM_HEADS = 8
+# UNET_PARAMS_USE_LINEAR_PROJECTION = False
+
+VAE_PARAMS_Z_CHANNELS = 4
+VAE_PARAMS_RESOLUTION = 256
+VAE_PARAMS_IN_CHANNELS = 3
+VAE_PARAMS_OUT_CH = 3
+VAE_PARAMS_CH = 128
+VAE_PARAMS_CH_MULT = [1, 2, 4, 4]
+VAE_PARAMS_NUM_RES_BLOCKS = 2
+
+# V2
+V2_UNET_PARAMS_ATTENTION_HEAD_DIM = [5, 10, 20, 20]
+V2_UNET_PARAMS_CONTEXT_DIM = 1024
+# V2_UNET_PARAMS_USE_LINEAR_PROJECTION = True
+
+TOKENIZER_V1_MODEL_NAME = "CompVis/stable-diffusion-v1-4"
+TOKENIZER_V2_MODEL_NAME = "stabilityai/stable-diffusion-2-1"
+
+AVAILABLE_SCHEDULERS = Literal["ddim", "ddpm", "lms", "euler_a", "euler", "uniPC"]
+
+SDXL_TEXT_ENCODER_TYPE = Union[CLIPTextModel, CLIPTextModelWithProjection]
+
+DIFFUSERS_CACHE_DIR = None  # if you want to change the cache dir, change this
+
+
+def load_checkpoint_with_text_encoder_conversion(ckpt_path: str, device="cpu"):
+    # text encoderの格納形式が違うモデルに対応する ('text_model'がない)
+    TEXT_ENCODER_KEY_REPLACEMENTS = [
+        (
+            "cond_stage_model.transformer.embeddings.",
+            "cond_stage_model.transformer.text_model.embeddings.",
+        ),
+        (
+            "cond_stage_model.transformer.encoder.",
+            "cond_stage_model.transformer.text_model.encoder.",
+        ),
+        (
+            "cond_stage_model.transformer.final_layer_norm.",
+            "cond_stage_model.transformer.text_model.final_layer_norm.",
+        ),
+    ]
+
+    if ckpt_path.endswith(".safetensors"):
+        checkpoint = None
+        state_dict = load_file(ckpt_path)  # , device) # may causes error
+    else:
+        checkpoint = torch.load(ckpt_path, map_location=device)
+        if "state_dict" in checkpoint:
+            state_dict = checkpoint["state_dict"]
+        else:
+            state_dict = checkpoint
+            checkpoint = None
+
+    key_reps = []
+    for rep_from, rep_to in TEXT_ENCODER_KEY_REPLACEMENTS:
+        for key in state_dict.keys():
+            if key.startswith(rep_from):
+                new_key = rep_to + key[len(rep_from) :]
+                key_reps.append((key, new_key))
+
+    for key, new_key in key_reps:
+        state_dict[new_key] = state_dict[key]
+        del state_dict[key]
+
+    return checkpoint, state_dict
+
+
+def create_unet_diffusers_config(v2, use_linear_projection_in_v2=False):
+    """
+    Creates a config for the diffusers based on the config of the LDM model.
+    """
+    # unet_params = original_config.model.params.unet_config.params
+
+    block_out_channels = [
+        UNET_PARAMS_MODEL_CHANNELS * mult for mult in UNET_PARAMS_CHANNEL_MULT
+    ]
+
+    down_block_types = []
+    resolution = 1
+    for i in range(len(block_out_channels)):
+        block_type = (
+            "CrossAttnDownBlock2D"
+            if resolution in UNET_PARAMS_ATTENTION_RESOLUTIONS
+            else "DownBlock2D"
+        )
+        down_block_types.append(block_type)
+        if i != len(block_out_channels) - 1:
+            resolution *= 2
+
+    up_block_types = []
+    for i in range(len(block_out_channels)):
+        block_type = (
+            "CrossAttnUpBlock2D"
+            if resolution in UNET_PARAMS_ATTENTION_RESOLUTIONS
+            else "UpBlock2D"
+        )
+        up_block_types.append(block_type)
+        resolution //= 2
+
+    config = dict(
+        sample_size=UNET_PARAMS_IMAGE_SIZE,
+        in_channels=UNET_PARAMS_IN_CHANNELS,
+        out_channels=UNET_PARAMS_OUT_CHANNELS,
+        down_block_types=tuple(down_block_types),
+        up_block_types=tuple(up_block_types),
+        block_out_channels=tuple(block_out_channels),
+        layers_per_block=UNET_PARAMS_NUM_RES_BLOCKS,
+        cross_attention_dim=UNET_PARAMS_CONTEXT_DIM
+        if not v2
+        else V2_UNET_PARAMS_CONTEXT_DIM,
+        attention_head_dim=UNET_PARAMS_NUM_HEADS
+        if not v2
+        else V2_UNET_PARAMS_ATTENTION_HEAD_DIM,
+        # use_linear_projection=UNET_PARAMS_USE_LINEAR_PROJECTION if not v2 else V2_UNET_PARAMS_USE_LINEAR_PROJECTION,
+    )
+    if v2 and use_linear_projection_in_v2:
+        config["use_linear_projection"] = True
+
+    return config
+
+
+def load_diffusers_model(
+    pretrained_model_name_or_path: str,
+    v2: bool = False,
+    clip_skip: Optional[int] = None,
+    weight_dtype: torch.dtype = torch.float32,
+) -> Tuple[CLIPTokenizer, CLIPTextModel, UNet2DConditionModel,]:
+    if v2:
+        tokenizer = CLIPTokenizer.from_pretrained(
+            TOKENIZER_V2_MODEL_NAME,
+            subfolder="tokenizer",
+            torch_dtype=weight_dtype,
+            cache_dir=DIFFUSERS_CACHE_DIR,
+        )
+        text_encoder = CLIPTextModel.from_pretrained(
+            pretrained_model_name_or_path,
+            subfolder="text_encoder",
+            # default is clip skip 2
+            num_hidden_layers=24 - (clip_skip - 1) if clip_skip is not None else 23,
+            torch_dtype=weight_dtype,
+            cache_dir=DIFFUSERS_CACHE_DIR,
+        )
+    else:
+        tokenizer = CLIPTokenizer.from_pretrained(
+            TOKENIZER_V1_MODEL_NAME,
+            subfolder="tokenizer",
+            torch_dtype=weight_dtype,
+            cache_dir=DIFFUSERS_CACHE_DIR,
+        )
+        text_encoder = CLIPTextModel.from_pretrained(
+            pretrained_model_name_or_path,
+            subfolder="text_encoder",
+            num_hidden_layers=12 - (clip_skip - 1) if clip_skip is not None else 12,
+            torch_dtype=weight_dtype,
+            cache_dir=DIFFUSERS_CACHE_DIR,
+        )
+
+    unet = UNet2DConditionModel.from_pretrained(
+        pretrained_model_name_or_path,
+        subfolder="unet",
+        torch_dtype=weight_dtype,
+        cache_dir=DIFFUSERS_CACHE_DIR,
+    )
+
+    vae = AutoencoderKL.from_pretrained(pretrained_model_name_or_path, subfolder="vae")
+
+    return tokenizer, text_encoder, unet, vae
+
+
+def load_checkpoint_model(
+    checkpoint_path: str,
+    v2: bool = False,
+    clip_skip: Optional[int] = None,
+    weight_dtype: torch.dtype = torch.float32,
+) -> Tuple[CLIPTokenizer, CLIPTextModel, UNet2DConditionModel,]:
+    pipe = StableDiffusionPipeline.from_single_file(
+        checkpoint_path,
+        upcast_attention=True if v2 else False,
+        torch_dtype=weight_dtype,
+        cache_dir=DIFFUSERS_CACHE_DIR,
+    )
+
+    _, state_dict = load_checkpoint_with_text_encoder_conversion(checkpoint_path)
+    unet_config = create_unet_diffusers_config(v2, use_linear_projection_in_v2=v2)
+    unet_config["class_embed_type"] = None
+    unet_config["addition_embed_type"] = None
+    converted_unet_checkpoint = convert_ldm_unet_checkpoint(state_dict, unet_config)
+    unet = UNet2DConditionModel(**unet_config)
+    unet.load_state_dict(converted_unet_checkpoint)
+
+    tokenizer = pipe.tokenizer
+    text_encoder = pipe.text_encoder
+    vae = pipe.vae
+    if clip_skip is not None:
+        if v2:
+            text_encoder.config.num_hidden_layers = 24 - (clip_skip - 1)
+        else:
+            text_encoder.config.num_hidden_layers = 12 - (clip_skip - 1)
+
+    del pipe
+
+    return tokenizer, text_encoder, unet, vae
+
+
+def load_models(
+    pretrained_model_name_or_path: str,
+    scheduler_name: str,
+    v2: bool = False,
+    v_pred: bool = False,
+    weight_dtype: torch.dtype = torch.float32,
+) -> Tuple[CLIPTokenizer, CLIPTextModel, UNet2DConditionModel, SchedulerMixin,]:
+    if pretrained_model_name_or_path.endswith(
+        ".ckpt"
+    ) or pretrained_model_name_or_path.endswith(".safetensors"):
+        tokenizer, text_encoder, unet, vae = load_checkpoint_model(
+            pretrained_model_name_or_path, v2=v2, weight_dtype=weight_dtype
+        )
+    else:  # diffusers
+        tokenizer, text_encoder, unet, vae = load_diffusers_model(
+            pretrained_model_name_or_path, v2=v2, weight_dtype=weight_dtype
+        )
+
+    if scheduler_name:
+        scheduler = create_noise_scheduler(
+            scheduler_name,
+            prediction_type="v_prediction" if v_pred else "epsilon",
+        )
+    else:
+        scheduler = None
+
+    return tokenizer, text_encoder, unet, scheduler, vae
+
+
+def load_diffusers_model_xl(
+    pretrained_model_name_or_path: str,
+    weight_dtype: torch.dtype = torch.float32,
+) -> Tuple[List[CLIPTokenizer], List[SDXL_TEXT_ENCODER_TYPE], UNet2DConditionModel,]:
+    # returns tokenizer, tokenizer_2, text_encoder, text_encoder_2, unet
+
+    tokenizers = [
+        CLIPTokenizer.from_pretrained(
+            pretrained_model_name_or_path,
+            subfolder="tokenizer",
+            torch_dtype=weight_dtype,
+            cache_dir=DIFFUSERS_CACHE_DIR,
+        ),
+        CLIPTokenizer.from_pretrained(
+            pretrained_model_name_or_path,
+            subfolder="tokenizer_2",
+            torch_dtype=weight_dtype,
+            cache_dir=DIFFUSERS_CACHE_DIR,
+            pad_token_id=0,  # same as open clip
+        ),
+    ]
+
+    text_encoders = [
+        CLIPTextModel.from_pretrained(
+            pretrained_model_name_or_path,
+            subfolder="text_encoder",
+            torch_dtype=weight_dtype,
+            cache_dir=DIFFUSERS_CACHE_DIR,
+        ),
+        CLIPTextModelWithProjection.from_pretrained(
+            pretrained_model_name_or_path,
+            subfolder="text_encoder_2",
+            torch_dtype=weight_dtype,
+            cache_dir=DIFFUSERS_CACHE_DIR,
+        ),
+    ]
+
+    unet = UNet2DConditionModel.from_pretrained(
+        pretrained_model_name_or_path,
+        subfolder="unet",
+        torch_dtype=weight_dtype,
+        cache_dir=DIFFUSERS_CACHE_DIR,
+    )
+    vae = AutoencoderKL.from_pretrained(pretrained_model_name_or_path, subfolder="vae")
+    return tokenizers, text_encoders, unet, vae
+
+
+def load_checkpoint_model_xl(
+    checkpoint_path: str,
+    weight_dtype: torch.dtype = torch.float32,
+) -> Tuple[List[CLIPTokenizer], List[SDXL_TEXT_ENCODER_TYPE], UNet2DConditionModel,]:
+    pipe = StableDiffusionXLPipeline.from_single_file(
+        checkpoint_path,
+        torch_dtype=weight_dtype,
+        cache_dir=DIFFUSERS_CACHE_DIR,
+    )
+
+    unet = pipe.unet
+    vae = pipe.vae
+    tokenizers = [pipe.tokenizer, pipe.tokenizer_2]
+    text_encoders = [pipe.text_encoder, pipe.text_encoder_2]
+    if len(text_encoders) == 2:
+        text_encoders[1].pad_token_id = 0
+
+    del pipe
+
+    return tokenizers, text_encoders, unet, vae
+
+
+def load_models_xl(
+    pretrained_model_name_or_path: str,
+    scheduler_name: str,
+    weight_dtype: torch.dtype = torch.float32,
+    noise_scheduler_kwargs=None,
+) -> Tuple[
+    List[CLIPTokenizer],
+    List[SDXL_TEXT_ENCODER_TYPE],
+    UNet2DConditionModel,
+    SchedulerMixin,
+]:
+    if pretrained_model_name_or_path.endswith(
+        ".ckpt"
+    ) or pretrained_model_name_or_path.endswith(".safetensors"):
+        (tokenizers, text_encoders, unet, vae) = load_checkpoint_model_xl(
+            pretrained_model_name_or_path, weight_dtype
+        )
+    else:  # diffusers
+        (tokenizers, text_encoders, unet, vae) = load_diffusers_model_xl(
+            pretrained_model_name_or_path, weight_dtype
+        )
+    if scheduler_name:
+        scheduler = create_noise_scheduler(scheduler_name, noise_scheduler_kwargs)
+    else:
+        scheduler = None
+
+    return tokenizers, text_encoders, unet, scheduler, vae
+
+def create_noise_scheduler(
+    scheduler_name: AVAILABLE_SCHEDULERS = "ddpm",
+    noise_scheduler_kwargs=None,
+    prediction_type: Literal["epsilon", "v_prediction"] = "epsilon",
+) -> SchedulerMixin:
+    name = scheduler_name.lower().replace(" ", "_")
+    if name.lower() == "ddim":
+        # https://huggingface.co/docs/diffusers/v0.17.1/en/api/schedulers/ddim
+        scheduler = DDIMScheduler(**OmegaConf.to_container(noise_scheduler_kwargs))
+    elif name.lower() == "ddpm":
+        # https://huggingface.co/docs/diffusers/v0.17.1/en/api/schedulers/ddpm
+        scheduler = DDPMScheduler(**OmegaConf.to_container(noise_scheduler_kwargs))
+    elif name.lower() == "lms":
+        # https://huggingface.co/docs/diffusers/v0.17.1/en/api/schedulers/lms_discrete
+        scheduler = LMSDiscreteScheduler(
+            **OmegaConf.to_container(noise_scheduler_kwargs)
+        )
+    elif name.lower() == "euler_a":
+        # https://huggingface.co/docs/diffusers/v0.17.1/en/api/schedulers/euler_ancestral
+        scheduler = EulerAncestralDiscreteScheduler(
+            **OmegaConf.to_container(noise_scheduler_kwargs)
+        )
+    elif name.lower() == "euler":
+        # https://huggingface.co/docs/diffusers/v0.17.1/en/api/schedulers/euler_ancestral
+        scheduler = EulerDiscreteScheduler(
+            **OmegaConf.to_container(noise_scheduler_kwargs)
+        )
+    elif name.lower() == "unipc":
+        # https://huggingface.co/docs/diffusers/v0.17.1/en/api/schedulers/unipc
+        scheduler = UniPCMultistepScheduler(
+            **OmegaConf.to_container(noise_scheduler_kwargs)
+        )
+    else:
+        raise ValueError(f"Unknown scheduler name: {name}")
+
+    return scheduler
+
+
+def torch_gc():
+    import gc
+
+    gc.collect()
+    if torch.cuda.is_available():
+        with torch.cuda.device("cuda"):
+            torch.cuda.empty_cache()
+            torch.cuda.ipc_collect()
+
+
+from enum import Enum
+
+
+class CPUState(Enum):
+    GPU = 0
+    CPU = 1
+    MPS = 2
+
+
+cpu_state = CPUState.GPU
+xpu_available = False
+directml_enabled = False
+
+
+def is_intel_xpu():
+    global cpu_state
+    global xpu_available
+    if cpu_state == CPUState.GPU:
+        if xpu_available:
+            return True
+    return False
+
+
+try:
+    import intel_extension_for_pytorch as ipex
+
+    if torch.xpu.is_available():
+        xpu_available = True
+except:
+    pass
+
+try:
+    if torch.backends.mps.is_available():
+        cpu_state = CPUState.MPS
+        import torch.mps
+except:
+    pass
+
+
+def get_torch_device():
+    global directml_enabled
+    global cpu_state
+    if directml_enabled:
+        global directml_device
+        return directml_device
+    if cpu_state == CPUState.MPS:
+        return torch.device("mps")
+    if cpu_state == CPUState.CPU:
+        return torch.device("cpu")
+    else:
+        if is_intel_xpu():
+            return torch.device("xpu")
+        else:
+            return torch.device(torch.cuda.current_device())

gradio_demo/preprocess.py

@@ -0,0 +1,232 @@
+import os
+import random
+import csv
+import gc
+import glob
+from datetime import datetime
+import time
+from pathlib import Path
+from style_template import style_list
+from PIL import Image, ImageOps
+
+# Default Configuration variables
+INPUT_FOLDER_NAME = 'examples'
+OUTPUT_FOLDER_NAME = 'generated_images'
+LOG_FILENAME = 'generation_log.csv'
+logfile_path = os.path.join(os.getcwd(), LOG_FILENAME)
+
+PROMPT = "human, sharp focus"
+NEGATIVE_PROMPT = "(blurry, blur, text, abstract, glitch, lowres, low quality, worst quality:1.2), (text:1.2), watermark, painting, drawing, illustration, glitch, deformed, mutated, cross-eyed, ugly, disfigured"
+IDENTITYNET_STRENGTH_RATIO_RANGE = (1.0, 1.5)
+ADAPTER_STRENGTH_RATIO_RANGE = (0.7, 1.0)
+NUM_INFERENCE_STEPS_RANGE = (40, 60)
+GUIDANCE_SCALE_RANGE = (7.0, 12.0)
+MAX_SIDE = 1280
+MIN_SIDE = 1024
+NUMBER_OF_LOOPS = 1
+
+# Dynamically create the STYLES list from imported style_list
+STYLES = [style["name"] for style in style_list]
+USE_RANDOM_STYLE = False
+
+def choose_random_style():
+    return random.choice(STYLES)
+
+def get_random_image_file(input_folder):
+    valid_extensions = [".jpg", ".jpeg", ".png"]
+    files = [file for file in Path(input_folder).glob("*") if file.suffix.lower() in valid_extensions]
+    if not files:
+        raise FileNotFoundError(f"No images found in directory {input_folder}")
+    return str(random.choice(files))
+
+def resize_and_pad_image(image_path, max_side, min_side, pad_color=(255, 255, 255)):
+    # Open an image using PIL
+    image = Image.open(image_path)
+
+    # Calculate the scale and new size
+    ratio = min(min_side / min(image.size), max_side / max(image.size))
+    new_size = (int(image.size[0] * ratio), int(image.size[1] * ratio))
+
+    # Resize the image
+    image = image.resize(new_size, Image.BILINEAR)
+    
+    # Calculate padding
+    delta_w = max_side - new_size[0]
+    delta_h = max_side - new_size[1]
+
+    # Pad the resized image to make it square
+    padding = (delta_w // 2, delta_h // 2, delta_w - (delta_w // 2), delta_h - (delta_h // 2))
+    image = ImageOps.expand(image, padding, pad_color)
+
+    return image
+
+def log_to_csv(logfile_path, image_name, new_file_name='Unknown', identitynet_strength_ratio=0.0, adapter_strength_ratio=0.0, num_inference_steps=0, guidance_scale=0.0, seed=0, success=True, error_message='', style_name="", prompt="", negative_prompt="", time_taken=0.0, current_timestamp=""):
+    os.makedirs(os.path.dirname(logfile_path), exist_ok=True)
+    file_exists = os.path.isfile(logfile_path)
+
+    with open(logfile_path, 'a', newline='', encoding='utf-8') as csvfile:
+        fieldnames = ['image_name', 'new_file_name', 'identitynet_strength_ratio', 'adapter_strength_ratio', 'num_inference_steps', 'guidance_scale', 'seed', 'success', 'error_message', 'style_name', 'prompt', 'negative_prompt', 'time_taken', 'current_timestamp']
+        writer = csv.DictWriter(csvfile, fieldnames=fieldnames)
+        if not file_exists:
+            writer.writeheader()
+        writer.writerow({
+            'image_name': image_name,
+            'new_file_name': new_file_name,
+            'identitynet_strength_ratio': identitynet_strength_ratio,
+            'adapter_strength_ratio': adapter_strength_ratio,
+            'num_inference_steps': num_inference_steps,
+            'guidance_scale': guidance_scale,
+            'seed': seed,
+            'success': success,
+            'error_message': error_message,
+            'style_name': style_name,
+            'prompt': prompt,
+            'negative_prompt': negative_prompt,
+            'time_taken': time_taken,
+            'current_timestamp': current_timestamp
+        })
+
+def initial_image(generate_image_func):
+    overall_start_time = time.time()
+    total_time_taken = 0.0 
+
+    # Initialize a counter for processed images at the beginning of the function
+    processed_images_count = 0
+
+    # List all image files in the `INPUT_FOLDER_NAME`
+    image_files = glob.glob(f'{INPUT_FOLDER_NAME}/*.png') + \
+                  glob.glob(f'{INPUT_FOLDER_NAME}/*.jpg') + \
+                  glob.glob(f'{INPUT_FOLDER_NAME}/*.jpeg')
+    
+    # Check if we found any images
+    if not image_files:
+        raise FileNotFoundError(f"No images found in directory {INPUT_FOLDER_NAME}")
+    
+    # Print the count of detected image files
+    print(f"Processing a total of {len(image_files)} image(s) in '{INPUT_FOLDER_NAME}'")
+
+    # Shuffle the image files randomly
+    random.shuffle(image_files)
+    
+    total_images = len(image_files)  # Get the total number of images to process
+    
+    for loop in range(NUMBER_OF_LOOPS):
+        print(f"Starting loop {loop+1} of {NUMBER_OF_LOOPS}")
+
+        for image_number, face_image_path in enumerate(image_files, start=1):
+            loop_start_time = datetime.now()
+            face_image = [face_image_path]
+            basename = os.path.basename(face_image_path)
+            processed_images_count += 1
+
+            # Resize and pad the image before processing
+            processed_image = resize_and_pad_image(
+                image_path=face_image_path,
+                max_side=MAX_SIDE,
+                min_side=MIN_SIDE
+            )
+
+            if USE_RANDOM_STYLE:
+                style_name = choose_random_style()
+            else:
+                style_name = "(No style)"
+
+            identitynet_strength_ratio = random.uniform(*IDENTITYNET_STRENGTH_RATIO_RANGE)
+            adapter_strength_ratio = random.uniform(*ADAPTER_STRENGTH_RATIO_RANGE)
+            num_inference_steps = random.randint(*NUM_INFERENCE_STEPS_RANGE)
+            guidance_scale = random.uniform(*GUIDANCE_SCALE_RANGE)
+            seed = random.randint(0, 2**32 - 1)
+            
+            # Print settings for the current image BEFORE processing it
+            print_generation_settings(basename, style_name, identitynet_strength_ratio, 
+                                      adapter_strength_ratio, num_inference_steps, guidance_scale, seed,
+                                      image_number, total_images)
+
+            # Here, the generate_image_func is supposedly called and image processing happens
+            _, _, generated_file_paths = generate_image_func(
+                face_image=face_image,
+                pose_image=None,
+                prompt=PROMPT,
+                negative_prompt=NEGATIVE_PROMPT,
+                style_name=style_name,
+                enhance_face_region=True,
+                num_steps=num_inference_steps,
+                identitynet_strength_ratio=identitynet_strength_ratio,
+                adapter_strength_ratio=adapter_strength_ratio,
+                guidance_scale=guidance_scale,
+                seed=seed
+            )
+            
+            loop_end_time = datetime.now()
+            loop_time_taken = (loop_end_time - loop_start_time).total_seconds()
+
+            # Immediately print the time taken and current time.
+            print(f"Time taken to process image: {loop_time_taken:.2f} seconds")
+
+            # Update the total time taken with this image's processing time
+            total_time_taken += loop_time_taken
+
+            # Calculate the average time taken per image
+            average_time_per_image = total_time_taken / image_number
+
+            current_timestamp = loop_end_time.strftime("%Y-%m-%d %H:%M:%S")  # Current time after processing
+            print(f"Current timestamp: {current_timestamp}")
+
+            # Calculate estimated remaining time considering the images left in this loop and the additional loops
+            remaining_images_this_loop = total_images - image_number
+            remaining_images_in_additional_loops = (NUMBER_OF_LOOPS - (loop + 1)) * total_images
+            total_remaining_images = remaining_images_this_loop + remaining_images_in_additional_loops
+            estimated_time_remaining = average_time_per_image * total_remaining_images
+
+            # Display the estimated time remaining including remaining loops
+            print(f"Estimated time remaining (including loops): {estimated_time_remaining // 60:.0f} minutes, {estimated_time_remaining % 60:.0f} seconds")
+
+            # Display the overall average time per image in seconds
+            print(f"Overall average time per image: {average_time_per_image:.2f} seconds")
+
+            # Display the total number of remaining images to process including looping
+            print(f"Total remaining images to process (including loops): {total_remaining_images}")
+
+
+            success = True  # Assuming generation was successful.
+            error_message = ""  # Assuming no error.
+
+            # Log to CSV after the image generation.
+            for generated_file_path in generated_file_paths:
+                new_file_name = os.path.basename(generated_file_path)
+                log_to_csv(logfile_path, basename, new_file_name, identitynet_strength_ratio,
+                           adapter_strength_ratio, num_inference_steps, guidance_scale, seed, success,
+                           error_message, style_name, PROMPT, NEGATIVE_PROMPT, loop_time_taken, current_timestamp)
+            
+                
+            del generated_file_paths  # Explicitly delete large variables
+            gc.collect()  # Call garbage collection
+
+
+    # At the end of the initial_image() function, add:
+    total_elapsed_time = time.time() - overall_start_time
+    print("\n===FINAL SUMMARY===")
+    print(f"Total loops completed: {NUMBER_OF_LOOPS}")
+    print(f"Total images processed per loop: {len(image_files)}")
+    print(f"Overall total images processed: {NUMBER_OF_LOOPS * len(image_files)}") # Multiplied by the number of loops
+    print(f"Overall total time: {total_elapsed_time / 60:.2f} minutes")
+
+               
+def print_generation_settings(basename, style_name, identitynet_strength_ratio, adapter_strength_ratio, num_inference_steps, guidance_scale, seed, image_number, total_images):
+    print("===IMAGE GENERATION DATA SUMMARY===")
+    # Print settings for the current image
+    print(f"- Image {image_number} of {total_images}\n"
+          f"- Filename: {basename}\n"
+          f"- Style: {style_name}\n"
+          f"- IdentityNet strength ratio: {identitynet_strength_ratio:0.2f}\n"
+          f"- Adapter strength ratio: {adapter_strength_ratio:0.2f}\n"
+          f"- Number of inference steps: {num_inference_steps}\n"
+          f"- Guidance scale: {guidance_scale:0.2f}\n"
+          f"- Seed: {seed}\n"
+          f"- Input folder name: {INPUT_FOLDER_NAME}\n"
+          f"- Output folder name: {OUTPUT_FOLDER_NAME}\n"
+          f"- Prompt: {PROMPT}\n"
+          f"- Negative prompt: {NEGATIVE_PROMPT}\n"
+          f"- Number of loops: {NUMBER_OF_LOOPS}\n"
+          f"- Use random style: {USE_RANDOM_STYLE}\n")
+    print("===DEFINING COMPLETE, GENERATING IMAGE...===")

gradio_demo/requirements.txt

@@ -0,0 +1,19 @@
+diffusers==0.25.1
+torch==2.0.0
+torchvision==0.15.1
+transformers==4.37.1
+accelerate==0.25.0
+safetensors==0.4.3
+einops==0.7.0
+onnxruntime-gpu==1.18.1
+spaces==0.19.4
+omegaconf==2.3.0
+peft==0.11.1
+huggingface-hub==0.23.4
+opencv-python==4.10.0.84
+insightface==0.7.3
+gradio==4.38.1
+controlnet_aux==0.0.9
+gdown==5.2.0
+peft==0.11.1
+setuptools=71.1.0

gradio_demo/style_template.py

@@ -0,0 +1,136 @@
+# style_list = [
+#     {
+#         "name": "Professional",
+#         "prompt": ["Minimalist style, Simple, clean, uncluttered, modern, elegant, white background, professional photo, linkedin profile photo, formal attire, professional look","Minimalist style, Simple, clean, uncluttered, modern, elegant, white background, professional photo, linkedin profile photo, formal attire, professional look","Minimalist style, Simple, clean, uncluttered, modern, elegant, white background, professional photo, linkedin profile photo, formal attire, professional look","Minimalist style, Simple, clean, uncluttered, modern, elegant, white background, professional photo, linkedin profile photo, formal attire, professional look"],
+#         # "prompt": ["Minimalist style, Simple, clean, uncluttered, modern, elegant, white background, professional photo, linkedin profile photo, formal attire, professional look,highly detailed, sharp focus","Minimalist style, Simple, clean, uncluttered, modern, elegant, white background, professional photo, linkedin profile photo, formal attire, professional look","Minimalist style, Simple, clean, uncluttered, modern, elegant, white background, professional photo, linkedin profile photo, formal attire, professional look","Minimalist style, Simple, clean, uncluttered, modern, elegant, white background, professional photo, linkedin profile photo, formal attire, professional look"],
+#         "negative_prompt":
+#         # "photographic, realistic, realism, 35mm film, dslr, cropped, frame, text, deformed, glitch, noise, noisy, blurry, off-center, deformed, cross-eyed, closed eyes, bad anatomy, ugly, disfigured, sloppy, duplicate, mutated, black and white, multiple people, green, deformed"
+#         # "drawing, painting, crayon, sketch, graphite, impressionist, noisy, blurry, soft, deformed, ugly"
+        
+#         # "lowres, low quality, worst quality:1.2), (text:1.2), watermark, (frame:1.2), deformed, ugly, deformed eyes, blur, out of focus, blurry,  deformed"
+
+#         "high saturation, multiple people, two people, patchy, photographic, realistic, realism, 35mm film, dslr, cropped, frame, text, deformed, glitch,blurred, blurry, noise, noisy, off-center, deformed, cross-eyed, closed eyes, bad anatomy, ugly, disfigured, sloppy, duplicate, mutated, black and white"
+#     },
+#     {
+#         "name": "Quirky",
+#         "prompt": ["vibrant colorful, ink sketch|vector|2d colors, sharp focus, superman/wonderwoman, highly detailed, the clouds,colorful,ultra sharpness,4k","watercolor painting, japanese anime character with white/neon hair. vibrant, beautiful, painterly, detailed, textural, artistic","vibrant colorful, ink sketch|vector|2d colors, sharp focus, scooba diver, highly detailed, the ocean,fishes,colorful,ultra sharpness,4k","individual dressed as an eskimo, surrounded by snowy mountains and igloo, snow crystals, cold, windy background, frozen natural landscape in background,highly detailed, sharp focus, intricate design, 4k resolution"],
+#         "negative_prompt": "saturation, highly saturated,(lowres, low quality, worst quality:1.2), (text:1.2), watermark, (frame:1.2), deformed, ugly, deformed eyes, blur, out of focus, blurry, multiple people, buildings in background, green"
+#     },
+#     {
+#         "name": "Sci-fi",
+#         "prompt": ["ethereal fantasy concept art individual, magnificent, celestial, ethereal, painterly, epic, majestic, magical, fantasy art, cover art, dreamy","Dystopian style cyborg. Bleak, post-apocalyptic, somber, dramatic, highly detailed","Alien-themed,  Extraterrestrial, cosmic, otherworldly, mysterious, sci-fi, highly detailed", "Legend of Zelda style . Vibrant, fantasy, detailed, epic, heroic, reminiscent of The Legend of Zelda series"],
+#         "negative_prompt": "photographic, realistic, realism, 35mm film, dslr, cropped, frame, text, deformed, glitch, noise, noisy, off-center, deformed, cross-eyed, closed eyes, bad anatomy, ugly, disfigured, sloppy, duplicate, mutated, black and white, multiple people, green, deformed",
+#     }
+# ]
+
+# styles = {k["name"]: (k["prompt"], k["negative_prompt"]) for k in style_list}
+
+# # lowres, low quality, worst quality:1.2), (text:1.2), watermark, (frame:1.2), deformed, ugly, deformed eyes, blur, out of focus, blurry,  deformed
+
+# style_list = [
+#     {
+#         "name": "(No style)",
+#         "prompt": "Realistic, 4k resolution, ultra sharpness, {prompt} sitiing at a desk, office environment, professional photoshoot",
+#         "negative_prompt": "(lowres, low quality, worst quality:1.2), (text:1.2), watermark, (frame:1.2), deformed, ugly, deformed eyes, blur, out of focus, blurry, deformed cat, deformed, photo, anthropomorphic cat, monochrome, photo, pet collar, gun, weapon, blue, 3d, drones, drone, buildings in background, green",
+#     },
+#     {
+#         "name": "Watercolor",
+#         "prompt": "watercolor painting, japanese anime character with white/neon hair. vibrant, beautiful, painterly, detailed, textural, artistic",
+#         "negative_prompt": "(lowres, low quality, worst quality:1.2), (text:1.2), watermark, anime, photorealistic, 35mm film, deformed, glitch, low contrast, noisy",
+#     },
+#     {
+#         "name": "Film Noir",
+#         "prompt": "film noir style, ink sketch|vector, {prompt} highly detailed, sharp focus, ultra sharpness, monochrome, high contrast, dramatic shadows, 1940s style, mysterious, cinematic",
+#         "negative_prompt": "(lowres, low quality, worst quality:1.2), (text:1.2), watermark, (frame:1.2), deformed, ugly, deformed eyes, blur, out of focus, blurry, deformed cat, deformed, photo, anthropomorphic cat, monochrome, photo, pet collar, gun, weapon, blue, 3d, drones, drone, buildings in background, green",
+#     },
+#     {
+#         "name": "Neon",
+#         "prompt": "masterpiece painting, buildings in the backdrop, kaleidoscope, lilac orange blue cream fuchsia bright vivid gradient colors, the scene is cinematic, {prompt}, emotional realism, double exposure, watercolor ink pencil, graded wash, color layering, magic realism, figurative painting, intricate motifs, organic tracery, polished",
+#         "negative_prompt": "(lowres, low quality, worst quality:1.2), (text:1.2), watermark, (frame:1.2), deformed, ugly, deformed eyes, blur, out of focus, blurry, deformed cat, deformed, photo, anthropomorphic cat, monochrome, photo, pet collar, gun, weapon, blue, 3d, drones, drone, buildings in background, green",
+#     },
+#     {
+#         "name": "Jungle",
+#         "prompt": 'waist-up "{prompt} in a Jungle" by Syd Mead, tangerine cold color palette, muted colors, detailed, 8k,photo r3al,dripping paint,3d toon style,3d style,Movie Still',
+#         "negative_prompt": "(lowres, low quality, worst quality:1.2), (text:1.2), watermark, (frame:1.2), deformed, ugly, deformed eyes, blur, out of focus, blurry, deformed cat, deformed, photo, anthropomorphic cat, monochrome, photo, pet collar, gun, weapon, blue, 3d, drones, drone, buildings in background, green",
+#     },
+#     {
+#         "name": "Mars",
+#         "prompt": "{prompt}, Post-apocalyptic. Mars Colony, Scavengers roam the wastelands searching for valuable resources, rovers, bright morning sunlight shining, (detailed) (intricate) (8k) (HDR) (cinematic lighting) (sharp focus)",
+#         "negative_prompt": "(lowres, low quality, worst quality:1.2), (text:1.2), watermark, (frame:1.2), deformed, ugly, deformed eyes, blur, out of focus, blurry, deformed cat, deformed, photo, anthropomorphic cat, monochrome, photo, pet collar, gun, weapon, blue, 3d, drones, drone, buildings in background, green",
+#     },
+#     {
+#         "name": "Vibrant Color",
+#         "prompt": "vibrant colorful, ink sketch|vector|2d colors, sharp focus, {prompt}, highly detailed, the clouds,colorful,ultra sharpness,4k",
+#         "negative_prompt": "(lowres, low quality, worst quality:1.2), (text:1.2), watermark, (frame:1.2), deformed, ugly,distorted, deformed eyes, blur, out of focus, blurry, deformed cat, deformed, photo, anthropomorphic cat, monochrome, photo, pet collar, gun, weapon, blue, 3d, drones, drone, buildings in background, green",
+#     },
+#     {
+#         "name": "Snow",
+#         "prompt": "individual dressed as an {prompt}, high contrast, surrounded by snowy mountains and igloo, snow crystals, cold, windy background, frozen natural landscape in background,highly detailed, sharp focus, intricate design, 4k resolution",
+#         "negative_prompt": "(lowres, low quality, worst quality:1.2), (text:1.2), watermark, (frame:1.2), deformed, ugly, deformed eyes, blur, out of focus, blurry, deformed cat, deformed, photo, anthropomorphic cat, monochrome, photo, pet collar, gun, weapon, blue, 3d, drones, drone, buildings in background, green",
+#     },
+#     {
+#         "name": "Line art",
+#         "prompt": "vibrant colorful, sharp focus,individual wearing {prompt} costume, highly detailed, sharp focus, the ocean, fishes swimming in the background,coral reef behind, ocean landscape, 4k, colorful",
+#         "negative_prompt": "(lowres, low quality, worst quality:1.2), (text:1.2), watermark, (frame:1.2), deformed, ugly, deformed eyes, blur, out of focus, blurry, deformed cat, deformed, photo, anthropomorphic cat, monochrome, photo, pet collar, gun, weapon, blue, 3d, drones, drone, buildings in background, green",
+#     },
+# ]
+
+# styles = {k["name"]: (k["prompt"], k["negative_prompt"]) for k in style_list}
+
+style_list = [
+    {
+        "name": "Professional",
+        # "prompt": ["Minimalist style, Simple, clean, uncluttered, modern, elegant, White background, suit and tie, professional look","Minimalist style, Simple, clean, uncluttered, modern, elegant, white background, professional photo, linkedin profile photo, formal attire, professional look","Minimalist style, Simple, clean, uncluttered, modern, elegant, formal attire, sitting on a chair, professional look","Minimalist style, Simple, clean, uncluttered, modern, elegant, formal attire, individual sitiing at a desk, office environment, professional look"],
+        "prompt": ["professional portrait, gender-aligned, natural skin tones, cinematic lighting, highly detailed, well-composed, professional photography, subtle background blur","Minimalist portrait, clean lines, soft colors, simple background, modern, elegant, subtle details, focus on facial features","Professional, Corporate, formal attire, polished, sharp features, clean background, high clarity, refined, business style","LinkedIn professional, business attire, neutral background, sharp focus, approachable, polished, suited for professional networking"],
+        "negative_prompt": "oversaturated, unnatural skin tones, deformed, disfigured, low resolution, cartoonish, unrealistic"
+    },
+    # {
+    #     "name": "Watercolor",
+    #     "prompt": "watercolor painting, {prompt}. vibrant, beautiful, painterly, detailed, textural, artistic",
+    #     "negative_prompt": "(lowres, low quality, worst quality:1.2), (text:1.2), watermark, anime, photorealistic, 35mm film, deformed, glitch, low contrast, noisy",
+    # },
+    # {
+    #     "name": "Film Noir",
+    #     "prompt": "film noir style, ink sketch|vector, {prompt} highly detailed, sharp focus, ultra sharpness, monochrome, high contrast, dramatic shadows, 1940s style, mysterious, cinematic",
+    #     "negative_prompt": "(lowres, low quality, worst quality:1.2), (text:1.2), watermark, (frame:1.2), deformed, ugly, deformed eyes, blur, out of focus, blurry, deformed cat, deformed, photo, anthropomorphic cat, monochrome, photo, pet collar, gun, weapon, blue, 3d, drones, drone, buildings in background, green",
+    # },
+    # {
+    #     "name": "Neon",
+    #     "prompt": "masterpiece painting, buildings in the backdrop, kaleidoscope, lilac orange blue cream fuchsia bright vivid gradient colors, the scene is cinematic, {prompt}, emotional realism, double exposure, watercolor ink pencil, graded wash, color layering, magic realism, figurative painting, intricate motifs, organic tracery, polished",
+    #     "negative_prompt": "(lowres, low quality, worst quality:1.2), (text:1.2), watermark, (frame:1.2), deformed, ugly, deformed eyes, blur, out of focus, blurry, deformed cat, deformed, photo, anthropomorphic cat, monochrome, photo, pet collar, gun, weapon, blue, 3d, drones, drone, buildings in background, green",
+    # },
+    # {
+    #     "name": "Jungle",
+    #     "prompt": 'waist-up "{prompt} in a Jungle" by Syd Mead, tangerine cold color palette, muted colors, detailed, 8k,photo r3al,dripping paint,3d toon style,3d style,Movie Still',
+    #     "negative_prompt": "(lowres, low quality, worst quality:1.2), (text:1.2), watermark, (frame:1.2), deformed, ugly, deformed eyes, blur, out of focus, blurry, deformed cat, deformed, photo, anthropomorphic cat, monochrome, photo, pet collar, gun, weapon, blue, 3d, drones, drone, buildings in background, green",
+    # },
+    # {
+    #     "name": "Mars",
+    #     "prompt": "{prompt}, Post-apocalyptic. Mars Colony, Scavengers roam the wastelands searching for valuable resources, rovers, bright morning sunlight shining, (detailed) (intricate) (8k) (HDR) (cinematic lighting) (sharp focus)",
+    #     "negative_prompt": "(lowres, low quality, worst quality:1.2), (text:1.2), watermark, (frame:1.2), deformed, ugly, deformed eyes, blur, out of focus, blurry, deformed cat, deformed, photo, anthropomorphic cat, monochrome, photo, pet collar, gun, weapon, blue, 3d, drones, drone, buildings in background, green",
+    # },
+    # {
+    #     "name": "Vibrant Color",
+    #     "prompt": "vibrant colorful, ink sketch|vector|2d colors, sharp focus, {prompt}, highly detailed, the clouds,colorful,ultra sharpness,4k",
+    #     "negative_prompt": "(lowres, low quality, worst quality:1.2), (text:1.2), watermark, (frame:1.2), deformed, ugly,distorted, deformed eyes, blur, out of focus, blurry, deformed cat, deformed, photo, anthropomorphic cat, monochrome, photo, pet collar, gun, weapon, blue, 3d, drones, drone, buildings in background, green",
+    # },
+    # {
+    #     "name": "Snow",
+    #     "prompt": "individual dressed as an {prompt}, high contrast, surrounded by snowy mountains and igloo, snow crystals, cold, windy background, frozen natural landscape in background,highly detailed, sharp focus, intricate design, 4k resolution",
+    #     "negative_prompt": "(lowres, low quality, worst quality:1.2), (text:1.2), watermark, (frame:1.2), deformed, ugly, deformed eyes, blur, out of focus, blurry, deformed cat, deformed, photo, anthropomorphic cat, monochrome, photo, pet collar, gun, weapon, blue, 3d, drones, drone, buildings in background, green",
+    # },
+    {
+        "name": "Quirky",
+        "prompt": ["vibrant colorful, ink sketch|vector|2d colors, sharp focus, superman/wonderwoman, highly detailed, the clouds,colorful,ultra sharpness,4k","watercolor painting, japanese anime character with white/neon hair. vibrant, beautiful, painterly, detailed, textural, artistic","vibrant colorful, ink sketch|vector|2d colors, sharp focus, scooba diver, highly detailed, the ocean,fishes,colorful,ultra sharpness,4k","individual dressed as an eskimo, high contrast, surrounded by snowy mountains and igloo, snow crystals, cold, windy background, frozen natural landscape in background,highly detailed, sharp focus, intricate design, 4k resolution"],
+        "negative_prompt": "(lowres, low quality, worst quality:1.2), (text:1.2), watermark, (frame:1.2), deformed, ugly, deformed eyes, blur, out of focus, blurry, deformed cat, deformed, photo, anthropomorphic cat, monochrome, pet collar, gun, weapon, blue, 3d, drones, drone, buildings in background, green"
+    },
+    {
+        "name": "Sci-fi",
+        "prompt": ["ethereal fantasy concept art individual, magnificent, celestial, ethereal, painterly, epic, majestic, magical, fantasy art, cover art, dreamy","Dystopian style cyborg. Bleak, post-apocalyptic, somber, dramatic, highly detailed","Alien-themed,  Extraterrestrial, cosmic, otherworldly, mysterious, sci-fi, highly detailed", "Legend of Zelda style . Vibrant, fantasy, detailed, epic, heroic, reminiscent of The Legend of Zelda series"],
+        "negative_prompt": "photographic, realistic, realism, 35mm film, dslr, cropped, frame, text, deformed, glitch, noise, noisy, off-center, deformed, cross-eyed, closed eyes, bad anatomy, ugly, disfigured, sloppy, duplicate, mutated, black and white, multiple people",
+    }
+]
+
+styles = {k["name"]: (k["prompt"], k["negative_prompt"]) for k in style_list}
+
+# lowres

gradio_demo/test.py

@@ -0,0 +1,400 @@
+import sys
+sys.path.append('./')
+
+from typing import Tuple
+
+import os
+import cv2
+import math
+import torch
+import random
+import numpy as np
+import argparse
+import pandas as pd
+
+import PIL
+from PIL import Image
+
+import diffusers
+from diffusers.utils import load_image
+from diffusers.models import ControlNetModel
+from diffusers import LCMScheduler
+
+from huggingface_hub import hf_hub_download
+
+import insightface
+from insightface.app import FaceAnalysis
+
+from style_template import styles
+from pipeline_stable_diffusion_xl_instantid_full import StableDiffusionXLInstantIDPipeline
+from model_util import load_models_xl, get_torch_device, torch_gc
+
+
+# global variable
+MAX_SEED = np.iinfo(np.int32).max
+device = get_torch_device()
+dtype = torch.float16 if str(device).__contains__("cuda") else torch.float32
+STYLE_NAMES = list(styles.keys())
+DEFAULT_STYLE_NAME = "Watercolor"
+
+# Load face encoder
+app = FaceAnalysis(name='antelopev2', root='./', providers=['CUDAExecutionProvider', 'CPUExecutionProvider'])
+app.prepare(ctx_id=0, det_size=(320, 320))
+
+# Path to InstantID models
+face_adapter = f'./checkpoints/ip-adapter.bin'
+controlnet_path = f'./checkpoints/ControlNetModel'
+
+# Load pipeline
+controlnet = ControlNetModel.from_pretrained(controlnet_path, torch_dtype=dtype)
+
+logo = Image.open("./gradio_demo/logo.png")
+
+from cv2 import imencode
+import base64
+
+# def encode_pil_to_base64_new(pil_image):
+#     print("AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA")
+#     image_arr = np.asarray(pil_image)[:,:,::-1]
+#     _, byte_data = imencode('.png', image_arr)        
+#     base64_data = base64.b64encode(byte_data)
+#     base64_string_opencv = base64_data.decode("utf-8")
+#     return "data:image/png;base64," + base64_string_opencv
+
+import gradio as gr
+
+# gr.processing_utils.encode_pil_to_base64 = encode_pil_to_base64_new
+
+def main(pretrained_model_name_or_path="wangqixun/YamerMIX_v8", enable_lcm_arg=False):
+
+    if pretrained_model_name_or_path.endswith(
+            ".ckpt"
+        ) or pretrained_model_name_or_path.endswith(".safetensors"):
+            scheduler_kwargs = hf_hub_download(
+                repo_id="wangqixun/YamerMIX_v8",
+                subfolder="scheduler",
+                filename="scheduler_config.json",
+            )
+
+            (tokenizers, text_encoders, unet, _, vae) = load_models_xl(
+                pretrained_model_name_or_path=pretrained_model_name_or_path,
+                scheduler_name=None,
+                weight_dtype=dtype,
+            )
+
+            scheduler = diffusers.EulerDiscreteScheduler.from_config(scheduler_kwargs)
+            pipe = StableDiffusionXLInstantIDPipeline(
+                vae=vae,
+                text_encoder=text_encoders[0],
+                text_encoder_2=text_encoders[1],
+                tokenizer=tokenizers[0],
+                tokenizer_2=tokenizers[1],
+                unet=unet,
+                scheduler=scheduler,
+                controlnet=controlnet,
+            ).to(device)
+
+    else:
+        pipe = StableDiffusionXLInstantIDPipeline.from_pretrained(
+            pretrained_model_name_or_path,
+            controlnet=controlnet,
+            torch_dtype=dtype,
+            safety_checker=None,
+            feature_extractor=None,
+        ).to(device)
+
+        pipe.scheduler = diffusers.EulerDiscreteScheduler.from_config(pipe.scheduler.config)
+
+    pipe.load_ip_adapter_instantid(face_adapter)
+    # load and disable LCM
+    pipe.load_lora_weights("latent-consistency/lcm-lora-sdxl")
+    pipe.disable_lora()
+    
+    def remove_tips():
+        return gr.update(visible=False)
+
+    
+    # prompts = [
+    #     ["superman","Vibrant Color"], ["japanese anime character with white/neon hair","Watercolor"], 
+    #     # ["Suited professional","(No style)"], 
+    #     ["Scooba diver","Line art"], ["eskimo","Snow"]
+    # ]
+
+    def convert_from_cv2_to_image(img: np.ndarray) -> Image:
+        return Image.fromarray(cv2.cvtColor(img, cv2.COLOR_BGR2RGB))
+
+    def convert_from_image_to_cv2(img: Image) -> np.ndarray:
+        return cv2.cvtColor(np.array(img), cv2.COLOR_RGB2BGR)
+
+    def run_for_prompts1(face_file,style,progress=gr.Progress(track_tqdm=True)):
+        # if email != "":
+        p,n = styles.get(style, styles.get(STYLE_NAMES[1]))
+        return generate_image(face_file, p[0], n)
+        # else:
+            # raise gr.Error("Email ID is compulsory")
+    def run_for_prompts2(face_file,style,progress=gr.Progress(track_tqdm=True)):
+        # if email != "":
+        p,n = styles.get(style, styles.get(STYLE_NAMES[1]))
+        return generate_image(face_file, p[1], n)
+    def run_for_prompts3(face_file,style,progress=gr.Progress(track_tqdm=True)):
+        # if email != "":
+        p,n = styles.get(style, styles.get(STYLE_NAMES[1]))
+        return generate_image(face_file, p[2], n)
+    def run_for_prompts4(face_file,style,progress=gr.Progress(track_tqdm=True)):
+        # if email != "":
+        p,n = styles.get(style, styles.get(STYLE_NAMES[1]))
+        return generate_image(face_file, p[3], n)
+    
+#     def validate_and_process(face_file, style, email):
+    
+#     # Your processing logic here
+#     gallery1, gallery2, gallery3, gallery4 = run_for_prompts1(face_file, style), run_for_prompts2(face_file, style), run_for_prompts3(face_file, style), run_for_prompts4(face_file, style)
+#     return gallery1, gallery2, gallery3, gallery4
+
+    def draw_kps(image_pil, kps, color_list=[(255,0,0), (0,255,0), (0,0,255), (255,255,0), (255,0,255)]):
+        stickwidth = 4
+        limbSeq = np.array([[0, 2], [1, 2], [3, 2], [4, 2]])
+        kps = np.array(kps)
+
+        w, h = image_pil.size
+        out_img = np.zeros([h, w, 3])
+
+        for i in range(len(limbSeq)):
+            index = limbSeq[i]
+            color = color_list[index[0]]
+
+            x = kps[index][:, 0]
+            y = kps[index][:, 1]
+            length = ((x[0] - x[1]) ** 2 + (y[0] - y[1]) ** 2) ** 0.5
+            angle = math.degrees(math.atan2(y[0] - y[1], x[0] - x[1]))
+            polygon = cv2.ellipse2Poly((int(np.mean(x)), int(np.mean(y))), (int(length / 2), stickwidth), int(angle), 0, 360, 1)
+            out_img = cv2.fillConvexPoly(out_img.copy(), polygon, color)
+        out_img = (out_img * 0.6).astype(np.uint8)
+
+        for idx_kp, kp in enumerate(kps):
+            color = color_list[idx_kp]
+            x, y = kp
+            out_img = cv2.circle(out_img.copy(), (int(x), int(y)), 10, color, -1)
+
+        out_img_pil = Image.fromarray(out_img.astype(np.uint8))
+        return out_img_pil
+
+    def resize_img(input_image, max_side=640, min_side=640, size=None, 
+                pad_to_max_side=True, mode=PIL.Image.BILINEAR, base_pixel_number=64):
+
+            w, h = input_image.size
+            print(w)
+            print(h)
+            if size is not None:
+                w_resize_new, h_resize_new = size
+            else:
+                ratio = min_side / min(h, w)
+                w, h = round(ratio*w), round(ratio*h)
+                ratio = max_side / max(h, w)
+                input_image = input_image.resize([round(ratio*w), round(ratio*h)], mode)
+                w_resize_new = (round(ratio * w) // base_pixel_number) * base_pixel_number
+                h_resize_new = (round(ratio * h) // base_pixel_number) * base_pixel_number
+            input_image = input_image.resize([w_resize_new, h_resize_new], mode)
+
+            if pad_to_max_side:
+                res = np.ones([max_side, max_side, 3], dtype=np.uint8) * 255
+                offset_x = (max_side - w_resize_new) // 2
+                offset_y = (max_side - h_resize_new) // 2
+                res[offset_y:offset_y+h_resize_new, offset_x:offset_x+w_resize_new] = np.array(input_image)
+                input_image = Image.fromarray(res)
+            return input_image
+
+    def store_images(email, gallery1, gallery2, gallery3, gallery4):
+        galleries = []
+        for i, img in enumerate([gallery1, gallery2, gallery3, gallery4], start=1):
+            if isinstance(img, np.ndarray):
+                img = Image.fromarray(img)
+            print(f"Gallery {i} type after conversion: {type(img)}")
+            galleries.append(img)
+        # Create the images directory if it doesn't exist
+        if not os.path.exists('images'):
+            os.makedirs('images')
+
+        # Define image file paths
+        image_paths = []
+        for i, img in enumerate(galleries, start=1):
+            img_path = f'images/{email}_gallery{i}.png'
+            img.save(img_path)
+            image_paths.append(img_path)
+
+        # Define the CSV file path
+        csv_file_path = 'image_data.csv'
+
+        # Create a DataFrame for the email and image paths
+        df = pd.DataFrame({
+            'email': [email],
+            'img1_path': [image_paths[0]],
+            'img2_path': [image_paths[1]],
+            'img3_path': [image_paths[2]],
+            'img4_path': [image_paths[3]],
+        })
+
+        # Write to CSV (append if the file exists, create a new one if it doesn't)
+        if not os.path.isfile(csv_file_path):
+            df.to_csv(csv_file_path, index=False)
+        else:
+            df.to_csv(csv_file_path, mode='a', header=False, index=False)
+
+
+    def generate_image(face_image,prompt,negative_prompt):
+        pose_image_path = None
+        # prompt = "superman"
+        enable_LCM = False
+        identitynet_strength_ratio = 0.95
+        adapter_strength_ratio = 0.60
+        num_steps = 15
+        guidance_scale = 8.5
+        seed = random.randint(0, MAX_SEED)
+        # negative_prompt = ""
+        # negative_prompt += neg
+        enhance_face_region = True
+        if enable_LCM:
+            pipe.enable_lora()
+            pipe.scheduler = LCMScheduler.from_config(pipe.scheduler.config)
+        else:
+            pipe.disable_lora()
+            pipe.scheduler = diffusers.EulerDiscreteScheduler.from_config(pipe.scheduler.config)
+    
+        if face_image is None:
+            raise gr.Error(f"Cannot find any input face image! Please upload the face image")
+        
+        # if prompt is None:
+        #     prompt = "a person"
+        
+        # apply the style template
+        # prompt, negative_prompt = apply_style(style_name, prompt, negative_prompt)
+        
+        # face_image = load_image(face_image_path)
+        face_image = resize_img(face_image)
+        face_image_cv2 = convert_from_image_to_cv2(face_image)
+        height, width, _ = face_image_cv2.shape
+        
+        # Extract face features
+        face_info = app.get(face_image_cv2)
+        
+        if len(face_info) == 0:
+            raise gr.Error(f"Cannot find any face in the image! Please upload another person image")
+        
+        face_info = sorted(face_info, key=lambda x:(x['bbox'][2]-x['bbox'][0])*(x['bbox'][3]-x['bbox'][1]))[-1]  # only use the maximum face
+        face_emb = face_info['embedding']
+        face_kps = draw_kps(convert_from_cv2_to_image(face_image_cv2), face_info['kps'])
+        
+        if pose_image_path is not None:
+            pose_image = load_image(pose_image_path)
+            pose_image = resize_img(pose_image)
+            pose_image_cv2 = convert_from_image_to_cv2(pose_image)
+            
+            face_info = app.get(pose_image_cv2)
+            
+            if len(face_info) == 0:
+                raise gr.Error(f"Cannot find any face in the reference image! Please upload another person image")
+            
+            face_info = face_info[-1]
+            face_kps = draw_kps(pose_image, face_info['kps'])
+            
+            width, height = face_kps.size
+
+        if enhance_face_region:
+            control_mask = np.zeros([height, width, 3])
+            x1, y1, x2, y2 = face_info["bbox"]
+            x1, y1, x2, y2 = int(x1), int(y1), int(x2), int(y2)
+            control_mask[y1:y2, x1:x2] = 255
+            control_mask = Image.fromarray(control_mask.astype(np.uint8))
+        else:
+            control_mask = None
+                        
+        generator = torch.Generator(device=device).manual_seed(seed)
+        
+        print("Start inference...")
+        print(f"[Debug] Prompt: {prompt}, \n[Debug] Neg Prompt: {negative_prompt}")
+        
+        pipe.set_ip_adapter_scale(adapter_strength_ratio)
+        images = pipe(
+            prompt=prompt,
+            negative_prompt=negative_prompt,
+            image_embeds=face_emb,
+            image=face_kps,
+            control_mask=control_mask,
+            controlnet_conditioning_scale=float(identitynet_strength_ratio),
+            num_inference_steps=num_steps,
+            guidance_scale=guidance_scale,
+            height=height,
+            width=width,
+            generator=generator,
+            # num_images_per_prompt = 4
+        ).images
+        
+        print(images[0])
+
+        return images[0]
+    
+    
+    ### Description
+    title = r"""
+    <h1 align="center">Choose your AVATAR</h1>
+    """
+
+    description = r"""
+    <h2> Powered by IDfy </h2>"""
+
+    article = r""""""
+
+    tips = r""""""
+    
+    css = '''
+    .gradio-container {width: 95% !important; background-color: #E6F3FF;} 
+    .image-gallery {height: 100vh !important; overflow: auto;}
+    .gradio-row .gradio-element { margin: 0 !important; }
+    '''
+    with gr.Blocks(css=css) as demo:
+        title = "<h1 align='center'>Choose your AVATAR</h1>"
+        description = "<h2> Powered by IDfy </h2>"
+        
+        # Description
+        gr.Markdown(title)
+        with gr.Row():
+            gr.Image("./gradio_demo/logo.png",scale=0,min_width=50,show_label=False,show_download_button=False)
+            gr.Markdown(description)
+        with gr.Row():
+            with gr.Column():
+                style = gr.Dropdown(label="Choose your STYLE", choices=STYLE_NAMES)
+                face_file = gr.Image(label="Upload a photo of your face", type="pil")
+                submit = gr.Button("Submit", variant="primary")
+            with gr.Column():
+                with gr.Row():
+                    gallery1 = gr.Image(label="Generated Images")
+                    gallery2 = gr.Image(label="Generated Images")
+                with gr.Row():
+                    gallery3 = gr.Image(label="Generated Images")
+                    gallery4 = gr.Image(label="Generated Images")
+                email = gr.Textbox(label="Email",
+                        info="Enter your email address",
+                        value="")
+                submit1 = gr.Button("STORE", variant="primary")
+            usage_tips = gr.Markdown(label="Usage tips of InstantID", value="", visible=False)
+
+            # Image upload and processing chain
+            face_file.upload(remove_tips, outputs=usage_tips).then(run_for_prompts1, inputs=[face_file, style], outputs=[gallery1]).then(run_for_prompts2, inputs=[face_file, style], outputs=[gallery2]).then(run_for_prompts3, inputs=[face_file, style], outputs=[gallery3]).then(run_for_prompts4, inputs=[face_file, style], outputs=[gallery4])
+            submit.click(remove_tips, outputs=usage_tips).then(run_for_prompts1, inputs=[face_file, style], outputs=[gallery1]).then(run_for_prompts2, inputs=[face_file, style], outputs=[gallery2]).then(run_for_prompts3, inputs=[face_file, style], outputs=[gallery3]).then(run_for_prompts4, inputs=[face_file, style], outputs=[gallery4])
+
+            # Store data on button click
+            submit1.click(
+                fn=store_images,
+                inputs=[email,gallery1,gallery2,gallery3,gallery4],
+                outputs=None)
+
+        gr.Markdown("")
+
+    demo.launch(share=True)
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--pretrained_model_name_or_path", type=str, default="wangqixun/YamerMIX_v8")
+    args = parser.parse_args()
+
+    main(args.pretrained_model_name_or_path, False)

image_data.csv

@@ -0,0 +1,11 @@
+email,img1_path,img2_path,img3_path,img4_path
+ll,images/ll_gallery1.png,images/ll_gallery2.png,images/ll_gallery3.png,images/ll_gallery4.png
+kajal@img,images/kajal@img_gallery1.png,images/kajal@img_gallery2.png,images/kajal@img_gallery3.png,images/kajal@img_gallery4.png
+heeral@img,images/heeral@img_gallery1.png,images/heeral@img_gallery2.png,images/heeral@img_gallery3.png,images/heeral@img_gallery4.png
+sanskruti@img-scifi,images/sanskruti@img-scifi_gallery1.png,images/sanskruti@img-scifi_gallery2.png,images/sanskruti@img-scifi_gallery3.png,images/sanskruti@img-scifi_gallery4.png
+sanskruti@img-scifi,images/sanskruti@img-scifi_gallery1.png,images/sanskruti@img-scifi_gallery2.png,images/sanskruti@img-scifi_gallery3.png,images/sanskruti@img-scifi_gallery4.png
+sanskruti@img-quirky,images/sanskruti@img-quirky_gallery1.png,images/sanskruti@img-quirky_gallery2.png,images/sanskruti@img-quirky_gallery3.png,images/sanskruti@img-quirky_gallery4.png
+kajal@quirky,images/kajal@quirky_gallery1.png,images/kajal@quirky_gallery2.png,images/kajal@quirky_gallery3.png,images/kajal@quirky_gallery4.png
+kajal@prof,images/kajal@prof_gallery1.png,images/kajal@prof_gallery2.png,images/kajal@prof_gallery3.png,images/kajal@prof_gallery4.png
+kajal@quirky,images/kajal@quirky_gallery1.png,images/kajal@quirky_gallery2.png,images/kajal@quirky_gallery3.png,images/kajal@quirky_gallery4.png
+kajal@sci-fi,images/kajal@sci-fi_gallery1.png,images/kajal@sci-fi_gallery2.png,images/kajal@sci-fi_gallery3.png,images/kajal@sci-fi_gallery4.png