Upload 3 files

Multicontrolnet.ipynb

@@ -0,0 +1,659 @@
+{
+  "nbformat": 4,
+  "nbformat_minor": 0,
+  "metadata": {
+    "colab": {
+      "provenance": [],
+      "gpuType": "T4"
+    },
+    "kernelspec": {
+      "name": "python3",
+      "display_name": "Python 3"
+    },
+    "language_info": {
+      "name": "python"
+    },
+    "accelerator": "GPU"
+  },
+  "cells": [
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "cellView": "form",
+        "id": "GjF0Vw1G3CsS"
+      },
+      "outputs": [],
+      "source": [
+        "# @title Install requirements\n",
+        "import os\n",
+        "#!git clone https://github.com/huggingface/diffusers\n",
+        "!git clone https://huggingface.co/waveydaveygravy/controlnetexpts\n",
+        "!pip install controlnet-aux==0.0.7\n",
+        "#!pip install -U openmim\n",
+        "#!pip install cog\n",
+        "#!pip install mediapipe\n",
+        "#!mim install mmengine\n",
+        "#!mim install \"mmcv>=2.0.1\"\n",
+        "#!mim install \"mmdet>=3.1.0\"\n",
+        "#!mim install \"mmpose>=1.1.0\"\n",
+        "!pip install diffusers\n",
+        "!pip install moviepy\n",
+        "!pip install argparse\n",
+        "!pip install transformers\n",
+        "!pip install pillow\n",
+        "!pip install accelerate\n",
+        "!pip install xformers\n",
+        "#!pip install https://github.com/karaokenerds/python-audio-separator/releases/download/v0.12.1/onnxruntime_gpu-1.17.0-cp310-cp310-linux_x86_64.whl\n",
+        "#!git clone https://github.com/danielgatis/rembg.git\n",
+        "#!git clone https://huggingface.co/spaces/LiheYoung/Depth-Anything\n",
+        "\n",
+        "# Create the directory /content/test\n",
+        "import os\n",
+        "os.makedirs(\"/content/test\", exist_ok=True)\n",
+        "os.makedirs(\"/content/frames\", exist_ok=True)\n",
+        "os.makedirs(\"/content/op\", exist_ok=True)\n",
+        "os.makedirs(\"/content/dp\", exist_ok=True)\n",
+        "os.makedirs(\"/content/checkpoints\")\n",
+        "os.makedirs(\"/content/checkpoints/openpose\")\n",
+        "os.makedirs(\"/content/checkpoints/depth\")\n",
+        "os.makedirs(\"/content/checkpoints/realisticvision\")\n",
+        "#INPUT_DIR = \"/content/frames\"  # replace with your input directory\n",
+        "#OUTPUT_DIR = \"/content/test\"  # replace with your output directory"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "source": [
+        "#@title upload\n",
+        "from google.colab import files\n",
+        "uploaded = files.upload()"
+      ],
+      "metadata": {
+        "cellView": "form",
+        "id": "uLaX0p173O-A"
+      },
+      "execution_count": null,
+      "outputs": []
+    },
+    {
+      "cell_type": "code",
+      "source": [
+        "# @title break video down into frames\n",
+        "import cv2\n",
+        "import os\n",
+        "\n",
+        "# Create the directory /content/test\n",
+        "os.makedirs(\"/content/test\", exist_ok=True)\n",
+        "os.makedirs(\"/content/frames\", exist_ok=True)\n",
+        "\n",
+        "#INPUT_DIR = \"/content/frames\"  # replace with your input directory\n",
+        "#OUTPUT_DIR = \"/content/testIP\"  # replace with your output directory\n",
+        "\n",
+        "\n",
+        "# Open the video file\n",
+        "cap = cv2.VideoCapture('/content/trumpoverlay_1.mp4')\n",
+        "\n",
+        "i = 0\n",
+        "while(cap.isOpened()):\n",
+        "    ret, frame = cap.read()\n",
+        "\n",
+        "    if ret == False:\n",
+        "        break\n",
+        "\n",
+        "    # Save each frame of the video\n",
+        "    cv2.imwrite('/content/testIP/frame_' + str(i) + '.jpg', frame)\n",
+        "\n",
+        "    i += 1\n",
+        "\n",
+        "cap.release()\n",
+        "cv2.destroyAllWindows()"
+      ],
+      "metadata": {
+        "cellView": "form",
+        "id": "LqdApe0Y3VtH"
+      },
+      "execution_count": null,
+      "outputs": []
+    },
+    {
+      "cell_type": "code",
+      "source": [
+        "# @title COMMENT OUT PROCESSORS YOU DONT WANT TO USE ALSO COMMENT OUT ONES WITH LARGE MODELS IF YOU WANT TO SAVE SPACE\n",
+        "### based on https://github.com/patrickvonplaten/controlnet_aux\n",
+        "### which is derived from https://github.com/lllyasviel/ControlNet/tree/main/annotator and connected to the 🤗 Hub.\n",
+        "#All credit & copyright goes to https://github.com/lllyasviel .\n",
+        "#some of the models are large comment them out to save space if not needed\n",
+        "\n",
+        "import torch\n",
+        "import os\n",
+        "import shutil\n",
+        "import logging\n",
+        "import math\n",
+        "import numpy as np\n",
+        "from PIL import Image\n",
+        "from tqdm.auto import tqdm\n",
+        "from PIL import Image\n",
+        "from tqdm import tqdm\n",
+        "import matplotlib.pyplot as plt\n",
+        "import matplotlib.image as mpimg\n",
+        "from transformers import AutoModel\n",
+        "from diffusers import DiffusionPipeline\n",
+        "#from depth_anything.dpt import DepthAnything\n",
+        "#from depth_anything.util.transform import Resize, NormalizeImage, PrepareForNet\n",
+        "from controlnet_aux import (CannyDetector, ContentShuffleDetector, HEDdetector,\n",
+        "                            LeresDetector, LineartAnimeDetector,\n",
+        "                            LineartDetector, MediapipeFaceDetector,\n",
+        "                            MidasDetector, MLSDdetector, NormalBaeDetector,\n",
+        "                            OpenposeDetector, PidiNetDetector, SamDetector,\n",
+        "                            ZoeDetector, DWposeDetector)\n",
+        "\n",
+        "# Create the directory /content/test\n",
+        "os.makedirs(\"/content/test\", exist_ok=True)\n",
+        "os.makedirs(\"/content/frames\", exist_ok=True)\n",
+        "\n",
+        "INPUT_DIR = \"/content/frames\"  # replace with your input directory\n",
+        "OUTPUT_DIR = \"/content/test\"  # replace with your output directory\n",
+        "\n",
+        "#controlnet_model_path = \"/content/checkpoints\"\n",
+        "#controlnet = ControlNetModel.from_pretrained(controlnet_model_path, torch_dtype=torch.float16)\n",
+        "\n",
+        "# Check if CUDA is available and set the device accordingly\n",
+        "device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n",
+        "\n",
+        "\n",
+        "def output(filename, img):\n",
+        "    img.save(os.path.join(OUTPUT_DIR, filename))\n",
+        "\n",
+        "def process_image(processor, img):\n",
+        "    return processor(img)\n",
+        "\n",
+        "def load_images():\n",
+        "    if os.path.exists(OUTPUT_DIR):\n",
+        "        shutil.rmtree(OUTPUT_DIR)\n",
+        "    os.mkdir(OUTPUT_DIR)\n",
+        "    images = []\n",
+        "    filenames = []\n",
+        "    for filename in os.listdir(INPUT_DIR):\n",
+        "        if filename.endswith(\".png\") or filename.endswith(\".jpg\"):\n",
+        "            img_path = os.path.join(INPUT_DIR, filename)\n",
+        "            img = Image.open(img_path).convert(\"RGB\").resize((512, 512))\n",
+        "            images.append(img)\n",
+        "            filenames.append(filename)\n",
+        "    return images, filenames\n",
+        "\n",
+        "def process_images(processor):\n",
+        "    images, filenames = load_images()\n",
+        "    for img, filename in tqdm(zip(images, filenames), total=len(images), desc=\"Processing images\"):\n",
+        "        output_img = process_image(processor, img)\n",
+        "        output(filename, output_img)\n",
+        "\n",
+        "# Initialize the detectors\n",
+        "\n",
+        "#canny = CannyDetector()\n",
+        "#hed = HEDdetector.from_pretrained(\"lllyasviel/Annotators\")\n",
+        "#shuffle = ContentShuffleDetector()\n",
+        "leres = LeresDetector.from_pretrained(\"lllyasviel/Annotators\")\n",
+        "#lineart_anime = LineartAnimeDetector.from_pretrained(\"lllyasviel/Annotators\")\n",
+        "#lineart = LineartDetector.from_pretrained(\"lllyasviel/Annotators\")\n",
+        "#mediapipe_face = MediapipeFaceDetector()\n",
+        "#midas = MidasDetector.from_pretrained(\"lllyasviel/Annotators\").to('cuda')\n",
+        "#mlsd = MLSDdetector.from_pretrained(\"lllyasviel/Annotators\")\n",
+        "#normal_bae = NormalBaeDetector.from_pretrained(\"lllyasviel/Annotators\")\n",
+        "openpose = OpenposeDetector.from_pretrained(\"lllyasviel/Annotators\")\n",
+        "#pidi_net = PidiNetDetector.from_pretrained(\"lllyasviel/Annotators\")\n",
+        "#sam = SamDetector.from_pretrained(\"ybelkada/segment-anything\", subfolder=\"checkpoints\")\n",
+        "#zoe = ZoeDetector\n",
+        "#depth_anything = AutoModel.from_pretrained(\"waveydaveygravy/depth-anything_pruned\")\n",
+        "\n",
+        "\n",
+        "\n",
+        "# Run the image processing\n",
+        "# Uncomment the line for the detector you want to use\n",
+        "#process_images(canny)\n",
+        "#process_images(hed)\n",
+        "process_images(openpose)\n",
+        "#process_images(midas)\n",
+        "process_images(leres)"
+      ],
+      "metadata": {
+        "cellView": "form",
+        "id": "ExeWABuE3vjo"
+      },
+      "execution_count": null,
+      "outputs": []
+    },
+    {
+      "cell_type": "code",
+      "source": [
+        "#@title zip frames\n",
+        "!zip -r (name) path"
+      ],
+      "metadata": {
+        "cellView": "form",
+        "id": "hJzsEihl4BVx"
+      },
+      "execution_count": null,
+      "outputs": []
+    },
+    {
+      "cell_type": "code",
+      "source": [
+        "#@title download models\n",
+        "#%cd /content/checkpoints\n",
+        "#!wget https://huggingface.co/lllyasviel/control_v11p_sd15_openpose/resolve/main/diffusion_pytorch_model.fp16.bin?download=true\n",
+        "#!wget https://huggingface.co/lllyasviel/control_v11f1p_sd15_depth/resolve/main/diffusion_pytorch_model.fp16.bin?download=true\n",
+        "%cd /content/checkpoints/realisticvision\n",
+        "!wget https://huggingface.co/SG161222/Realistic_Vision_V4.0_noVAE/resolve/main/Realistic_Vision_V4.0_fp16-no-ema-inpainting.ckpt?download=true\n",
+        "%cd /content/"
+      ],
+      "metadata": {
+        "cellView": "form",
+        "id": "evRfuqs74HqW"
+      },
+      "execution_count": null,
+      "outputs": []
+    },
+    {
+      "cell_type": "code",
+      "source": [
+        "#@title config file for openpose\n",
+        "{\n",
+        "  \"_class_name\": \"ControlNetModel\",\n",
+        "  \"_diffusers_version\": \"0.16.0.dev0\",\n",
+        "  \"_name_or_path\": \"/home/patrick/controlnet_v1_1/control_v11p_sd15_openpose\",\n",
+        "  \"act_fn\": \"silu\",\n",
+        "  \"attention_head_dim\": 8,\n",
+        "  \"block_out_channels\": [\n",
+        "    320,\n",
+        "    640,\n",
+        "    1280,\n",
+        "    1280\n",
+        "  ],\n",
+        "  \"class_embed_type\": null,\n",
+        "  \"conditioning_embedding_out_channels\": [\n",
+        "    16,\n",
+        "    32,\n",
+        "    96,\n",
+        "    256\n",
+        "  ],\n",
+        "  \"controlnet_conditioning_channel_order\": \"rgb\",\n",
+        "  \"cross_attention_dim\": 768,\n",
+        "  \"down_block_types\": [\n",
+        "    \"CrossAttnDownBlock2D\",\n",
+        "    \"CrossAttnDownBlock2D\",\n",
+        "    \"CrossAttnDownBlock2D\",\n",
+        "    \"DownBlock2D\"\n",
+        "  ],\n",
+        "  \"downsample_padding\": 1,\n",
+        "  \"flip_sin_to_cos\": true,\n",
+        "  \"freq_shift\": 0,\n",
+        "  \"in_channels\": 4,\n",
+        "  \"layers_per_block\": 2,\n",
+        "  \"mid_block_scale_factor\": 1,\n",
+        "  \"norm_eps\": 1e-05,\n",
+        "  \"norm_num_groups\": 32,\n",
+        "  \"num_class_embeds\": null,\n",
+        "  \"only_cross_attention\": false,\n",
+        "  \"projection_class_embeddings_input_dim\": null,\n",
+        "  \"resnet_time_scale_shift\": \"default\",\n",
+        "  \"upcast_attention\": false,\n",
+        "  \"use_linear_projection\": false\n",
+        "}"
+      ],
+      "metadata": {
+        "cellView": "form",
+        "id": "ZpkAQLlO4RVf"
+      },
+      "execution_count": null,
+      "outputs": []
+    },
+    {
+      "cell_type": "code",
+      "source": [
+        "#@title WORKING MUTICONTROLNET SCRIPT DO NOT CHANGE! save as multi1.py and use as shown in next cell\n",
+        "\n",
+        "from diffusers import StableDiffusionControlNetPipeline, ControlNetModel, UniPCMultistepScheduler\n",
+        "import torch\n",
+        "import os\n",
+        "from diffusers import StableDiffusionControlNetPipeline, ControlNetModel, UniPCMultistepScheduler\n",
+        "from diffusers.utils import load_image\n",
+        "import argparse\n",
+        "from PIL import Image\n",
+        "import cv2\n",
+        "import numpy as np\n",
+        "import torch\n",
+        "import os\n",
+        "import shutil\n",
+        "from tqdm import tqdm\n",
+        "import numpy as np\n",
+        "\n",
+        "device = (\"cuda\")\n",
+        "\n",
+        "# Initialize the argument parser\n",
+        "parser = argparse.ArgumentParser(description='Choose a processor to run.')\n",
+        "parser.add_argument('--op_image', type=str, help='path to pose image')\n",
+        "parser.add_argument('--dp_image', type=str, help='path to depth image')\n",
+        "parser.add_argument('--output_dir', type=str, default='/content/multi', help='The directory to save the output.')\n",
+        "# Parse the arguments\n",
+        "args = parser.parse_args()\n",
+        "\n",
+        "op_image = load_image(args.op_image)\n",
+        "dp_image = load_image(args.dp_image)\n",
+        "\n",
+        "controlnet = [\n",
+        "    ControlNetModel.from_pretrained(\"/content/checkpoints/openpose\", torch_dtype=torch.float16).to('cuda'),\n",
+        "    ControlNetModel.from_pretrained(\"/content/checkpoints/depth\", torch_dtype=torch.float16).to('cuda'),\n",
+        "]\n",
+        "\n",
+        "pipe = StableDiffusionControlNetPipeline.from_pretrained(\n",
+        "    \"SG161222/Realistic_Vision_V4.0_noVAE\", controlnet=controlnet, torch_dtype=torch.float16\n",
+        ").to('cuda')\n",
+        "\n",
+        "pipe.scheduler = UniPCMultistepScheduler.from_config(pipe.scheduler.config)\n",
+        "\n",
+        "prompt = \"a boxer in a boxing ring, best quality\"\n",
+        "negative_prompt = \"monochrome, lowres, bad anatomy, worst quality, low quality\"\n",
+        "\n",
+        "images = [op_image, dp_image]\n",
+        "\n",
+        "image = pipe(\n",
+        "    prompt,\n",
+        "    images,\n",
+        "    num_inference_steps=20,\n",
+        "    negative_prompt=negative_prompt,\n",
+        "    controlnet_conditioning_scale=[1.0, 0.8],\n",
+        ").images[0]\n",
+        "\n",
+        "# Extract the filename and extension from args.op_image\n",
+        "filename, extension = os.path.splitext(os.path.basename(args.op_image))\n",
+        "\n",
+        "# Construct the full output path with the specified output directory\n",
+        "output_path = os.path.join(args.output_dir, filename + extension)\n",
+        "\n",
+        "print(type(image))\n",
+        "# Save the image using PIL\n",
+        "image.save(output_path)  # Assuming image is from PIL\n",
+        "print(\"saved in output directory!\")\n"
+      ],
+      "metadata": {
+        "cellView": "form",
+        "id": "55GnGKo74VLg"
+      },
+      "execution_count": null,
+      "outputs": []
+    },
+    {
+      "cell_type": "code",
+      "source": [
+        "#@title list of commands for multi1.py change to whichever paths needed\n",
+        "#!python /content/multi1.py --op_image \"/content/op/test/frame_0.jpg\" --dp_image \"/content/dp/test/frame_0.jpg\"\n",
+        "\n",
+        "#!python /content/multi1.py --op_image \"/content/op/test/frame_1.jpg\" --dp_image \"/content/dp/test/frame_1.jpg\"\n",
+        "\n",
+        "#!python /content/multi1.py --op_image \"/content/op/test/frame_2.jpg\" --dp_image \"/content/dp/test/frame_2.jpg\"\n",
+        "\n",
+        "!python /content/multi1.py --op_image \"/content/op/test/frame_3.jpg\" --dp_image \"/content/dp/test/frame_3.jpg\"\n",
+        "\n",
+        "!python /content/multi1.py --op_image \"/content/op/test/frame_4.jpg\" --dp_image \"/content/dp/test/frame_4.jpg\"\n",
+        "\n",
+        "!python /content/multi1.py --op_image \"/content/op/test/frame_5.jpg\" --dp_image \"/content/dp/test/frame_5.jpg\"\n",
+        "\n",
+        "!python /content/multi1.py --op_image \"/content/op/test/frame_6.jpg\" --dp_image \"/content/dp/test/frame_6.jpg\"\n",
+        "\n",
+        "!python /content/multi1.py --op_image \"/content/op/test/frame_7.jpg\" --dp_image \"/content/dp/test/frame_7.jpg\"\n",
+        "\n",
+        "!python /content/multi1.py --op_image \"/content/op/test/frame_8.jpg\" --dp_image \"/content/dp/test/frame_8.jpg\"\n",
+        "\n",
+        "!python /content/multi1.py --op_image \"/content/op/test/frame_9.jpg\" --dp_image \"/content/dp/test/frame_9.jpg\"\n",
+        "\n",
+        "!python /content/multi1.py --op_image \"/content/op/test/frame_10.jpg\" --dp_image \"/content/dp/test/frame_10.jpg\"\n",
+        "\n",
+        "!python /content/multi1.py --op_image \"/content/op/test/frame_11.jpg\" --dp_image \"/content/dp/test/frame_11.jpg\"\n",
+        "\n",
+        "!python /content/multi1.py --op_image \"/content/op/test/frame_12.jpg\" --dp_image \"/content/dp/test/frame_12.jpg\"\n",
+        "\n",
+        "!python /content/multi1.py --op_image \"/content/op/test/frame_13.jpg\" --dp_image \"/content/dp/test/frame_13.jpg\"\n",
+        "\n",
+        "!python /content/multi1.py --op_image \"/content/op/test/frame_14.jpg\" --dp_image \"/content/dp/test/frame_14.jpg\"\n",
+        "\n",
+        "!python /content/multi1.py --op_image \"/content/op/test/frame_15.jpg\" --dp_image \"/content/dp/test/frame_15.jpg\"\n",
+        "\n",
+        "!python /content/multi1.py --op_image \"/content/op/test/frame_16.jpg\" --dp_image \"/content/dp/test/frame_16.jpg\"\n",
+        "\n",
+        "!python /content/multi1.py --op_image \"/content/op/test/frame_17.jpg\" --dp_image \"/content/dp/test/frame_17.jpg\"\n",
+        "\n",
+        "!python /content/multi1.py --op_image \"/content/op/test/frame_18.jpg\" --dp_image \"/content/dp/test/frame_18.jpg\"\n",
+        "\n",
+        "!python /content/multi1.py --op_image \"/content/op/test/frame_19.jpg\" --dp_image \"/content/dp/test/frame_19.jpg\"\n",
+        "\n",
+        "!python /content/multi1.py --op_image \"/content/op/test/frame_20.jpg\" --dp_image \"/content/dp/test/frame_20.jpg\"\n",
+        "\n",
+        "!python /content/multi1.py --op_image \"/content/op/test/frame_21.jpg\" --dp_image \"/content/dp/test/frame_21.jpg\"\n",
+        "\n",
+        "!python /content/multi1.py --op_image \"/content/op/test/frame_22.jpg\" --dp_image \"/content/dp/test/frame_22.jpg\"\n",
+        "\n",
+        "!python /content/multi1.py --op_image \"/content/op/test/frame_23.jpg\" --dp_image \"/content/dp/test/frame_23.jpg\"\n",
+        "\n",
+        "!python /content/multi1.py --op_image \"/content/op/test/frame_24.jpg\" --dp_image \"/content/dp/test/frame_24.jpg\"\n",
+        "\n",
+        "!python /content/multi1.py --op_image \"/content/op/test/frame_25.jpg\" --dp_image \"/content/dp/test/frame_25.jpg\"\n",
+        "\n",
+        "!python /content/multi1.py --op_image \"/content/op/test/frame_26.jpg\" --dp_image \"/content/dp/test/frame_26.jpg\"\n",
+        "\n",
+        "!python /content/multi1.py --op_image \"/content/op/test/frame_27.jpg\" --dp_image \"/content/dp/test/frame_27.jpg\"\n",
+        "\n",
+        "!python /content/multi1.py --op_image \"/content/op/test/frame_28.jpg\" --dp_image \"/content/dp/test/frame_28.jpg\"\n",
+        "\n",
+        "!python /content/multi1.py --op_image \"/content/op/test/frame_29.jpg\" --dp_image \"/content/dp/test/frame_29.jpg\"\n",
+        "\n",
+        "!python /content/multi1.py --op_image \"/content/op/test/frame_30.jpg\" --dp_image \"/content/dp/test/frame_30.jpg\"\n",
+        "\n",
+        "!python /content/multi1.py --op_image \"/content/op/test/frame_31.jpg\" --dp_image \"/content/dp/test/frame_31.jpg\"\n",
+        "\n",
+        "!python /content/multi1.py --op_image \"/content/op/test/frame_32.jpg\" --dp_image \"/content/dp/test/frame_32.jpg\"\n",
+        "\n",
+        "!python /content/multi1.py --op_image \"/content/op/test/frame_33.jpg\" --dp_image \"/content/dp/test/frame_33.jpg\"\n",
+        "\n",
+        "!python /content/multi1.py --op_image \"/content/op/test/frame_34.jpg\" --dp_image \"/content/dp/test/frame_34.jpg\"\n",
+        "\n",
+        "!python /content/multi1.py --op_image \"/content/op/test/frame_35.jpg\" --dp_image \"/content/dp/test/frame_35.jpg\"\n",
+        "\n",
+        "!python /content/multi1.py --op_image \"/content/op/test/frame_36.jpg\" --dp_image \"/content/dp/test/frame_36.jpg\"\n",
+        "\n",
+        "!python /content/multi1.py --op_image \"/content/op/test/frame_37.jpg\" --dp_image \"/content/dp/test/frame_37.jpg\"\n",
+        "\n",
+        "!python /content/multi1.py --op_image \"/content/op/test/frame_38.jpg\" --dp_image \"/content/dp/test/frame_38.jpg\"\n",
+        "\n",
+        "!python /content/multi1.py --op_image \"/content/op/test/frame_39.jpg\" --dp_image \"/content/dp/test/frame_39.jpg\"\n",
+        "\n",
+        "!python /content/multi1.py --op_image \"/content/op/test/frame_40.jpg\" --dp_image \"/content/dp/test/frame_40.jpg\"\n",
+        "\n",
+        "!python /content/multi1.py --op_image \"/content/op/test/frame_41.jpg\" --dp_image \"/content/dp/test/frame_41.jpg\"\n",
+        "\n",
+        "!python /content/multi1.py --op_image \"/content/op/test/frame_42.jpg\" --dp_image \"/content/dp/test/frame_42.jpg\"\n",
+        "\n",
+        "!python /content/multi1.py --op_image \"/content/op/test/frame_43.jpg\" --dp_image \"/content/dp/test/frame_43.jpg\"\n",
+        "\n",
+        "!python /content/multi1.py --op_image \"/content/op/test/frame_44.jpg\" --dp_image \"/content/dp/test/frame_44.jpg\"\n",
+        "\n",
+        "!python /content/multi1.py --op_image \"/content/op/test/frame_45.jpg\" --dp_image \"/content/dp/test/frame_45.jpg\"\n",
+        "\n",
+        "!python /content/multi1.py --op_image \"/content/op/test/frame_46.jpg\" --dp_image \"/content/dp/test/frame_46.jpg\"\n",
+        "\n",
+        "!python /content/multi1.py --op_image \"/content/op/test/frame_47.jpg\" --dp_image \"/content/dp/test/frame_47.jpg\""
+      ],
+      "metadata": {
+        "cellView": "form",
+        "id": "IQ4wkiE04sN3"
+      },
+      "execution_count": null,
+      "outputs": []
+    },
+    {
+      "cell_type": "code",
+      "source": [
+        "#@title interpolate processed frames (best to keep fps same as input video)\n",
+        "!ffmpeg -r 12 -i /content/output_processed/frame_%d.jpg -vf \"format=yuv420p\" -c:v libx264 -crf 1 boxermulti1.mp4\n",
+        "\n"
+      ],
+      "metadata": {
+        "cellView": "form",
+        "id": "fjpga-Ra5-RN"
+      },
+      "execution_count": null,
+      "outputs": []
+    },
+    {
+      "cell_type": "code",
+      "source": [
+        "#@title  display video\n",
+        "from IPython.display import HTML\n",
+        "from base64 import b64encode\n",
+        "\n",
+        "# Open the video file and read its contents\n",
+        "mp4 = open('/content/boxermulti1.mp4', 'rb').read()\n",
+        "\n",
+        "# Encode the video data as a base64 string\n",
+        "data_url = \"data:video/mp4;base64,\" + b64encode(mp4).decode()\n",
+        "\n",
+        "# Display the video using an HTML video element\n",
+        "HTML(f\"\"\"\n",
+        "<video width=600 controls>\n",
+        "    <source src=\"{data_url}\" type=\"video/mp4\">\n",
+        "</video>\n",
+        "\"\"\")"
+      ],
+      "metadata": {
+        "cellView": "form",
+        "id": "4PaTDmZw5_2P"
+      },
+      "execution_count": null,
+      "outputs": []
+    },
+    {
+      "cell_type": "code",
+      "source": [
+        "#@title clear variables and empty vram cache\n",
+        "import gc\n",
+        "gc.collect()\n",
+        "torch.cuda.empty_cache()"
+      ],
+      "metadata": {
+        "cellView": "form",
+        "id": "30XaQxxq5cU5"
+      },
+      "execution_count": null,
+      "outputs": []
+    },
+    {
+      "cell_type": "code",
+      "source": [
+        "#@title zip up frames\n",
+        "!zip -r multiframes.zip /content/multi"
+      ],
+      "metadata": {
+        "cellView": "form",
+        "id": "Qjocs_K45A1u"
+      },
+      "execution_count": null,
+      "outputs": []
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "FaF3RdKdaFa8",
+        "cellView": "form"
+      },
+      "outputs": [],
+      "source": [
+        "#@title Login to HuggingFace 🤗\n",
+        "\n",
+        "#@markdown You need to accept the model license before downloading or using the Stable Diffusion weights. Please, visit the [model card](https://huggingface.co/runwayml/stable-diffusion-v1-5), read the license and tick the checkbox if you agree. You have to be a registered user in 🤗 Hugging Face Hub, and you'll also need to use an access token for the code to work.\n",
+        "# https://huggingface.co/settings/tokens\n",
+        "!mkdir -p ~/.huggingface\n",
+        "HUGGINGFACE_TOKEN = \"\" #@param {type:\"string\"}\n",
+        "!echo -n \"{HUGGINGFACE_TOKEN}\" > ~/.huggingface/token"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "aEJZoFQ2YHIb",
+        "cellView": "form"
+      },
+      "outputs": [],
+      "source": [
+        "#@title upload to huggingface\n",
+        "from huggingface_hub import HfApi\n",
+        "api = HfApi()\n",
+        "api.upload_file(\n",
+        "    path_or_fileobj=\"\",\n",
+        "    path_in_repo=\"\",\n",
+        "    repo_id=\"\",\n",
+        "    repo_type=\"model\",\n",
+        ")"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "source": [
+        "#=============================="
+      ],
+      "metadata": {
+        "id": "LFm40CCy5Upw"
+      },
+      "execution_count": null,
+      "outputs": []
+    },
+    {
+      "cell_type": "code",
+      "source": [
+        "#@title  OG multi code for reference do not change https://huggingface.co/blog/controlnet\n",
+        "from diffusers import StableDiffusionControlNetPipeline, ControlNetModel, UniPCMultistepScheduler\n",
+        "import torch\n",
+        "\n",
+        "controlnet = [\n",
+        "    ControlNetModel.from_pretrained(\"lllyasviel/sd-controlnet-openpose\", torch_dtype=torch.float16),\n",
+        "    ControlNetModel.from_pretrained(\"lllyasviel/sd-controlnet-canny\", torch_dtype=torch.float16),\n",
+        "]\n",
+        "\n",
+        "pipe = StableDiffusionControlNetPipeline.from_pretrained(\n",
+        "    \"runwayml/stable-diffusion-v1-5\", controlnet=controlnet, torch_dtype=torch.float16\n",
+        ")\n",
+        "pipe.scheduler = UniPCMultistepScheduler.from_config(pipe.scheduler.config)\n",
+        "\n",
+        "pipe.enable_xformers_memory_efficient_attention()\n",
+        "pipe.enable_model_cpu_offload()\n",
+        "\n",
+        "prompt = \"a giant standing in a fantasy landscape, best quality\"\n",
+        "negative_prompt = \"monochrome, lowres, bad anatomy, worst quality, low quality\"\n",
+        "\n",
+        "generator = torch.Generator(device=\"cpu\").manual_seed(1)\n",
+        "\n",
+        "images = [openpose_image, canny_image]\n",
+        "\n",
+        "image = pipe(\n",
+        "    prompt,\n",
+        "    images,\n",
+        "    num_inference_steps=20,\n",
+        "    generator=generator,\n",
+        "    negative_prompt=negative_prompt,\n",
+        "    controlnet_conditioning_scale=[1.0, 0.8],\n",
+        ").images[0]\n",
+        "\n",
+        "image.save(\"./multi_controlnet_output.png\")\n",
+        "\n",
+        "\n"
+      ],
+      "metadata": {
+        "cellView": "form",
+        "id": "a44MnBt-5N6d"
+      },
+      "execution_count": null,
+      "outputs": []
+    }
+  ]
+}

boxermultiframes.zip

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:32113b8ee5a722d171b7d5a5ac35b5bc8ff47d05482d1edad6e653c375f204fa
+size 1170711

multi1.py

@@ -0,0 +1,62 @@
+from diffusers import StableDiffusionControlNetPipeline, ControlNetModel, UniPCMultistepScheduler
+import torch
+import os
+from diffusers import StableDiffusionControlNetPipeline, ControlNetModel, UniPCMultistepScheduler
+from diffusers.utils import load_image
+import argparse
+from PIL import Image
+import cv2
+import numpy as np
+import torch
+import os
+import shutil
+from tqdm import tqdm
+import numpy as np
+
+device = ("cuda")
+
+# Initialize the argument parser
+parser = argparse.ArgumentParser(description='Choose a processor to run.')
+parser.add_argument('--op_image', type=str, help='path to pose image')
+parser.add_argument('--dp_image', type=str, help='path to depth image')
+parser.add_argument('--output_dir', type=str, default='/content/multi', help='The directory to save the output.')
+# Parse the arguments
+args = parser.parse_args()
+
+op_image = load_image(args.op_image)
+dp_image = load_image(args.dp_image)
+
+controlnet = [
+    ControlNetModel.from_pretrained("/content/checkpoints/openpose", torch_dtype=torch.float16).to('cuda'),
+    ControlNetModel.from_pretrained("/content/checkpoints/depth", torch_dtype=torch.float16).to('cuda'),
+]
+
+pipe = StableDiffusionControlNetPipeline.from_pretrained(
+    "SG161222/Realistic_Vision_V4.0_noVAE", controlnet=controlnet, torch_dtype=torch.float16
+).to('cuda')
+
+pipe.scheduler = UniPCMultistepScheduler.from_config(pipe.scheduler.config)
+
+prompt = "a boxer in a boxing ring, best quality"
+negative_prompt = "monochrome, lowres, bad anatomy, worst quality, low quality"
+
+images = [op_image, dp_image]
+
+image = pipe(
+    prompt,
+    images,
+    num_inference_steps=20,
+    negative_prompt=negative_prompt,
+    controlnet_conditioning_scale=[1.0, 0.8],
+).images[0]
+
+# Extract the filename and extension from args.op_image
+filename, extension = os.path.splitext(os.path.basename(args.op_image))
+
+# Construct the full output path with the specified output directory
+output_path = os.path.join(args.output_dir, filename + extension)
+
+print(type(image))
+# Save the image using PIL
+image.save(output_path)  # Assuming image is from PIL
+print("saved in output directory!")