{ "cells": [ { "cell_type": "markdown", "metadata": { "id": "K9mZW3uquvhW" }, "source": [ "Adapted code from https://github.com/Linaqruf/kohya-trainer" ] }, { "cell_type": "markdown", "metadata": { "id": "tTVqCAgSmie4" }, "source": [ "# I. Install Kohya Trainer" ] }, { "cell_type": "code", "execution_count": 1, "metadata": { "cellView": "form", "colab": { "base_uri": "https://localhost:8080/" }, "id": "_u3q60di584x", "outputId": "5a9dd797-741c-482e-aa53-28e6f7fe823d" }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Mounted at /content/drive\n", "Cloning into '/content/kohya-trainer'...\n", "remote: Enumerating objects: 2514, done.\u001b[K\n", "remote: Counting objects: 100% (2514/2514), done.\u001b[K\n", "remote: Compressing objects: 100% (938/938), done.\u001b[K\n", "remote: Total 2514 (delta 1692), reused 2242 (delta 1574), pack-reused 0\u001b[K\n", "Receiving objects: 100% (2514/2514), 4.82 MiB | 11.12 MiB/s, done.\n", "Resolving deltas: 100% (1692/1692), done.\n", "The following additional packages will be installed:\n", " libaria2-0 libc-ares2\n", "The following NEW packages will be installed:\n", " aria2 libaria2-0 libc-ares2\n", "0 upgraded, 3 newly installed, 0 to remove and 24 not upgraded.\n", "Need to get 1,513 kB of archives.\n", "After this operation, 5,441 kB of additional disk space will be used.\n", "Selecting previously unselected package libc-ares2:amd64.\n", "(Reading database ... 121654 files and directories currently installed.)\n", "Preparing to unpack .../libc-ares2_1.18.1-1ubuntu0.22.04.2_amd64.deb ...\n", "Unpacking libc-ares2:amd64 (1.18.1-1ubuntu0.22.04.2) ...\n", "Selecting previously unselected package libaria2-0:amd64.\n", "Preparing to unpack .../libaria2-0_1.36.0-1_amd64.deb ...\n", "Unpacking libaria2-0:amd64 (1.36.0-1) ...\n", "Selecting previously unselected package aria2.\n", "Preparing to unpack .../aria2_1.36.0-1_amd64.deb ...\n", "Unpacking aria2 (1.36.0-1) ...\n", "Setting up libc-ares2:amd64 (1.18.1-1ubuntu0.22.04.2) ...\n", "Setting up libaria2-0:amd64 (1.36.0-1) ...\n", "Setting up aria2 (1.36.0-1) ...\n", "Processing triggers for man-db (2.10.2-1) ...\n", "Processing triggers for libc-bin (2.35-0ubuntu3.4) ...\n", "/sbin/ldconfig.real: /usr/local/lib/libtbbbind.so.3 is not a symbolic link\n", "\n", "/sbin/ldconfig.real: /usr/local/lib/libtbb.so.12 is not a symbolic link\n", "\n", "/sbin/ldconfig.real: /usr/local/lib/libtbbmalloc_proxy.so.2 is not a symbolic link\n", "\n", "/sbin/ldconfig.real: /usr/local/lib/libtbbmalloc.so.2 is not a symbolic link\n", "\n", "/sbin/ldconfig.real: /usr/local/lib/libtbbbind_2_0.so.3 is not a symbolic link\n", "\n", "/sbin/ldconfig.real: /usr/local/lib/libtbbbind_2_5.so.3 is not a symbolic link\n", "\n", " Preparing metadata (setup.py) ... \u001b[?25l\u001b[?25hdone\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m191.5/191.5 kB\u001b[0m \u001b[31m3.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m6.3/6.3 MB\u001b[0m \u001b[31m28.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m53.1/53.1 kB\u001b[0m \u001b[31m7.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m41.6/41.6 kB\u001b[0m \u001b[31m6.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m503.1/503.1 kB\u001b[0m \u001b[31m34.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m825.8/825.8 kB\u001b[0m \u001b[31m31.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m92.6/92.6 MB\u001b[0m \u001b[31m9.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m475.2/475.2 MB\u001b[0m \u001b[31m3.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m549.1/549.1 kB\u001b[0m \u001b[31m44.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m266.3/266.3 kB\u001b[0m \u001b[31m32.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[?25h Installing build dependencies ... \u001b[?25l\u001b[?25hdone\n", " Getting requirements to build wheel ... \u001b[?25l\u001b[?25hdone\n", " Installing backend dependencies ... \u001b[?25l\u001b[?25hdone\n", " Preparing metadata (pyproject.toml) ... \u001b[?25l\u001b[?25hdone\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m200.1/200.1 kB\u001b[0m \u001b[31m25.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m599.8/599.8 kB\u001b[0m \u001b[31m48.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[?25h Installing build dependencies ... \u001b[?25l\u001b[?25hdone\n", " Getting requirements to build wheel ... \u001b[?25l\u001b[?25hdone\n", " Preparing metadata (pyproject.toml) ... \u001b[?25l\u001b[?25hdone\n", " Preparing metadata (setup.py) ... \u001b[?25l\u001b[?25hdone\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m211.8/211.8 MB\u001b[0m \u001b[31m5.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m7.8/7.8 MB\u001b[0m \u001b[31m72.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m840.2/840.2 kB\u001b[0m \u001b[31m58.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.8/1.8 MB\u001b[0m \u001b[31m70.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[?25h Preparing metadata (setup.py) ... \u001b[?25l\u001b[?25hdone\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m247.3/247.3 kB\u001b[0m \u001b[31m25.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m43.1/43.1 kB\u001b[0m \u001b[31m5.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m57.7/57.7 kB\u001b[0m \u001b[31m8.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m115.0/115.0 kB\u001b[0m \u001b[31m6.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m247.1/247.1 kB\u001b[0m \u001b[31m18.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[?25h Building wheel for fairscale (pyproject.toml) ... \u001b[?25l\u001b[?25hdone\n", " Building wheel for dadaptation (pyproject.toml) ... \u001b[?25l\u001b[?25hdone\n", " Building wheel for lycoris-lora (setup.py) ... \u001b[?25l\u001b[?25hdone\n", " Building wheel for library (setup.py) ... \u001b[?25l\u001b[?25hdone\n", " Building wheel for elfinder-client (setup.py) ... \u001b[?25l\u001b[?25hdone\n" ] } ], "source": [ "# @title ## 1.1. Install Dependencies\n", "# @markdown Clone Kohya Trainer from GitHub and check for updates. Use textbox below if you want to checkout other branch or old commit. Leave it empty to stay the HEAD on main. This will also install the required libraries.\n", "import os\n", "import zipfile\n", "import shutil\n", "import time\n", "from subprocess import getoutput\n", "from IPython.utils import capture\n", "from google.colab import drive\n", "\n", "%store -r\n", "\n", "# root_dir\n", "root_dir = \"/content\"\n", "deps_dir = os.path.join(root_dir, \"deps\")\n", "repo_dir = os.path.join(root_dir, \"kohya-trainer\")\n", "training_dir = os.path.join(root_dir, \"fine_tune\")\n", "pretrained_model = os.path.join(root_dir, \"pretrained_model\")\n", "vae_dir = os.path.join(root_dir, \"vae\")\n", "config_dir = os.path.join(training_dir, \"config\")\n", "\n", "# repo_dir\n", "accelerate_config = os.path.join(repo_dir, \"accelerate_config/config.yaml\")\n", "tools_dir = os.path.join(repo_dir, \"tools\")\n", "finetune_dir = os.path.join(repo_dir, \"finetune\")\n", "\n", "for store in [\n", " \"root_dir\",\n", " \"deps_dir\",\n", " \"repo_dir\",\n", " \"training_dir\",\n", " \"pretrained_model\",\n", " \"vae_dir\",\n", " \"accelerate_config\",\n", " \"tools_dir\",\n", " \"finetune_dir\",\n", " \"config_dir\",\n", "]:\n", " with capture.capture_output() as cap:\n", " %store {store}\n", " del cap\n", "\n", "repo_url = \"https://github.com/Linaqruf/kohya-trainer\"\n", "bitsandytes_main_py = \"/usr/local/lib/python3.10/dist-packages/bitsandbytes/cuda_setup/main.py\"\n", "branch = \"\" # @param {type: \"string\"}\n", "mount_drive = True # @param {type: \"boolean\"}\n", "verbose = False # @param {type: \"boolean\"}\n", "\n", "def read_file(filename):\n", " with open(filename, \"r\") as f:\n", " contents = f.read()\n", " return contents\n", "\n", "\n", "def write_file(filename, contents):\n", " with open(filename, \"w\") as f:\n", " f.write(contents)\n", "\n", "\n", "def clone_repo(url):\n", " if not os.path.exists(repo_dir):\n", " os.chdir(root_dir)\n", " !git clone {url} {repo_dir}\n", " else:\n", " os.chdir(repo_dir)\n", " !git pull origin {branch} if branch else !git pull\n", "\n", "def install_dependencies():\n", " s = getoutput('nvidia-smi')\n", "\n", " if 'T4' in s:\n", " !sed -i \"s@cpu@cuda@\" library/model_util.py\n", "\n", " !pip install {'-q' if not verbose else ''} --upgrade -r requirements.txt\n", "\n", " from accelerate.utils import write_basic_config\n", "\n", " if not os.path.exists(accelerate_config):\n", " write_basic_config(save_location=accelerate_config)\n", "\n", "\n", "def remove_bitsandbytes_message(filename):\n", " welcome_message = \"\"\"\n", "def evaluate_cuda_setup():\n", " print('')\n", " print('='*35 + 'BUG REPORT' + '='*35)\n", " print('Welcome to bitsandbytes. For bug reports, please submit your error trace to: https://github.com/TimDettmers/bitsandbytes/issues')\n", " print('For effortless bug reporting copy-paste your error into this form: https://docs.google.com/forms/d/e/1FAIpQLScPB8emS3Thkp66nvqwmjTEgxp8Y9ufuWTzFyr9kJ5AoI47dQ/viewform?usp=sf_link')\n", " print('='*80)\"\"\"\n", "\n", " new_welcome_message = \"\"\"\n", "def evaluate_cuda_setup():\n", " import os\n", " if 'BITSANDBYTES_NOWELCOME' not in os.environ or str(os.environ['BITSANDBYTES_NOWELCOME']) == '0':\n", " print('')\n", " print('=' * 35 + 'BUG REPORT' + '=' * 35)\n", " print('Welcome to bitsandbytes. For bug reports, please submit your error trace to: https://github.com/TimDettmers/bitsandbytes/issues')\n", " print('For effortless bug reporting copy-paste your error into this form: https://docs.google.com/forms/d/e/1FAIpQLScPB8emS3Thkp66nvqwmjTEgxp8Y9ufuWTzFyr9kJ5AoI47dQ/viewform?usp=sf_link')\n", " print('To hide this message, set the BITSANDBYTES_NOWELCOME variable like so: export BITSANDBYTES_NOWELCOME=1')\n", " print('=' * 80)\"\"\"\n", "\n", " contents = read_file(filename)\n", " new_contents = contents.replace(welcome_message, new_welcome_message)\n", " write_file(filename, new_contents)\n", "\n", "\n", "def main():\n", " os.chdir(root_dir)\n", "\n", " if mount_drive:\n", " if not os.path.exists(\"/content/drive\"):\n", " drive.mount(\"/content/drive\")\n", "\n", " for dir in [\n", " deps_dir,\n", " training_dir,\n", " config_dir,\n", " pretrained_model,\n", " vae_dir\n", " ]:\n", " os.makedirs(dir, exist_ok=True)\n", "\n", " clone_repo(repo_url)\n", "\n", " if branch:\n", " os.chdir(repo_dir)\n", " status = os.system(f\"git checkout {branch}\")\n", " if status != 0:\n", " raise Exception(\"Failed to checkout branch or commit\")\n", "\n", " os.chdir(repo_dir)\n", "\n", " !apt install aria2 {'-qq' if not verbose else ''}\n", "\n", " install_dependencies()\n", " time.sleep(3)\n", "\n", " remove_bitsandbytes_message(bitsandytes_main_py)\n", "\n", " os.environ[\"TF_CPP_MIN_LOG_LEVEL\"] = \"3\"\n", " os.environ[\"BITSANDBYTES_NOWELCOME\"] = \"1\"\n", " os.environ[\"SAFETENSORS_FAST_GPU\"] = \"1\"\n", "\n", " cuda_path = \"/usr/local/cuda-11.8/targets/x86_64-linux/lib/\"\n", " ld_library_path = os.environ.get(\"LD_LIBRARY_PATH\", \"\")\n", " os.environ[\"LD_LIBRARY_PATH\"] = f\"{ld_library_path}:{cuda_path}\"\n", "\n", "main()\n" ] }, { "cell_type": "markdown", "metadata": { "id": "3gob9_OwTlwh" }, "source": [ "# II. Pretrained Model Selection" ] }, { "cell_type": "code", "execution_count": 2, "metadata": { "cellView": "form", "colab": { "base_uri": "https://localhost:8080/" }, "id": "wmnsZwClN1XL", "outputId": "2794cbcc-da35-4aaf-e00d-93b9082f3fc4" }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ " *** Download Progress Summary as of Mon Jan 15 01:30:24 2024 *** \n", "=\n", "[#1f7e7b 1.5GiB/1.9GiB(77%) CN:16 DL:150MiB ETA:2s]\n", "FILE: /content/pretrained_model/AnyLoRA.safetensors\n", "-\n", "\n", "\u001b[0m\n", "Download Results:\n", "gid |stat|avg speed |path/URI\n", "======+====+===========+=======================================================\n", "1f7e7b|\u001b[1;32mOK\u001b[0m | 162MiB/s|/content/pretrained_model/AnyLoRA.safetensors\n", "\n", "Status Legend:\n", "(OK):download completed.\n" ] } ], "source": [ "# @title ## 2.1. Download Available Model\n", "import os\n", "\n", "%store -r\n", "\n", "os.chdir(root_dir)\n", "\n", "models = {\n", " \"Anything-v3-1\": \"https://huggingface.co/cag/anything-v3-1/resolve/main/anything-v3-1.safetensors\",\n", " \"AnyLoRA\": \"https://huggingface.co/Linaqruf/stolen/resolve/main/pruned-models/AnyLoRA_noVae_fp16-pruned.safetensors\",\n", " \"Stable-Diffusion-v1-5\": \"https://huggingface.co/Linaqruf/stolen/resolve/main/pruned-models/stable_diffusion_1_5-pruned.safetensors\",\n", "}\n", "\n", "v2_models = {\n", " \"stable-diffusion-2-1-base\": \"https://huggingface.co/stabilityai/stable-diffusion-2-1-base/resolve/main/v2-1_512-ema-pruned.safetensors\",\n", " \"stable-diffusion-2-1-768v\": \"https://huggingface.co/stabilityai/stable-diffusion-2-1/resolve/main/v2-1_768-ema-pruned.safetensors\",\n", "}\n", "\n", "installModels = []\n", "installv2Models = []\n", "\n", "# @markdown ### SD1.x model\n", "model_name = \"AnyLoRA\" # @param [\"\", \"Anything-v3-1\", \"AnyLoRA\", \"Stable-Diffusion-v1-5\"]\n", "# @markdown ### SD2.x model\n", "v2_model_name = \"\" # @param [\"\", \"stable-diffusion-2-1-base\", \"stable-diffusion-2-1-768v\"]\n", "\n", "if model_name:\n", " model_url = models.get(model_name)\n", " if model_url:\n", " installModels.append((model_name, model_url))\n", "\n", "if v2_model_name:\n", " v2_model_url = v2_models.get(v2_model_name)\n", " if v2_model_url:\n", " installv2Models.append((v2_model_name, v2_model_url))\n", "\n", "\n", "def install(checkpoint_name, url):\n", " ext = \"ckpt\" if url.endswith(\".ckpt\") else \"safetensors\"\n", "\n", " hf_token = \"token\" # @param {type: \"string\"}\n", " user_header = f'\"Authorization: Bearer {hf_token}\"'\n", " !aria2c --console-log-level=error --summary-interval=10 --header={user_header} -c -x 16 -k 1M -s 16 -d {pretrained_model} -o {checkpoint_name}.{ext} \"{url}\"\n", "\n", "\n", "def install_checkpoint():\n", " for model in installModels:\n", " install(model[0], model[1])\n", " for v2model in installv2Models:\n", " install(v2model[0], v2model[1])\n", "\n", "\n", "install_checkpoint()\n", "\n" ] }, { "cell_type": "markdown", "metadata": { "id": "En9UUwGNMRMM" }, "source": [ "# III. Data Acquisition\n", "\n", "You have three options for acquiring your dataset:\n", "\n", "1. Uploading it to Colab's local files.\n", "2. Bulk downloading images from Danbooru using the `Simple Booru Scraper`.\n", "3. Locating your dataset from Google Drive.\n" ] }, { "cell_type": "code", "execution_count": 3, "metadata": { "cellView": "form", "colab": { "base_uri": "https://localhost:8080/" }, "id": "T95ErCuCSCDC", "outputId": "1d743d41-be63-4d64-d96d-b26249e31907" }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Stored 'train_data_dir' (str)\n", "Your train data directory : /content/fine_tune/train_data\n" ] } ], "source": [ "# @title ## 3.1. Locating Train Data Directory\n", "# @markdown Define location of your training data. This cell will also create a folder based on your input.\n", "# @markdown This folder will serve as the target folder for scraping, tagging, bucketing, and training in the next cell.\n", "import os\n", "\n", "%store -r\n", "\n", "train_data_dir = \"/content/fine_tune/train_data\" # @param {'type' : 'string'}\n", "%store train_data_dir\n", "\n", "os.makedirs(train_data_dir, exist_ok=True)\n", "print(f\"Your train data directory : {train_data_dir}\")" ] }, { "cell_type": "code", "execution_count": 4, "metadata": { "cellView": "form", "colab": { "base_uri": "https://localhost:8080/" }, "id": "eFFHVTWNZGbp", "outputId": "69948ed2-db3e-4d88-c330-1e8887e56ea8" }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Archive: /content/rhnd.zip\n", " inflating: /content/fine_tune/train_data/1.jpg \n", " inflating: /content/fine_tune/train_data/1.txt \n", " inflating: /content/fine_tune/train_data/10.jpg \n", " inflating: /content/fine_tune/train_data/10.txt \n", " inflating: /content/fine_tune/train_data/11.jpg \n", " inflating: /content/fine_tune/train_data/11.txt \n", " inflating: /content/fine_tune/train_data/12.jpg \n", " inflating: /content/fine_tune/train_data/12.txt \n", " inflating: /content/fine_tune/train_data/13.jpg \n", " inflating: /content/fine_tune/train_data/13.txt \n", " inflating: /content/fine_tune/train_data/14.jpg \n", " inflating: /content/fine_tune/train_data/14.txt \n", " inflating: /content/fine_tune/train_data/15.jpg \n", " inflating: /content/fine_tune/train_data/15.txt \n", " inflating: /content/fine_tune/train_data/16.jpg \n", " inflating: /content/fine_tune/train_data/16.txt \n", " inflating: /content/fine_tune/train_data/17.jpg \n", " inflating: /content/fine_tune/train_data/17.txt \n", " inflating: /content/fine_tune/train_data/18.txt \n", " inflating: /content/fine_tune/train_data/19.jpg \n", " inflating: /content/fine_tune/train_data/19.txt \n", " inflating: /content/fine_tune/train_data/2.jpg \n", " inflating: /content/fine_tune/train_data/2.txt \n", " inflating: /content/fine_tune/train_data/20.jpg \n", " inflating: /content/fine_tune/train_data/20.txt \n", " inflating: /content/fine_tune/train_data/21.jpg \n", " inflating: /content/fine_tune/train_data/21.txt \n", " inflating: /content/fine_tune/train_data/22.jpg \n", " inflating: /content/fine_tune/train_data/22.txt \n", " inflating: /content/fine_tune/train_data/23.jpg \n", " inflating: /content/fine_tune/train_data/23.txt \n", " inflating: /content/fine_tune/train_data/24.jpg \n", " inflating: /content/fine_tune/train_data/24.txt \n", " inflating: /content/fine_tune/train_data/25.jpg \n", " inflating: /content/fine_tune/train_data/25.txt \n", " inflating: /content/fine_tune/train_data/26.jpg \n", " inflating: /content/fine_tune/train_data/26.txt \n", " inflating: /content/fine_tune/train_data/27.jpg \n", " inflating: /content/fine_tune/train_data/27.txt \n", " inflating: /content/fine_tune/train_data/28.jpg \n", " inflating: /content/fine_tune/train_data/28.txt \n", " inflating: /content/fine_tune/train_data/29.jpg \n", " inflating: /content/fine_tune/train_data/29.txt \n", " inflating: /content/fine_tune/train_data/3.jpg \n", " inflating: /content/fine_tune/train_data/3.txt \n", " inflating: /content/fine_tune/train_data/30.jpg \n", " inflating: /content/fine_tune/train_data/30.txt \n", " inflating: /content/fine_tune/train_data/31.jpg \n", " inflating: /content/fine_tune/train_data/31.txt \n", " inflating: /content/fine_tune/train_data/32.jpg \n", " inflating: /content/fine_tune/train_data/32.txt \n", " inflating: /content/fine_tune/train_data/33.jpg \n", " inflating: /content/fine_tune/train_data/33.txt \n", " inflating: /content/fine_tune/train_data/34.jpg \n", " inflating: /content/fine_tune/train_data/34.txt \n", " inflating: /content/fine_tune/train_data/35.jpg \n", " inflating: /content/fine_tune/train_data/35.txt \n", " inflating: /content/fine_tune/train_data/36.jpg \n", " inflating: /content/fine_tune/train_data/36.txt \n", " inflating: /content/fine_tune/train_data/37.jpg \n", " inflating: /content/fine_tune/train_data/37.txt \n", " inflating: /content/fine_tune/train_data/38.jpg \n", " inflating: /content/fine_tune/train_data/38.txt \n", " inflating: /content/fine_tune/train_data/39.jpg \n", " inflating: /content/fine_tune/train_data/39.txt \n", " inflating: /content/fine_tune/train_data/4.jpg \n", " inflating: /content/fine_tune/train_data/4.txt \n", " inflating: /content/fine_tune/train_data/40.jpg \n", " inflating: /content/fine_tune/train_data/40.txt \n", " inflating: /content/fine_tune/train_data/41.jpg \n", " inflating: /content/fine_tune/train_data/41.txt \n", " inflating: /content/fine_tune/train_data/42.jpg \n", " inflating: /content/fine_tune/train_data/42.txt \n", " inflating: /content/fine_tune/train_data/43.jpg \n", " inflating: /content/fine_tune/train_data/43.txt \n", " inflating: /content/fine_tune/train_data/44.jpg \n", " inflating: /content/fine_tune/train_data/44.txt \n", " inflating: /content/fine_tune/train_data/45.jpg \n", " inflating: /content/fine_tune/train_data/45.txt \n", " inflating: /content/fine_tune/train_data/46.jpg \n", " inflating: /content/fine_tune/train_data/46.txt \n", " inflating: /content/fine_tune/train_data/47.jpg \n", " inflating: /content/fine_tune/train_data/47.txt \n", " inflating: /content/fine_tune/train_data/48.jpg \n", " inflating: /content/fine_tune/train_data/48.txt \n", " inflating: /content/fine_tune/train_data/49.jpg \n", " inflating: /content/fine_tune/train_data/49.txt \n", " inflating: /content/fine_tune/train_data/5.jpg \n", " inflating: /content/fine_tune/train_data/5.txt \n", " inflating: /content/fine_tune/train_data/50.jpg \n", " inflating: /content/fine_tune/train_data/50.txt \n", " inflating: /content/fine_tune/train_data/51.jpg \n", " inflating: /content/fine_tune/train_data/51.txt \n", " inflating: /content/fine_tune/train_data/52.jpg \n", " inflating: /content/fine_tune/train_data/52.txt \n", " inflating: /content/fine_tune/train_data/53.jpg \n", " inflating: /content/fine_tune/train_data/53.txt \n", " inflating: /content/fine_tune/train_data/54.jpg \n", " inflating: /content/fine_tune/train_data/54.txt \n", " inflating: /content/fine_tune/train_data/55.jpg \n", " inflating: /content/fine_tune/train_data/55.txt \n", " inflating: /content/fine_tune/train_data/56.jpg \n", " inflating: /content/fine_tune/train_data/56.txt \n", " inflating: /content/fine_tune/train_data/57.jpg \n", " inflating: /content/fine_tune/train_data/57.txt \n", " inflating: /content/fine_tune/train_data/58.jpg \n", " inflating: /content/fine_tune/train_data/58.txt \n", " inflating: /content/fine_tune/train_data/59.jpg \n", " inflating: /content/fine_tune/train_data/59.txt \n", " inflating: /content/fine_tune/train_data/6.jpg \n", " inflating: /content/fine_tune/train_data/6.txt \n", " inflating: /content/fine_tune/train_data/60.jpg \n", " inflating: /content/fine_tune/train_data/60.txt \n", " inflating: /content/fine_tune/train_data/61.jpg \n", " inflating: /content/fine_tune/train_data/61.txt \n", " inflating: /content/fine_tune/train_data/62.jpg \n", " inflating: /content/fine_tune/train_data/62.txt \n", " inflating: /content/fine_tune/train_data/63.jpg \n", " inflating: /content/fine_tune/train_data/63.txt \n", " inflating: /content/fine_tune/train_data/64.jpg \n", " inflating: /content/fine_tune/train_data/64.txt \n", " inflating: /content/fine_tune/train_data/65.jpg \n", " inflating: /content/fine_tune/train_data/65.txt \n", " inflating: /content/fine_tune/train_data/66.jpg \n", " inflating: /content/fine_tune/train_data/66.txt \n", " inflating: /content/fine_tune/train_data/67.jpg \n", " inflating: /content/fine_tune/train_data/67.txt \n", " inflating: /content/fine_tune/train_data/68.jpg \n", " inflating: /content/fine_tune/train_data/68.txt \n", " inflating: /content/fine_tune/train_data/69.jpg \n", " inflating: /content/fine_tune/train_data/69.txt \n", " inflating: /content/fine_tune/train_data/7.jpg \n", " inflating: /content/fine_tune/train_data/7.txt \n", " inflating: /content/fine_tune/train_data/70.jpg \n", " inflating: /content/fine_tune/train_data/70.txt \n", " inflating: /content/fine_tune/train_data/71.jpg \n", " inflating: /content/fine_tune/train_data/71.txt \n", " inflating: /content/fine_tune/train_data/72.jpg \n", " inflating: /content/fine_tune/train_data/72.txt \n", " inflating: /content/fine_tune/train_data/73.jpg \n", " inflating: /content/fine_tune/train_data/73.txt \n", " inflating: /content/fine_tune/train_data/74.jpg \n", " inflating: /content/fine_tune/train_data/74.txt \n", " inflating: /content/fine_tune/train_data/75.jpg \n", " inflating: /content/fine_tune/train_data/75.txt \n", " inflating: /content/fine_tune/train_data/8.jpg \n", " inflating: /content/fine_tune/train_data/8.txt \n", " inflating: /content/fine_tune/train_data/9.txt \n" ] } ], "source": [ "# @title ## 3.2. Unzip Dataset\n", "\n", "import os\n", "import shutil\n", "from pathlib import Path\n", "\n", "#@title ## Unzip Dataset\n", "# @markdown Use this section if your dataset is in a `zip` file and has been uploaded somewhere. This code cell will download your dataset and automatically extract it to the `train_data_dir` if the `unzip_to` variable is empty.\n", "zipfile_url = \"/content/rhnd.zip\" #@param {type:\"string\"}\n", "zipfile_name = \"zipfile.zip\"\n", "unzip_to = \"\" #@param {type:\"string\"}\n", "\n", "hf_token = \"token\" # @param {type: \"string\"}\n", "user_header = f'\"Authorization: Bearer {hf_token}\"'\n", "\n", "if unzip_to:\n", " os.makedirs(unzip_to, exist_ok=True)\n", "else:\n", " unzip_to = train_data_dir\n", "\n", "\n", "def download_dataset(url):\n", " if url.startswith(\"/content\"):\n", " return url\n", " elif \"drive.google.com\" in url:\n", " os.chdir(root_dir)\n", " !gdown --fuzzy {url}\n", " return f\"{root_dir}/{zipfile_name}\"\n", " elif \"huggingface.co\" in url:\n", " if \"/blob/\" in url:\n", " url = url.replace(\"/blob/\", \"/resolve/\")\n", " !aria2c --console-log-level=error --summary-interval=10 --header={user_header} -c -x 16 -k 1M -s 16 -d {root_dir} -o {zipfile_name} {url}\n", " return f\"{root_dir}/{zipfile_name}\"\n", " else:\n", " !aria2c --console-log-level=error --summary-interval=10 -c -x 16 -k 1M -s 16 -d {root_dir} -o {zipfile_name} {url}\n", " return f\"{root_dir}/{zipfile_name}\"\n", "\n", "\n", "def extract_dataset(zip_file, output_path):\n", " if zip_file.startswith(\"/content\"):\n", " !unzip -j -o {zip_file} -d \"{output_path}\"\n", " else:\n", " !unzip -j -o \"{zip_file}\" -d \"{output_path}\"\n", "\n", "\n", "def remove_files(train_dir, files_to_move):\n", " for filename in os.listdir(train_dir):\n", " file_path = os.path.join(train_dir, filename)\n", " if filename in files_to_move:\n", " if not os.path.exists(file_path):\n", " shutil.move(file_path, training_dir)\n", " else:\n", " os.remove(file_path)\n", "\n", "\n", "zip_file = download_dataset(zipfile_url)\n", "extract_dataset(zip_file, unzip_to)\n", "os.remove(zip_file)\n", "\n", "files_to_move = (\n", " \"meta_cap.json\",\n", " \"meta_cap_dd.json\",\n", " \"meta_lat.json\",\n", " \"meta_clean.json\",\n", ")\n", "\n", "remove_files(train_data_dir, files_to_move)\n" ] }, { "cell_type": "markdown", "metadata": { "id": "T-0qKyEgTchp" }, "source": [ "# IV. Data Preprocessing" ] }, { "cell_type": "code", "execution_count": 5, "metadata": { "cellView": "form", "colab": { "base_uri": "https://localhost:8080/" }, "id": "MEHNh0P8Yyxi", "outputId": "d3b92038-6bfc-48ab-b114-1002be6e4e6d" }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "CUDA backend failed to initialize: Found CUDA version 12010, but JAX was built against version 12020, which is newer. The copy of CUDA that is installed must be at least as new as the version against which JAX was built. (Set TF_CPP_MIN_LOG_LEVEL=0 and rerun for more info.)\n", "Found 73 images.\n", "Creating a new metadata file\n", "Merging tags and captions into metadata json.\n", "100% 73/73 [00:00<00:00, 17597.80it/s]\n", "No captions found for any of the 73 images\n", "All 73 images have tags\n", "Writing metadata: /content/fine_tune/meta_clean.json\n", "Done!\n" ] } ], "source": [ "#@title ## 4.3. Create Metadata File\n", "import os\n", "%store -r\n", "\n", "os.chdir(finetune_dir)\n", "\n", "# @markdown Merge tags and/or captions exist in `train_data_dir` into one metadata JSON file, which will be used as the input for the bucketing section.\n", "metadata = \"/content/fine_tune/meta_clean.json\" #@param {type:\"string\"}\n", "# @markdown Use `recursive` option to process subfolders as well\n", "recursive = False #@param {type:\"boolean\"}\n", "# @markdown Use `clean_caption` option to clean such as duplicate tags, `women` to `girl`, etc\n", "clean_caption = False #@param {type:\"boolean\"}\n", "\n", "config = {\n", " \"_train_data_dir\": train_data_dir,\n", " \"_out_json\": metadata,\n", " \"recursive\": recursive,\n", " \"full_path\": recursive,\n", " \"clean_caption\": clean_caption\n", "}\n", "\n", "args = \"\"\n", "for k, v in config.items():\n", " if k.startswith(\"_\"):\n", " args += f'\"{v}\" '\n", " elif isinstance(v, str):\n", " args += f'--{k}=\"{v}\" '\n", " elif isinstance(v, bool) and v:\n", " args += f\"--{k} \"\n", " elif isinstance(v, float) and not isinstance(v, bool):\n", " args += f\"--{k}={v} \"\n", " elif isinstance(v, int) and not isinstance(v, bool):\n", " args += f\"--{k}={v} \"\n", "\n", "os.chdir(finetune_dir)\n", "final_args = f\"python merge_all_to_metadata.py {args}\"\n", "!{final_args}\n" ] }, { "cell_type": "code", "execution_count": 6, "metadata": { "cellView": "form", "colab": { "base_uri": "https://localhost:8080/" }, "id": "hhgatqF3leHJ", "outputId": "851f7151-f6c2-4b06-9b1b-3c5c4d50b182" }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "CUDA backend failed to initialize: Found CUDA version 12010, but JAX was built against version 12020, which is newer. The copy of CUDA that is installed must be at least as new as the version against which JAX was built. (Set TF_CPP_MIN_LOG_LEVEL=0 and rerun for more info.)\n", "found 73 images.\n", "loading existing metadata: /content/fine_tune/meta_clean.json\n", "load VAE: /content/pretrained_model/AnyLoRA.safetensors\n", "100% 73/73 [00:05<00:00, 14.27it/s]\n", "bucket 0 (256, 256): 73\n", "mean ar error: 0.0\n", "writing metadata: /content/fine_tune/meta_lat.json\n", "done!\n" ] } ], "source": [ "# @title ## 4.4. Bucketing and Latents Caching\n", "%store -r\n", "\n", "# @markdown This code will create buckets based on the `max_resolution` provided for multi-aspect ratio training, and then convert all images within the `train_data_dir` to latents.\n", "v2 = False # @param{type:\"boolean\"}\n", "model_dir = \"/content/pretrained_model/AnyLoRA.safetensors\" # @param {'type' : 'string'}\n", "input_json = \"/content/fine_tune/meta_clean.json\" # @param {'type' : 'string'}\n", "output_json = \"/content/fine_tune/meta_lat.json\" # @param {'type' : 'string'}\n", "batch_size = 8 # @param {'type':'integer'}\n", "max_data_loader_n_workers = 2 # @param {'type':'integer'}\n", "max_resolution = \"256,256\" # @param [\"256,256\", \"512,512\", \"640,640\", \"768,768\"] {allow-input: false}\n", "mixed_precision = \"no\" # @param [\"no\", \"fp16\", \"bf16\"] {allow-input: false}\n", "flip_aug = False # @param{type:\"boolean\"}\n", "#@markdown Use the `recursive` option to process subfolders as well\n", "recursive = False #@param {type:\"boolean\"}\n", "\n", "config = {\n", " \"_train_data_dir\": train_data_dir,\n", " \"_in_json\": input_json,\n", " \"_out_json\": output_json,\n", " \"_model_name_or_path\": model_dir,\n", " \"recursive\": recursive,\n", " \"full_path\": recursive,\n", " \"v2\": v2,\n", " \"flip_aug\": flip_aug,\n", " \"min_bucket_reso\": 256 if max_resolution != \"256,256\" else 256,\n", " \"max_bucket_reso\": 1024 if max_resolution != \"256,256\" else 512,\n", " \"batch_size\": batch_size,\n", " \"max_data_loader_n_workers\": max_data_loader_n_workers,\n", " \"max_resolution\": max_resolution,\n", " \"mixed_precision\": mixed_precision,\n", "}\n", "\n", "args = \"\"\n", "for k, v in config.items():\n", " if k.startswith(\"_\"):\n", " args += f'\"{v}\" '\n", " elif isinstance(v, str):\n", " args += f'--{k}=\"{v}\" '\n", " elif isinstance(v, bool) and v:\n", " args += f\"--{k} \"\n", " elif isinstance(v, float) and not isinstance(v, bool):\n", " args += f\"--{k}={v} \"\n", " elif isinstance(v, int) and not isinstance(v, bool):\n", " args += f\"--{k}={v} \"\n", "\n", "os.chdir(finetune_dir)\n", "final_args = f\"python prepare_buckets_latents.py {args}\"\n", "!{final_args}" ] }, { "cell_type": "markdown", "metadata": { "id": "yHNbl3O_NSS0" }, "source": [ "# V. Training Model\n", "\n" ] }, { "cell_type": "code", "execution_count": 7, "metadata": { "cellView": "form", "colab": { "base_uri": "https://localhost:8080/" }, "id": "ur73rlY7bEef", "outputId": "0acc5c70-9365-4002-9c31-cbbace0c7d48" }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Stored 'project_name' (str)\n", "Project Name: rhnd\n", "Model Version: Stable Diffusion V1.x\n", "Pretrained Model Path: /content/pretrained_model/AnyLoRA.safetensors\n", "No VAE path specified.\n", "Output Path: /content/fine_tune/output\n", "No resume path specified.\n" ] } ], "source": [ "# @title ## 5.1. Model Config\n", "from google.colab import drive\n", "\n", "v2 = False # @param {type:\"boolean\"}\n", "v_parameterization = False # @param {type:\"boolean\"}\n", "project_name = \"rhnd\" # @param {type:\"string\"}\n", "if not project_name:\n", " project_name = \"last\"\n", "%store project_name\n", "pretrained_model_name_or_path = \"/content/pretrained_model/AnyLoRA.safetensors\" # @param {type:\"string\"}\n", "vae = \"\" # @param {type:\"string\"}\n", "output_dir = \"/content/fine_tune/output\" # @param {'type':'string'}\n", "resume_path = \"\" # @param {'type':'string'}\n", "\n", "# @markdown `output_to_drive` sets default `output_dir` to `/content/drive/MyDrive/fine_tune/output`. This will override the `output_dir` variable defined above.\n", "output_to_drive = False # @param {'type':'boolean'}\n", "\n", "if output_to_drive:\n", " output_dir = \"/content/drive/MyDrive/fine_tune/output\"\n", "\n", " if not os.path.exists(\"/content/drive\"):\n", " drive.mount(\"/content/drive\")\n", "\n", "sample_dir = os.path.join(output_dir, \"sample\")\n", "for dir in [output_dir, sample_dir]:\n", " os.makedirs(dir, exist_ok=True)\n", "\n", "print(\"Project Name: \", project_name)\n", "print(\"Model Version: Stable Diffusion V1.x\") if not v2 else \"\"\n", "print(\"Model Version: Stable Diffusion V2.x\") if v2 and not v_parameterization else \"\"\n", "print(\"Model Version: Stable Diffusion V2.x 768v\") if v2 and v_parameterization else \"\"\n", "print(\n", " \"Pretrained Model Path: \", pretrained_model_name_or_path\n", ") if pretrained_model_name_or_path else print(\"No Pretrained Model path specified.\")\n", "print(\"VAE Path: \", vae) if vae else print(\"No VAE path specified.\")\n", "print(\"Output Path: \", output_dir)\n", "print(\"Resume Path: \", resume_path) if resume_path else print(\n", " \"No resume path specified.\"\n", ")" ] }, { "cell_type": "code", "execution_count": 8, "metadata": { "cellView": "form", "id": "XkwCzdkWPJ9u" }, "outputs": [], "source": [ "# @title ## 5.2. Dataset Config\n", "import toml\n", "import glob\n", "\n", "# @markdown This notebook support multi-folder training but not designed for multi-concept training. You can use [Kohya LoRA Dreambooth](https://github.com/Linaqruf/kohya-trainer/blob/main/kohya-LoRA-dreambooth.ipynb), or add an activation word for each train folder under `4.2.3. Custom Caption/Tag (Optional)` instead.\n", "dataset_repeats = 10 # @param {type:\"number\"}\n", "in_json = \"/content/fine_tune/meta_lat.json\" # @param {type:\"string\"}\n", "resolution = \"256,256\" # @param [\"256,256\", \"768,768\"]\n", "keep_tokens = 0 # @param {type:\"number\"}\n" ] }, { "cell_type": "code", "execution_count": 9, "metadata": { "cellView": "form", "colab": { "base_uri": "https://localhost:8080/" }, "id": "odb1bMxjvO87", "outputId": "4f87fe80-128d-4ff2-8fef-a787a8770a51" }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Using AdamW8bit as Optimizer\n", "Learning rate: 2e-06\n", "Learning rate warmup steps: 0\n", "Learning rate Scheduler: constant\n" ] } ], "source": [ "# @title ## 5.3. Optimizer Config\n", "from IPython.utils import capture\n", "\n", "# @markdown `NEW` Gamma for reducing the weight of high-loss timesteps. Lower numbers have a stronger effect. The paper recommends 5. Read the paper [here](https://arxiv.org/abs/2303.09556).\n", "min_snr_gamma = -1 #@param {type:\"number\"}\n", "# @markdown `AdamW8bit` was the old `--use_8bit_adam`.\n", "optimizer_type = \"AdamW8bit\" # @param [\"AdamW\", \"AdamW8bit\", \"Lion\", \"SGDNesterov\", \"SGDNesterov8bit\", \"DAdaptation\", \"AdaFactor\"]\n", "# @markdown Additional arguments for optimizer, e.g: `[\"decouple=true\",\"weight_decay=0.6\"]`\n", "optimizer_args = \"\" # @param {'type':'string'}\n", "# @markdown Set `learning_rate` to `1.0` if you use `DAdaptation` optimizer, as it's a [free learning rate](https://github.com/facebookresearch/dadaptation) algorithm.\n", "# @markdown You probably need to specify `optimizer_args` for custom optimizer, like using `[\"decouple=true\",\"weight_decay=0.6\"]` for `DAdaptation`.\n", "learning_rate = 2e-6 # @param {'type':'number'}\n", "train_text_encoder = False # @param {'type':'boolean'}\n", "lr_scheduler = \"constant\" # @param [\"linear\", \"cosine\", \"cosine_with_restarts\", \"polynomial\", \"constant\", \"constant_with_warmup\", \"adafactor\"] {allow-input: false}\n", "lr_warmup_steps = 0 # @param {'type':'number'}\n", "# @markdown You can define `num_cycles` value for `cosine_with_restarts` or `power` value for `polynomial` in the field below.\n", "lr_scheduler_num_cycles = 0 # @param {'type':'number'}\n", "lr_scheduler_power = 0 # @param {'type':'number'}\n", "\n", "print(f\" - Min-SNR Weighting: {min_snr_gamma}\") if not min_snr_gamma == -1 else \"\"\n", "print(f\"Using {optimizer_type} as Optimizer\")\n", "if optimizer_args:\n", " print(f\"Optimizer Args :\", optimizer_args)\n", "print(\"Learning rate: \", learning_rate)\n", "if train_text_encoder:\n", " print(f\"Train Text Encoder\")\n", "print(\"Learning rate warmup steps: \", lr_warmup_steps)\n", "print(\"Learning rate Scheduler:\", lr_scheduler)\n", "if lr_scheduler == \"cosine_with_restarts\":\n", " print(\"- lr_scheduler_num_cycles: \", lr_scheduler_num_cycles)\n", "elif lr_scheduler == \"polynomial\":\n", " print(\"- lr_scheduler_power: \", lr_scheduler_power)" ] }, { "cell_type": "code", "execution_count": 10, "metadata": { "cellView": "form", "colab": { "base_uri": "https://localhost:8080/" }, "id": "vnobC8A0P2lI", "outputId": "c9a32a25-f647-47d0-af89-6222d7c6d7d9" }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "[model_arguments]\n", "v2 = false\n", "v_parameterization = false\n", "pretrained_model_name_or_path = \"/content/pretrained_model/AnyLoRA.safetensors\"\n", "\n", "[optimizer_arguments]\n", "optimizer_type = \"AdamW8bit\"\n", "learning_rate = 2e-6\n", "max_grad_norm = 1.0\n", "train_text_encoder = false\n", "lr_scheduler = \"constant\"\n", "lr_warmup_steps = 0\n", "\n", "[dataset_arguments]\n", "debug_dataset = false\n", "in_json = \"/content/fine_tune/meta_lat.json\"\n", "train_data_dir = \"/content/fine_tune/train_data\"\n", "dataset_repeats = 10\n", "shuffle_caption = true\n", "keep_tokens = 0\n", "resolution = \"256,256\"\n", "caption_dropout_rate = 0\n", "caption_tag_dropout_rate = 0\n", "caption_dropout_every_n_epochs = 0\n", "color_aug = false\n", "token_warmup_min = 1\n", "token_warmup_step = 0\n", "\n", "[training_arguments]\n", "output_dir = \"/content/fine_tune/output\"\n", "output_name = \"rhnd\"\n", "save_precision = \"fp16\"\n", "save_n_epoch_ratio = 1\n", "save_state = false\n", "train_batch_size = 1\n", "max_token_length = 225\n", "mem_eff_attn = false\n", "xformers = true\n", "max_train_steps = 2500\n", "max_data_loader_n_workers = 8\n", "persistent_data_loader_workers = true\n", "gradient_checkpointing = false\n", "gradient_accumulation_steps = 1\n", "mixed_precision = \"fp16\"\n", "clip_skip = 2\n", "logging_dir = \"/content/fine_tune/logs\"\n", "log_prefix = \"rhnd\"\n", "\n", "[sample_prompt_arguments]\n", "sample_every_n_steps = 100\n", "sample_sampler = \"ddim\"\n", "\n", "[saving_arguments]\n", "save_model_as = \"ckpt\"\n", "\n" ] } ], "source": [ "# @title ## 5.4. Training Config\n", "import toml\n", "import os\n", "\n", "%store -r\n", "enable_sample_prompt = True # @param {type:\"boolean\"}\n", "sampler = \"ddim\" # @param [\"ddim\", \"pndm\", \"lms\", \"euler\", \"euler_a\", \"heun\", \"dpm_2\", \"dpm_2_a\", \"dpmsolver\",\"dpmsolver++\", \"dpmsingle\", \"k_lms\", \"k_euler\", \"k_euler_a\", \"k_dpm_2\", \"k_dpm_2_a\"]\n", "noise_offset = 0.0 # @param {type:\"number\"}\n", "max_train_steps = 2500 # @param {type:\"number\"}\n", "train_batch_size = 1 # @param {type:\"number\"}\n", "mixed_precision = \"fp16\" # @param [\"no\",\"fp16\",\"bf16\"] {allow-input: false}\n", "save_state = False # @param {type:\"boolean\"}\n", "save_precision = \"fp16\" # @param [\"float\", \"fp16\", \"bf16\"] {allow-input: false}\n", "save_n_epoch_ratio = 1 # @param {type:\"number\"}\n", "save_model_as = \"ckpt\" # @param [\"ckpt\", \"safetensors\", \"diffusers\", \"diffusers_safetensors\"] {allow-input: false}\n", "max_token_length = 225 # @param {type:\"number\"}\n", "clip_skip = 2 # @param {type:\"number\"}\n", "gradient_checkpointing = False # @param {type:\"boolean\"}\n", "gradient_accumulation_steps = 1 # @param {type:\"number\"}\n", "seed = -1 # @param {type:\"number\"}\n", "logging_dir = \"/content/fine_tune/logs\"\n", "prior_loss_weight = 1.0\n", "\n", "os.chdir(repo_dir)\n", "\n", "sample_str = f\"\"\"\n", " rhnd, open, hand, human hand, good anatomy, five fingers, good quality, plain background \\\n", " --n extra fingers, missing fingers, bad anatomy, lowres, bad hands, cropped, worst quality, low quality, jpeg artifacts, signature, watermark, username, blurry \\\n", " --w 256 \\\n", " --h 256 \\\n", " --l 7 \\\n", " --s 28\n", "\"\"\"\n", "\n", "config = {\n", " \"model_arguments\": {\n", " \"v2\": v2,\n", " \"v_parameterization\": v_parameterization if v2 and v_parameterization else False,\n", " \"pretrained_model_name_or_path\": pretrained_model_name_or_path,\n", " \"vae\": vae,\n", " },\n", " \"optimizer_arguments\": {\n", " \"min_snr_gamma\": min_snr_gamma if not min_snr_gamma == -1 else None,\n", " \"optimizer_type\": optimizer_type,\n", " \"learning_rate\": learning_rate,\n", " \"max_grad_norm\": 1.0,\n", " \"train_text_encoder\": train_text_encoder,\n", " \"optimizer_args\": eval(optimizer_args) if optimizer_args else None,\n", " \"lr_scheduler\": lr_scheduler,\n", " \"lr_warmup_steps\": lr_warmup_steps,\n", " \"lr_scheduler_num_cycles\": lr_scheduler_num_cycles if lr_scheduler == \"cosine_with_restarts\" else None,\n", " \"lr_scheduler_power\": lr_scheduler_power if lr_scheduler == \"polynomial\" else None,\n", " },\n", " \"dataset_arguments\": {\n", " \"debug_dataset\": False,\n", " \"in_json\": in_json,\n", " \"train_data_dir\": train_data_dir,\n", " \"dataset_repeats\": dataset_repeats,\n", " \"shuffle_caption\": True,\n", " \"keep_tokens\": keep_tokens,\n", " \"resolution\": resolution,\n", " \"caption_dropout_rate\": 0,\n", " \"caption_tag_dropout_rate\": 0,\n", " \"caption_dropout_every_n_epochs\": 0,\n", " \"color_aug\": False,\n", " \"face_crop_aug_range\": None,\n", " \"token_warmup_min\": 1,\n", " \"token_warmup_step\": 0,\n", " },\n", " \"training_arguments\": {\n", " \"output_dir\": output_dir,\n", " \"output_name\": project_name,\n", " \"save_precision\": save_precision,\n", " \"save_every_n_epochs\": None,\n", " \"save_n_epoch_ratio\": save_n_epoch_ratio,\n", " \"save_last_n_epochs\": None,\n", " \"save_state\": save_state,\n", " \"save_last_n_epochs_state\": None,\n", " \"resume\": resume_path,\n", " \"train_batch_size\": train_batch_size,\n", " \"max_token_length\": 225,\n", " \"mem_eff_attn\": False,\n", " \"xformers\": True,\n", " \"max_train_steps\": max_train_steps,\n", " \"max_data_loader_n_workers\": 8,\n", " \"persistent_data_loader_workers\": True,\n", " \"seed\": seed if seed > 0 else None,\n", " \"gradient_checkpointing\": gradient_checkpointing,\n", " \"gradient_accumulation_steps\": gradient_accumulation_steps,\n", " \"mixed_precision\": mixed_precision,\n", " \"clip_skip\": clip_skip if not v2 else None,\n", " \"logging_dir\": logging_dir,\n", " \"log_prefix\": project_name,\n", " \"noise_offset\": noise_offset if noise_offset > 0 else None,\n", " },\n", " \"sample_prompt_arguments\": {\n", " \"sample_every_n_steps\": 100 if enable_sample_prompt else 999999,\n", " \"sample_every_n_epochs\": None,\n", " \"sample_sampler\": sampler,\n", " },\n", " \"saving_arguments\": {\n", " \"save_model_as\": save_model_as\n", " },\n", "}\n", "\n", "config_path = os.path.join(config_dir, \"config_file.toml\")\n", "prompt_path = os.path.join(config_dir, \"sample_prompt.txt\")\n", "\n", "for key in config:\n", " if isinstance(config[key], dict):\n", " for sub_key in config[key]:\n", " if config[key][sub_key] == \"\":\n", " config[key][sub_key] = None\n", " elif config[key] == \"\":\n", " config[key] = None\n", "\n", "config_str = toml.dumps(config)\n", "\n", "def write_file(filename, contents):\n", " with open(filename, \"w\") as f:\n", " f.write(contents)\n", "\n", "write_file(config_path, config_str)\n", "write_file(prompt_path, sample_str)\n", "\n", "print(config_str)" ] }, { "cell_type": "code", "execution_count": 11, "metadata": { "cellView": "form", "colab": { "base_uri": "https://localhost:8080/" }, "id": "DNYGVTkaiQPf", "outputId": "3c72b509-29b2-4b5e-a92a-ce29f67a5ca0" }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "CUDA backend failed to initialize: Found CUDA version 12010, but JAX was built against version 12020, which is newer. The copy of CUDA that is installed must be at least as new as the version against which JAX was built. (Set TF_CPP_MIN_LOG_LEVEL=0 and rerun for more info.)\n", "Loading settings from /content/fine_tune/config/config_file.toml...\n", "/content/fine_tune/config/config_file\n", "prepare tokenizer\n", "Downloading vocab.json: 100% 961k/961k [00:00<00:00, 7.39MB/s]\n", "Downloading merges.txt: 100% 525k/525k [00:00<00:00, 14.4MB/s]\n", "Downloading (…)cial_tokens_map.json: 100% 389/389 [00:00<00:00, 2.52MB/s]\n", "Downloading tokenizer_config.json: 100% 905/905 [00:00<00:00, 5.79MB/s]\n", "update token length: 225\n", "loading existing metadata: /content/fine_tune/meta_lat.json\n", "metadata has bucket info, enable bucketing / メタデータにbucket情報があるためbucketを有効にします\n", "using bucket info in metadata / メタデータ内のbucket情報を使います\n", "[Dataset 0]\n", " batch_size: 1\n", " resolution: (256, 256)\n", " enable_bucket: True\n", " min_bucket_reso: None\n", " max_bucket_reso: None\n", " bucket_reso_steps: None\n", " bucket_no_upscale: None\n", "\n", " [Subset 0 of Dataset 0]\n", " image_dir: \"/content/fine_tune/train_data\"\n", " image_count: 73\n", " num_repeats: 10\n", " shuffle_caption: True\n", " keep_tokens: 0\n", " caption_dropout_rate: 0\n", " caption_dropout_every_n_epoches: 0\n", " caption_tag_dropout_rate: 0\n", " color_aug: False\n", " flip_aug: False\n", " face_crop_aug_range: None\n", " random_crop: False\n", " token_warmup_min: 1,\n", " token_warmup_step: 0,\n", " metadata_file: /content/fine_tune/meta_lat.json\n", "\n", "\n", "[Dataset 0]\n", "loading image sizes.\n", "100% 73/73 [00:00<00:00, 1020613.97it/s]\n", "make buckets\n", "number of images (including repeats) / 各bucketの画像枚数（繰り返し回数を含む）\n", "bucket 0: resolution (256, 256), count: 730\n", "mean ar error (without repeats): 0.0\n", "prepare accelerator\n", "Using accelerator 0.15.0 or above.\n", "load StableDiffusion checkpoint\n", "loading u-net: \n", "loading vae: \n", "Downloading config.json: 100% 4.52k/4.52k [00:00<00:00, 21.1MB/s]\n", "Downloading model.safetensors: 100% 1.71G/1.71G [00:21<00:00, 79.1MB/s]\n", "loading text encoder: \n", "Disable Diffusers' xformers\n", "Replace CrossAttention.forward to use xformers\n", "prepare optimizer, data loader etc.\n", "use 8-bit AdamW optimizer | {}\n", "running training / 学習開始\n", " num examples / サンプル数: 730\n", " num batches per epoch / 1epochのバッチ数: 730\n", " num epochs / epoch数: 4\n", " batch size per device / バッチサイズ: 1\n", " total train batch size (with parallel & distributed & accumulation) / 総バッチサイズ（並列学習、勾配合計含む）: 1\n", " gradient accumulation steps / 勾配を合計するステップ数 = 1\n", " total optimization steps / 学習ステップ数: 2500\n", "steps: 0% 0/2500 [00:00 Get your `ngrok_token` [here](https://dashboard.ngrok.com/get-started/your-authtoken)\n", " ngrok_token = \"\" #@param {type: 'string'}\n", " ngrok_region = \"ap\" #@param [\"us\", \"eu\", \"au\", \"ap\", \"sa\", \"jp\", \"in\"]\n", "\n", " with capture.capture_output() as cap:\n", " for file in os.listdir(output_dir):\n", " file_path = os.path.join(output_dir, file)\n", " if file_path.endswith((\".safetensors\", \".ckpt\")):\n", " !ln \"{file_path}\" {webui_models_dir}\n", "\n", " for file in os.listdir(pretrained_model):\n", " file_path = os.path.join(pretrained_model, file)\n", " if file_path.endswith((\".safetensors\", \".ckpt\")):\n", " !ln \"{file_path}\" {webui_models_dir}\n", "\n", " for file in os.listdir(vae_dir):\n", " file_path = os.path.join(vae_dir, file)\n", " if file_path.endswith(\".vae.pt\"):\n", " !ln \"{file_path}\" {webui_vaes_dir}\n", "\n", " del cap\n", " model_path = os.path.join(webui_models_dir, project_name + \".\" + save_model_as)\n", "\n", " os.chdir(webui_dir)\n", "\n", " print(\"\u001b[1;32m\")\n", "\n", " config = {\n", " \"enable-insecure-extension-access\": True,\n", " \"disable-safe-unpickle\": True,\n", " \"multiple\": True if not ngrok_token else False,\n", " \"ckpt\": model_path if os.path.exists(model_path) else None,\n", " \"ckpt-dir\": webui_models_dir,\n", " \"vae-dir\": webui_vaes_dir,\n", " \"share\": True if not ngrok_token else False,\n", " \"no-half-vae\": True,\n", " \"lowram\": True,\n", " \"gradio-queue\": True,\n", " \"no-hashing\": True,\n", " \"disable-console-progressbars\": True,\n", " \"ngrok\": ngrok_token if ngrok_token else None,\n", " \"ngrok-region\": ngrok_region if ngrok_token else None,\n", " \"xformers\": True,\n", " \"opt-sub-quad-attention\": True,\n", " \"opt-channelslast\": True,\n", " \"theme\": \"dark\"\n", " }\n", "\n", " args = \"\"\n", " for k, v in config.items():\n", " if k.startswith(\"_\"):\n", " args += f'\"{v}\" '\n", " elif isinstance(v, str):\n", " args += f'--{k}=\"{v}\" '\n", " elif isinstance(v, bool) and v:\n", " args += f\"--{k} \"\n", " elif isinstance(v, float) and not isinstance(v, bool):\n", " args += f\"--{k}={v} \"\n", " elif isinstance(v, int) and not isinstance(v, bool):\n", " args += f\"--{k}={v} \"\n", "\n", " final_args = f\"python launch.py {args}\"\n", "\n", " os.chdir(webui_dir)\n", " !{final_args}\n", "\n", "main()" ] }, { "cell_type": "markdown", "metadata": { "id": "N6ckzE2GWudi" }, "source": [ "# VII. Extras" ] }, { "cell_type": "markdown", "metadata": { "id": "nyIl9BhNXKUq" }, "source": [ "# VIII. Deployment" ] }, { "cell_type": "code", "execution_count": 12, "metadata": { "cellView": "form", "colab": { "base_uri": "https://localhost:8080/" }, "id": "QTXsM170GUpk", "outputId": "53996ad1-757c-4670-b98f-994e05a516eb" }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Token is valid.\n", "Your token has been saved in your configured git credential helpers (store).\n", "Your token has been saved to /root/.cache/huggingface/token\n", "Login successful\n", "Model repo 'sarahahatee/AnyLoRA' didn't exist, creating repo\n", "Model repo 'sarahahatee/AnyLoRA' link: https://huggingface.co/sarahahatee/AnyLoRA\n", "\n", "Dataset repo 'sarahahatee/rhnd' didn't exist, creating repo\n", "Dataset repo 'sarahahatee/rhnd' link: https://huggingface.co/datasets/sarahahatee/rhnd\n", "\n" ] } ], "source": [ "# @title ## 7.1. Upload Config\n", "from huggingface_hub import login\n", "from huggingface_hub import HfApi\n", "from huggingface_hub.utils import validate_repo_id, HfHubHTTPError\n", "\n", "# @markdown Login to Huggingface Hub\n", "# @markdown > Get **your** huggingface `WRITE` token [here](https://huggingface.co/settings/tokens)\n", "write_token = \"token\" # @param {type:\"string\"}\n", "# @markdown Fill this if you want to upload to your organization, or just leave it empty.\n", "orgs_name = \"\" # @param{type:\"string\"}\n", "# @markdown If your model/dataset repo does not exist, it will automatically create it.\n", "model_name = \"AnyLoRA\" # @param{type:\"string\"}\n", "dataset_name = \"rhnd\" # @param{type:\"string\"}\n", "make_private = False # @param{type:\"boolean\"}\n", "\n", "def authenticate(write_token):\n", " login(write_token, add_to_git_credential=True)\n", " api = HfApi()\n", " return api.whoami(write_token), api\n", "\n", "\n", "def create_repo(api, user, orgs_name, repo_name, repo_type, make_private=False):\n", " global model_repo\n", " global datasets_repo\n", "\n", " if orgs_name == \"\":\n", " repo_id = user[\"name\"] + \"/\" + repo_name.strip()\n", " else:\n", " repo_id = orgs_name + \"/\" + repo_name.strip()\n", "\n", " try:\n", " validate_repo_id(repo_id)\n", " api.create_repo(repo_id=repo_id, repo_type=repo_type, private=make_private)\n", " print(f\"{repo_type.capitalize()} repo '{repo_id}' didn't exist, creating repo\")\n", " except HfHubHTTPError as e:\n", " print(f\"{repo_type.capitalize()} repo '{repo_id}' exists, skipping create repo\")\n", "\n", " if repo_type == \"model\":\n", " model_repo = repo_id\n", " print(f\"{repo_type.capitalize()} repo '{repo_id}' link: https://huggingface.co/{repo_id}\\n\")\n", " else:\n", " datasets_repo = repo_id\n", " print(f\"{repo_type.capitalize()} repo '{repo_id}' link: https://huggingface.co/datasets/{repo_id}\\n\")\n", "\n", "user, api = authenticate(write_token)\n", "\n", "if model_name:\n", " create_repo(api, user, orgs_name, model_name, \"model\", make_private)\n", "if dataset_name:\n", " create_repo(api, user, orgs_name, dataset_name, \"dataset\", make_private)\n" ] }, { "cell_type": "markdown", "metadata": { "id": "Fuxghk8MnG6j" }, "source": [ "## 8.2. Upload with Huggingface Hub" ] }, { "cell_type": "code", "execution_count": 13, "metadata": { "cellView": "form", "colab": { "base_uri": "https://localhost:8080/", "height": 220, "referenced_widgets": [ "09f8a67ee38442e0bfc253f89ea9a9ca", "5c9b0571cb574b4eb66ee707e35b1ac6", "350eb462ded345528d65ba81ddb2b483", "974b75acc3e8493bba6fb36fcb6343eb", "4f78ef991080443989003cdcb047979b", "39fa41a3974244a9b4d2e0009400dfa0", "28e499c21333423fa60f8419b035bb01", "51d84d45d743466dafc95483c358f610", "e9b91e1b6de144478663b5541949e327", "eaad12614efc495da35e69ce0fbbc3e8", "a0be6aeedb7f4f029a55a7d9d7c71f60", "0c2e92c49b6e4b889cee8e8a53156b3c", "c741706290104712bb0cc76494bf9293", "e16294b552764e1ab7a9463b3e2ecc8f", "bfe58e44e06c4e63ab8f0b1eb45bb4c6", "d2a60dea38a2411d969e73c73f5a8a27", "0b838eaf358644f39f595dc43f4bdf0d", "f4a24df6f2924e3098b6efc1aedfd9da", "5457318dd53848e1ad6000f4e0275490", "e9eef9324dd24585b490d2272baf269d", "c2833ca0b044431c99a2c0534f160659", "007eec94ce8c49e1b8c8afa1f444bcea" ] }, "id": "CIeoJA-eO-8t", "outputId": "2d1f53a1-ce56-4b36-d575-fbad554c3604" }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Uploading output to https://huggingface.co/sarahahatee/AnyLoRA\n", "Please wait...\n" ] }, { "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "09f8a67ee38442e0bfc253f89ea9a9ca", "version_major": 2, "version_minor": 0 }, "text/plain": [ "rhnd.ckpt: 0%| | 0.00/2.13G [00:00