clean up

.gitattributes

@@ -36,3 +36,5 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 cat.png filter=lfs diff=lfs merge=lfs -text
 autoencoder_kl.ckpt filter=lfs diff=lfs merge=lfs -text
 clip_encoder.ckpt filter=lfs diff=lfs merge=lfs -text
+Checkpoint/ filter=lfs diff=lfs merge=lfs -text
+Checkpoint/*.ckpt filter=lfs diff=lfs merge=lfs -text

.gitignore

@@ -215,3 +215,5 @@ AppPackages/
 # End of core ignore list, below put you custom 'per project' settings (patterns or path)
 #####
 
+Output/
+

ClipTokenizer.cs

@@ -0,0 +1,59 @@
+using Microsoft.ML.Tokenizers;
+using System;
+using System.Collections.Generic;
+using System.IO;
+using System.Linq;
+using System.Net;
+using System.Net.Http;
+using System.Text;
+using System.Threading.Tasks;
+
+namespace StableDiffusionV2
+{
+    internal class ClipTokenizer
+    {
+        private readonly Tokenizer _tokenizer;
+        private readonly int _startToken;
+        private readonly int _endToken;
+
+        public ClipTokenizer(string vocabPath, string mergesPath, int startToken = 49406, int endToken = 49407)
+        {
+            if(!File.Exists(vocabPath) || !File.Exists(mergesPath))
+            {
+                // download vocab from https://huggingface.co/laion/CLIP-ViT-H-14-laion2B-s32B-b79K/raw/main/vocab.json
+                // download merges from https://huggingface.co/laion/CLIP-ViT-H-14-laion2B-s32B-b79K/blob/main/merges.txt
+
+                using (var client = new HttpClient())
+                {
+                    Console.WriteLine("download vocab.json and merges.txt");
+                    using (var s = client.GetStringAsync("https://huggingface.co/laion/CLIP-ViT-H-14-laion2B-s32B-b79K/raw/main/vocab.json"))
+                    {
+                        using (var fs = new StreamWriter(vocabPath))
+                        {
+                            fs.Write(s.Result);
+                        }
+                    }
+
+                    using (var s = client.GetStringAsync("https://huggingface.co/laion/CLIP-ViT-H-14-laion2B-s32B-b79K/raw/main/merges.txt"))
+                    {
+                        using (var fs = new StreamWriter(mergesPath))
+                        {
+                            fs.Write(s.Result);
+                        }
+                    }
+                }
+            }
+
+            _tokenizer = new Tokenizer(new Bpe(vocabPath, mergesPath, endOfWordSuffix: "</w>"));
+            _startToken = startToken;
+            _endToken = endToken;
+        }
+
+        public int[] Tokenize(string text, int maxTokens = 77)
+        {
+            var res = _tokenizer.Encode(text);
+            var tokens = new[] { _startToken }.Concat(res.Ids.Concat(Enumerable.Repeat(0, maxTokens - res.Ids.Count - 2))).Concat(new[] { _endToken }).ToArray();
+            return tokens;
+        }
+    }
+}

Program.cs

@@ -1,48 +1,48 @@
-using Microsoft.ML;
-using Microsoft.ML.Tokenizers;
+using StableDiffusionV2;
 using System;
-using System.Collections.Generic;
 using System.IO;
-using System.Linq;
 using TorchSharp;
 
 var batch = 1;
-var bpe = new Bpe("vocab.json", "merges.txt", endOfWordSuffix: "</w>");
-var tokenier = new Tokenizer(bpe);
-var start_token = 49406;
-var end_token = 49407;
+var device = torch.device("cuda:0");
+torchvision.io.DefaultImager = new torchvision.io.SkiaImager();
 var prompt = "a wild cute green cat";
-var res = tokenier.Encode(prompt);
-var tokens = new[] { start_token }.Concat(res.Ids.Concat(Enumerable.Repeat(0, 75 - res.Ids.Count))).Concat(new[] { end_token }).ToList();
-var uncontional_tokens = new[]{start_token, end_token}.Concat(Enumerable.Repeat(0, 75)).ToList();
-var tokenTensor = torch.tensor(tokens.ToArray(), dtype: torch.ScalarType.Int64, device: device);
+var outputFolder = "Output";
+if(!Directory.Exists(outputFolder))
+{
+    Directory.CreateDirectory(outputFolder);
+}
+
+var clipTokenizer = new ClipTokenizer("vocab.json", "merges.txt");
+var tokens = clipTokenizer.Tokenize(prompt);
+var uncontional_tokens = clipTokenizer.Tokenize("");
+var tokenTensor = torch.tensor(tokens, dtype: torch.ScalarType.Int64, device: device);
+var unconditional_tokenTensor = torch.tensor(uncontional_tokens, dtype: torch.ScalarType.Int64, device: device);
 tokenTensor = tokenTensor.repeat(batch, 1);
-var unconditional_tokenTensor = torch.tensor(uncontional_tokens.ToArray(), dtype: torch.ScalarType.Int64, device: device);
 unconditional_tokenTensor = unconditional_tokenTensor.repeat(batch, 1);
 
-torchvision.io.DefaultImager = new torchvision.io.SkiaImager();
-var device = TorchSharp.torch.device("cuda:0");
 var clipEncoder = new ClipEncoder("clip_encoder.ckpt", device);
 var img = torch.randn(batch, 4, 64, 64, dtype: torch.ScalarType.Float32, device: device);
-var t = torch.full(new[]{batch, 1L}, value: batch, dtype: torch.ScalarType.Int32, device: device);
 var condition = clipEncoder.Forward(tokenTensor);
 var unconditional_condition = clipEncoder.Forward(unconditional_tokenTensor);
-
 clipEncoder.Dispose();
+
 var ddpm = new DDPM("ddim_v_sampler.ckpt", device);
 var ddimSampler = new DDIMSampler(ddpm);
 var ddim_steps = 50;
 img = ddimSampler.Sample(img, condition, unconditional_condition, ddim_steps);
 ddpm.Dispose();
 
-var autoencoderKL = new AutoencoderKL("autoencoder_kl.ckpt", device);
-var decoded_images = (torch.Tensor)autoencoderKL.Forward(img);
+var vae = new AutoencoderKL("autoencoder_kl.ckpt", device);
+var decoded_images = vae.Forward(img);
 decoded_images = torch.clamp((decoded_images + 1.0) / 2.0, 0.0, 1.0);
 
-
 for(int i = 0; i!= batch; ++i)
 {
+    var savedPath = Path.Join(outputFolder, $"{i}.png");
     var image = decoded_images[i];
     image = (image * 255.0).to(torch.ScalarType.Byte).cpu();
-    torchvision.io.write_image(image, $"{i}.png", torchvision.ImageFormat.Png);
+    torchvision.io.write_image(image, savedPath, torchvision.ImageFormat.Png);
+
+    Console.WriteLine($"save image to {savedPath}, enjoy");
 }

README.md

@@ -16,7 +16,7 @@ This project serves as a proof-of-concept purpose of bringing StableDiffusion mo
 
 # Example output
 
-![a wild cute green cat](cat.png)
+![a wild cute green cat](Asset/cat.png)
 *a wild cute green cat*
 ---
 license: mit

clip.csproj → StableDiffusionV2.csproj

@@ -1,27 +1,22 @@
-<Project Sdk="Microsoft.NET.Sdk">
-
-  <PropertyGroup>
-    <OutputType>Exe</OutputType>
-    <TargetFramework>net6.0</TargetFramework>
-    <Nullable>enable</Nullable>
-    <Platform>x64</Platform>
-    <TorchVersion>0.99.3</TorchVersion>
-  </PropertyGroup>
-
-  <ItemGroup>
-    <PackageReference Include="Microsoft.ML" Version="2.0.1" />
-    <PackageReference Include="Microsoft.ML.Tokenizers" Version="0.20.1" />
-    <PackageReference Include="TorchVision" Version="$(TorchVersion)" />
-    <PackageReference Include="TorchSharp-cuda-linux" Version="$(TorchVersion)" />
-    <None Update="*.ckpt">
-      <CopyToOutputDirectory>Always</CopyToOutputDirectory>
-    </None>
-    <None Update="merges.txt">
-      <CopyToOutputDirectory>Always</CopyToOutputDirectory>
-    </None>
-    <None Update="vocab.json">
-      <CopyToOutputDirectory>Always</CopyToOutputDirectory>
-    </None>
-  </ItemGroup>
-
-</Project>
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <PropertyGroup>
+    <OutputType>Exe</OutputType>
+    <TargetFramework>net6.0</TargetFramework>
+    <Nullable>enable</Nullable>
+    <Platform>x64</Platform>
+    <TorchVersion>0.99.3</TorchVersion>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <PackageReference Include="Microsoft.ML" Version="2.0.1" />
+    <PackageReference Include="Microsoft.ML.Tokenizers" Version="0.20.1" />
+    <PackageReference Include="TorchVision" Version="$(TorchVersion)" />
+	<PackageReference Condition="'$(OS)' == 'Windows_NT'" Include="TorchSharp-cuda-windows" Version="$(TorchVersion)" />
+	<PackageReference Condition="'$(OS)' == 'Linux'" Include="TorchSharp-cuda-linux" Version="$(TorchVersion)" />
+    <None Update="Checkpoint/**/*">
+        <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
+    </None>
+  </ItemGroup>
+
+</Project>