fffiloni guoyww commited on
Commit
3ef15f9
0 Parent(s):

Duplicate from guoyww/AnimateDiff

Browse files

Co-authored-by: Yuwei Guo <[email protected]>

This view is limited to 50 files because it contains too many changes.   See raw diff
Files changed (50) hide show
  1. .gitattributes +35 -0
  2. .gitignore +2 -0
  3. LICENSE.txt +201 -0
  4. README.md +14 -0
  5. animatediff/models/attention.py +300 -0
  6. animatediff/models/motion_module.py +331 -0
  7. animatediff/models/resnet.py +197 -0
  8. animatediff/models/unet.py +489 -0
  9. animatediff/models/unet_blocks.py +733 -0
  10. animatediff/pipelines/pipeline_animation.py +428 -0
  11. animatediff/utils/convert_from_ckpt.py +959 -0
  12. animatediff/utils/convert_lora_safetensor_to_diffusers.py +128 -0
  13. animatediff/utils/util.py +84 -0
  14. app.py +294 -0
  15. configs/inference/inference.yaml +26 -0
  16. configs/prompts/1-ToonYou.yaml +22 -0
  17. configs/prompts/2-Lyriel.yaml +22 -0
  18. configs/prompts/3-RcnzCartoon.yaml +22 -0
  19. configs/prompts/4-MajicMix.yaml +22 -0
  20. configs/prompts/5-RealisticVision.yaml +22 -0
  21. configs/prompts/6-Tusun.yaml +20 -0
  22. configs/prompts/7-FilmVelvia.yaml +23 -0
  23. configs/prompts/8-GhibliBackground.yaml +20 -0
  24. download_bashscripts/0-MotionModule.sh +2 -0
  25. download_bashscripts/1-ToonYou.sh +2 -0
  26. download_bashscripts/2-Lyriel.sh +2 -0
  27. download_bashscripts/3-RcnzCartoon.sh +2 -0
  28. download_bashscripts/4-MajicMix.sh +2 -0
  29. download_bashscripts/5-RealisticVision.sh +2 -0
  30. download_bashscripts/6-Tusun.sh +3 -0
  31. download_bashscripts/7-FilmVelvia.sh +3 -0
  32. download_bashscripts/8-GhibliBackground.sh +3 -0
  33. models/DreamBooth_LoRA/Put personalized T2I checkpoints here.txt +0 -0
  34. models/DreamBooth_LoRA/lyriel_v16.safetensors +3 -0
  35. models/DreamBooth_LoRA/majicmixRealistic_v5Preview.safetensors +3 -0
  36. models/DreamBooth_LoRA/rcnzCartoon3d_v10.safetensors +3 -0
  37. models/DreamBooth_LoRA/realisticVisionV20_v20.safetensors +3 -0
  38. models/DreamBooth_LoRA/toonyou_beta3.safetensors +3 -0
  39. models/Motion_Module/Put motion module checkpoints here.txt +0 -0
  40. models/Motion_Module/mm_sd_v14.ckpt +3 -0
  41. models/Motion_Module/mm_sd_v15.ckpt +3 -0
  42. models/StableDiffusion/Put diffusers stable-diffusion-v1-5 repo here.txt +0 -0
  43. models/StableDiffusion/stable-diffusion-v1-5/.gitattributes +35 -0
  44. models/StableDiffusion/stable-diffusion-v1-5/README.md +207 -0
  45. models/StableDiffusion/stable-diffusion-v1-5/feature_extractor/preprocessor_config.json +20 -0
  46. models/StableDiffusion/stable-diffusion-v1-5/model_index.json +32 -0
  47. models/StableDiffusion/stable-diffusion-v1-5/safety_checker/config.json +175 -0
  48. models/StableDiffusion/stable-diffusion-v1-5/scheduler/scheduler_config.json +13 -0
  49. models/StableDiffusion/stable-diffusion-v1-5/text_encoder/config.json +25 -0
  50. models/StableDiffusion/stable-diffusion-v1-5/text_encoder/pytorch_model.bin +3 -0
.gitattributes ADDED
@@ -0,0 +1,35 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ *.7z filter=lfs diff=lfs merge=lfs -text
2
+ *.arrow filter=lfs diff=lfs merge=lfs -text
3
+ *.bin filter=lfs diff=lfs merge=lfs -text
4
+ *.bz2 filter=lfs diff=lfs merge=lfs -text
5
+ *.ckpt filter=lfs diff=lfs merge=lfs -text
6
+ *.ftz filter=lfs diff=lfs merge=lfs -text
7
+ *.gz filter=lfs diff=lfs merge=lfs -text
8
+ *.h5 filter=lfs diff=lfs merge=lfs -text
9
+ *.joblib filter=lfs diff=lfs merge=lfs -text
10
+ *.lfs.* filter=lfs diff=lfs merge=lfs -text
11
+ *.mlmodel filter=lfs diff=lfs merge=lfs -text
12
+ *.model filter=lfs diff=lfs merge=lfs -text
13
+ *.msgpack filter=lfs diff=lfs merge=lfs -text
14
+ *.npy filter=lfs diff=lfs merge=lfs -text
15
+ *.npz filter=lfs diff=lfs merge=lfs -text
16
+ *.onnx filter=lfs diff=lfs merge=lfs -text
17
+ *.ot filter=lfs diff=lfs merge=lfs -text
18
+ *.parquet filter=lfs diff=lfs merge=lfs -text
19
+ *.pb filter=lfs diff=lfs merge=lfs -text
20
+ *.pickle filter=lfs diff=lfs merge=lfs -text
21
+ *.pkl filter=lfs diff=lfs merge=lfs -text
22
+ *.pt filter=lfs diff=lfs merge=lfs -text
23
+ *.pth filter=lfs diff=lfs merge=lfs -text
24
+ *.rar filter=lfs diff=lfs merge=lfs -text
25
+ *.safetensors filter=lfs diff=lfs merge=lfs -text
26
+ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
27
+ *.tar.* filter=lfs diff=lfs merge=lfs -text
28
+ *.tar filter=lfs diff=lfs merge=lfs -text
29
+ *.tflite filter=lfs diff=lfs merge=lfs -text
30
+ *.tgz filter=lfs diff=lfs merge=lfs -text
31
+ *.wasm filter=lfs diff=lfs merge=lfs -text
32
+ *.xz filter=lfs diff=lfs merge=lfs -text
33
+ *.zip filter=lfs diff=lfs merge=lfs -text
34
+ *.zst filter=lfs diff=lfs merge=lfs -text
35
+ *tfevents* filter=lfs diff=lfs merge=lfs -text
.gitignore ADDED
@@ -0,0 +1,2 @@
 
 
 
1
+ models/
2
+ __pycache__/
LICENSE.txt ADDED
@@ -0,0 +1,201 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ Apache License
2
+ Version 2.0, January 2004
3
+ http://www.apache.org/licenses/
4
+
5
+ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
6
+
7
+ 1. Definitions.
8
+
9
+ "License" shall mean the terms and conditions for use, reproduction,
10
+ and distribution as defined by Sections 1 through 9 of this document.
11
+
12
+ "Licensor" shall mean the copyright owner or entity authorized by
13
+ the copyright owner that is granting the License.
14
+
15
+ "Legal Entity" shall mean the union of the acting entity and all
16
+ other entities that control, are controlled by, or are under common
17
+ control with that entity. For the purposes of this definition,
18
+ "control" means (i) the power, direct or indirect, to cause the
19
+ direction or management of such entity, whether by contract or
20
+ otherwise, or (ii) ownership of fifty percent (50%) or more of the
21
+ outstanding shares, or (iii) beneficial ownership of such entity.
22
+
23
+ "You" (or "Your") shall mean an individual or Legal Entity
24
+ exercising permissions granted by this License.
25
+
26
+ "Source" form shall mean the preferred form for making modifications,
27
+ including but not limited to software source code, documentation
28
+ source, and configuration files.
29
+
30
+ "Object" form shall mean any form resulting from mechanical
31
+ transformation or translation of a Source form, including but
32
+ not limited to compiled object code, generated documentation,
33
+ and conversions to other media types.
34
+
35
+ "Work" shall mean the work of authorship, whether in Source or
36
+ Object form, made available under the License, as indicated by a
37
+ copyright notice that is included in or attached to the work
38
+ (an example is provided in the Appendix below).
39
+
40
+ "Derivative Works" shall mean any work, whether in Source or Object
41
+ form, that is based on (or derived from) the Work and for which the
42
+ editorial revisions, annotations, elaborations, or other modifications
43
+ represent, as a whole, an original work of authorship. For the purposes
44
+ of this License, Derivative Works shall not include works that remain
45
+ separable from, or merely link (or bind by name) to the interfaces of,
46
+ the Work and Derivative Works thereof.
47
+
48
+ "Contribution" shall mean any work of authorship, including
49
+ the original version of the Work and any modifications or additions
50
+ to that Work or Derivative Works thereof, that is intentionally
51
+ submitted to Licensor for inclusion in the Work by the copyright owner
52
+ or by an individual or Legal Entity authorized to submit on behalf of
53
+ the copyright owner. For the purposes of this definition, "submitted"
54
+ means any form of electronic, verbal, or written communication sent
55
+ to the Licensor or its representatives, including but not limited to
56
+ communication on electronic mailing lists, source code control systems,
57
+ and issue tracking systems that are managed by, or on behalf of, the
58
+ Licensor for the purpose of discussing and improving the Work, but
59
+ excluding communication that is conspicuously marked or otherwise
60
+ designated in writing by the copyright owner as "Not a Contribution."
61
+
62
+ "Contributor" shall mean Licensor and any individual or Legal Entity
63
+ on behalf of whom a Contribution has been received by Licensor and
64
+ subsequently incorporated within the Work.
65
+
66
+ 2. Grant of Copyright License. Subject to the terms and conditions of
67
+ this License, each Contributor hereby grants to You a perpetual,
68
+ worldwide, non-exclusive, no-charge, royalty-free, irrevocable
69
+ copyright license to reproduce, prepare Derivative Works of,
70
+ publicly display, publicly perform, sublicense, and distribute the
71
+ Work and such Derivative Works in Source or Object form.
72
+
73
+ 3. Grant of Patent License. Subject to the terms and conditions of
74
+ this License, each Contributor hereby grants to You a perpetual,
75
+ worldwide, non-exclusive, no-charge, royalty-free, irrevocable
76
+ (except as stated in this section) patent license to make, have made,
77
+ use, offer to sell, sell, import, and otherwise transfer the Work,
78
+ where such license applies only to those patent claims licensable
79
+ by such Contributor that are necessarily infringed by their
80
+ Contribution(s) alone or by combination of their Contribution(s)
81
+ with the Work to which such Contribution(s) was submitted. If You
82
+ institute patent litigation against any entity (including a
83
+ cross-claim or counterclaim in a lawsuit) alleging that the Work
84
+ or a Contribution incorporated within the Work constitutes direct
85
+ or contributory patent infringement, then any patent licenses
86
+ granted to You under this License for that Work shall terminate
87
+ as of the date such litigation is filed.
88
+
89
+ 4. Redistribution. You may reproduce and distribute copies of the
90
+ Work or Derivative Works thereof in any medium, with or without
91
+ modifications, and in Source or Object form, provided that You
92
+ meet the following conditions:
93
+
94
+ (a) You must give any other recipients of the Work or
95
+ Derivative Works a copy of this License; and
96
+
97
+ (b) You must cause any modified files to carry prominent notices
98
+ stating that You changed the files; and
99
+
100
+ (c) You must retain, in the Source form of any Derivative Works
101
+ that You distribute, all copyright, patent, trademark, and
102
+ attribution notices from the Source form of the Work,
103
+ excluding those notices that do not pertain to any part of
104
+ the Derivative Works; and
105
+
106
+ (d) If the Work includes a "NOTICE" text file as part of its
107
+ distribution, then any Derivative Works that You distribute must
108
+ include a readable copy of the attribution notices contained
109
+ within such NOTICE file, excluding those notices that do not
110
+ pertain to any part of the Derivative Works, in at least one
111
+ of the following places: within a NOTICE text file distributed
112
+ as part of the Derivative Works; within the Source form or
113
+ documentation, if provided along with the Derivative Works; or,
114
+ within a display generated by the Derivative Works, if and
115
+ wherever such third-party notices normally appear. The contents
116
+ of the NOTICE file are for informational purposes only and
117
+ do not modify the License. You may add Your own attribution
118
+ notices within Derivative Works that You distribute, alongside
119
+ or as an addendum to the NOTICE text from the Work, provided
120
+ that such additional attribution notices cannot be construed
121
+ as modifying the License.
122
+
123
+ You may add Your own copyright statement to Your modifications and
124
+ may provide additional or different license terms and conditions
125
+ for use, reproduction, or distribution of Your modifications, or
126
+ for any such Derivative Works as a whole, provided Your use,
127
+ reproduction, and distribution of the Work otherwise complies with
128
+ the conditions stated in this License.
129
+
130
+ 5. Submission of Contributions. Unless You explicitly state otherwise,
131
+ any Contribution intentionally submitted for inclusion in the Work
132
+ by You to the Licensor shall be under the terms and conditions of
133
+ this License, without any additional terms or conditions.
134
+ Notwithstanding the above, nothing herein shall supersede or modify
135
+ the terms of any separate license agreement you may have executed
136
+ with Licensor regarding such Contributions.
137
+
138
+ 6. Trademarks. This License does not grant permission to use the trade
139
+ names, trademarks, service marks, or product names of the Licensor,
140
+ except as required for reasonable and customary use in describing the
141
+ origin of the Work and reproducing the content of the NOTICE file.
142
+
143
+ 7. Disclaimer of Warranty. Unless required by applicable law or
144
+ agreed to in writing, Licensor provides the Work (and each
145
+ Contributor provides its Contributions) on an "AS IS" BASIS,
146
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
147
+ implied, including, without limitation, any warranties or conditions
148
+ of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
149
+ PARTICULAR PURPOSE. You are solely responsible for determining the
150
+ appropriateness of using or redistributing the Work and assume any
151
+ risks associated with Your exercise of permissions under this License.
152
+
153
+ 8. Limitation of Liability. In no event and under no legal theory,
154
+ whether in tort (including negligence), contract, or otherwise,
155
+ unless required by applicable law (such as deliberate and grossly
156
+ negligent acts) or agreed to in writing, shall any Contributor be
157
+ liable to You for damages, including any direct, indirect, special,
158
+ incidental, or consequential damages of any character arising as a
159
+ result of this License or out of the use or inability to use the
160
+ Work (including but not limited to damages for loss of goodwill,
161
+ work stoppage, computer failure or malfunction, or any and all
162
+ other commercial damages or losses), even if such Contributor
163
+ has been advised of the possibility of such damages.
164
+
165
+ 9. Accepting Warranty or Additional Liability. While redistributing
166
+ the Work or Derivative Works thereof, You may choose to offer,
167
+ and charge a fee for, acceptance of support, warranty, indemnity,
168
+ or other liability obligations and/or rights consistent with this
169
+ License. However, in accepting such obligations, You may act only
170
+ on Your own behalf and on Your sole responsibility, not on behalf
171
+ of any other Contributor, and only if You agree to indemnify,
172
+ defend, and hold each Contributor harmless for any liability
173
+ incurred by, or claims asserted against, such Contributor by reason
174
+ of your accepting any such warranty or additional liability.
175
+
176
+ END OF TERMS AND CONDITIONS
177
+
178
+ APPENDIX: How to apply the Apache License to your work.
179
+
180
+ To apply the Apache License to your work, attach the following
181
+ boilerplate notice, with the fields enclosed by brackets "[]"
182
+ replaced with your own identifying information. (Don't include
183
+ the brackets!) The text should be enclosed in the appropriate
184
+ comment syntax for the file format. We also recommend that a
185
+ file or class name and description of purpose be included on the
186
+ same "printed page" as the copyright notice for easier
187
+ identification within third-party archives.
188
+
189
+ Copyright [yyyy] [name of copyright owner]
190
+
191
+ Licensed under the Apache License, Version 2.0 (the "License");
192
+ you may not use this file except in compliance with the License.
193
+ You may obtain a copy of the License at
194
+
195
+ http://www.apache.org/licenses/LICENSE-2.0
196
+
197
+ Unless required by applicable law or agreed to in writing, software
198
+ distributed under the License is distributed on an "AS IS" BASIS,
199
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
200
+ See the License for the specific language governing permissions and
201
+ limitations under the License.
README.md ADDED
@@ -0,0 +1,14 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ ---
2
+ title: AnimateDiff
3
+ emoji: 🐠
4
+ colorFrom: green
5
+ colorTo: gray
6
+ sdk: gradio
7
+ sdk_version: 3.37.0
8
+ app_file: app.py
9
+ pinned: false
10
+ license: apache-2.0
11
+ duplicated_from: guoyww/AnimateDiff
12
+ ---
13
+
14
+ Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
animatediff/models/attention.py ADDED
@@ -0,0 +1,300 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Adapted from https://github.com/huggingface/diffusers/blob/main/src/diffusers/models/attention.py
2
+
3
+ from dataclasses import dataclass
4
+ from typing import Optional
5
+
6
+ import torch
7
+ import torch.nn.functional as F
8
+ from torch import nn
9
+
10
+ from diffusers.configuration_utils import ConfigMixin, register_to_config
11
+ from diffusers.modeling_utils import ModelMixin
12
+ from diffusers.utils import BaseOutput
13
+ from diffusers.utils.import_utils import is_xformers_available
14
+ from diffusers.models.attention import CrossAttention, FeedForward, AdaLayerNorm
15
+
16
+ from einops import rearrange, repeat
17
+ import pdb
18
+
19
+ @dataclass
20
+ class Transformer3DModelOutput(BaseOutput):
21
+ sample: torch.FloatTensor
22
+
23
+
24
+ if is_xformers_available():
25
+ import xformers
26
+ import xformers.ops
27
+ else:
28
+ xformers = None
29
+
30
+
31
+ class Transformer3DModel(ModelMixin, ConfigMixin):
32
+ @register_to_config
33
+ def __init__(
34
+ self,
35
+ num_attention_heads: int = 16,
36
+ attention_head_dim: int = 88,
37
+ in_channels: Optional[int] = None,
38
+ num_layers: int = 1,
39
+ dropout: float = 0.0,
40
+ norm_num_groups: int = 32,
41
+ cross_attention_dim: Optional[int] = None,
42
+ attention_bias: bool = False,
43
+ activation_fn: str = "geglu",
44
+ num_embeds_ada_norm: Optional[int] = None,
45
+ use_linear_projection: bool = False,
46
+ only_cross_attention: bool = False,
47
+ upcast_attention: bool = False,
48
+
49
+ unet_use_cross_frame_attention=None,
50
+ unet_use_temporal_attention=None,
51
+ ):
52
+ super().__init__()
53
+ self.use_linear_projection = use_linear_projection
54
+ self.num_attention_heads = num_attention_heads
55
+ self.attention_head_dim = attention_head_dim
56
+ inner_dim = num_attention_heads * attention_head_dim
57
+
58
+ # Define input layers
59
+ self.in_channels = in_channels
60
+
61
+ self.norm = torch.nn.GroupNorm(num_groups=norm_num_groups, num_channels=in_channels, eps=1e-6, affine=True)
62
+ if use_linear_projection:
63
+ self.proj_in = nn.Linear(in_channels, inner_dim)
64
+ else:
65
+ self.proj_in = nn.Conv2d(in_channels, inner_dim, kernel_size=1, stride=1, padding=0)
66
+
67
+ # Define transformers blocks
68
+ self.transformer_blocks = nn.ModuleList(
69
+ [
70
+ BasicTransformerBlock(
71
+ inner_dim,
72
+ num_attention_heads,
73
+ attention_head_dim,
74
+ dropout=dropout,
75
+ cross_attention_dim=cross_attention_dim,
76
+ activation_fn=activation_fn,
77
+ num_embeds_ada_norm=num_embeds_ada_norm,
78
+ attention_bias=attention_bias,
79
+ only_cross_attention=only_cross_attention,
80
+ upcast_attention=upcast_attention,
81
+
82
+ unet_use_cross_frame_attention=unet_use_cross_frame_attention,
83
+ unet_use_temporal_attention=unet_use_temporal_attention,
84
+ )
85
+ for d in range(num_layers)
86
+ ]
87
+ )
88
+
89
+ # 4. Define output layers
90
+ if use_linear_projection:
91
+ self.proj_out = nn.Linear(in_channels, inner_dim)
92
+ else:
93
+ self.proj_out = nn.Conv2d(inner_dim, in_channels, kernel_size=1, stride=1, padding=0)
94
+
95
+ def forward(self, hidden_states, encoder_hidden_states=None, timestep=None, return_dict: bool = True):
96
+ # Input
97
+ assert hidden_states.dim() == 5, f"Expected hidden_states to have ndim=5, but got ndim={hidden_states.dim()}."
98
+ video_length = hidden_states.shape[2]
99
+ hidden_states = rearrange(hidden_states, "b c f h w -> (b f) c h w")
100
+ encoder_hidden_states = repeat(encoder_hidden_states, 'b n c -> (b f) n c', f=video_length)
101
+
102
+ batch, channel, height, weight = hidden_states.shape
103
+ residual = hidden_states
104
+
105
+ hidden_states = self.norm(hidden_states)
106
+ if not self.use_linear_projection:
107
+ hidden_states = self.proj_in(hidden_states)
108
+ inner_dim = hidden_states.shape[1]
109
+ hidden_states = hidden_states.permute(0, 2, 3, 1).reshape(batch, height * weight, inner_dim)
110
+ else:
111
+ inner_dim = hidden_states.shape[1]
112
+ hidden_states = hidden_states.permute(0, 2, 3, 1).reshape(batch, height * weight, inner_dim)
113
+ hidden_states = self.proj_in(hidden_states)
114
+
115
+ # Blocks
116
+ for block in self.transformer_blocks:
117
+ hidden_states = block(
118
+ hidden_states,
119
+ encoder_hidden_states=encoder_hidden_states,
120
+ timestep=timestep,
121
+ video_length=video_length
122
+ )
123
+
124
+ # Output
125
+ if not self.use_linear_projection:
126
+ hidden_states = (
127
+ hidden_states.reshape(batch, height, weight, inner_dim).permute(0, 3, 1, 2).contiguous()
128
+ )
129
+ hidden_states = self.proj_out(hidden_states)
130
+ else:
131
+ hidden_states = self.proj_out(hidden_states)
132
+ hidden_states = (
133
+ hidden_states.reshape(batch, height, weight, inner_dim).permute(0, 3, 1, 2).contiguous()
134
+ )
135
+
136
+ output = hidden_states + residual
137
+
138
+ output = rearrange(output, "(b f) c h w -> b c f h w", f=video_length)
139
+ if not return_dict:
140
+ return (output,)
141
+
142
+ return Transformer3DModelOutput(sample=output)
143
+
144
+
145
+ class BasicTransformerBlock(nn.Module):
146
+ def __init__(
147
+ self,
148
+ dim: int,
149
+ num_attention_heads: int,
150
+ attention_head_dim: int,
151
+ dropout=0.0,
152
+ cross_attention_dim: Optional[int] = None,
153
+ activation_fn: str = "geglu",
154
+ num_embeds_ada_norm: Optional[int] = None,
155
+ attention_bias: bool = False,
156
+ only_cross_attention: bool = False,
157
+ upcast_attention: bool = False,
158
+
159
+ unet_use_cross_frame_attention = None,
160
+ unet_use_temporal_attention = None,
161
+ ):
162
+ super().__init__()
163
+ self.only_cross_attention = only_cross_attention
164
+ self.use_ada_layer_norm = num_embeds_ada_norm is not None
165
+ self.unet_use_cross_frame_attention = unet_use_cross_frame_attention
166
+ self.unet_use_temporal_attention = unet_use_temporal_attention
167
+
168
+ # SC-Attn
169
+ assert unet_use_cross_frame_attention is not None
170
+ if unet_use_cross_frame_attention:
171
+ self.attn1 = SparseCausalAttention2D(
172
+ query_dim=dim,
173
+ heads=num_attention_heads,
174
+ dim_head=attention_head_dim,
175
+ dropout=dropout,
176
+ bias=attention_bias,
177
+ cross_attention_dim=cross_attention_dim if only_cross_attention else None,
178
+ upcast_attention=upcast_attention,
179
+ )
180
+ else:
181
+ self.attn1 = CrossAttention(
182
+ query_dim=dim,
183
+ heads=num_attention_heads,
184
+ dim_head=attention_head_dim,
185
+ dropout=dropout,
186
+ bias=attention_bias,
187
+ upcast_attention=upcast_attention,
188
+ )
189
+ self.norm1 = AdaLayerNorm(dim, num_embeds_ada_norm) if self.use_ada_layer_norm else nn.LayerNorm(dim)
190
+
191
+ # Cross-Attn
192
+ if cross_attention_dim is not None:
193
+ self.attn2 = CrossAttention(
194
+ query_dim=dim,
195
+ cross_attention_dim=cross_attention_dim,
196
+ heads=num_attention_heads,
197
+ dim_head=attention_head_dim,
198
+ dropout=dropout,
199
+ bias=attention_bias,
200
+ upcast_attention=upcast_attention,
201
+ )
202
+ else:
203
+ self.attn2 = None
204
+
205
+ if cross_attention_dim is not None:
206
+ self.norm2 = AdaLayerNorm(dim, num_embeds_ada_norm) if self.use_ada_layer_norm else nn.LayerNorm(dim)
207
+ else:
208
+ self.norm2 = None
209
+
210
+ # Feed-forward
211
+ self.ff = FeedForward(dim, dropout=dropout, activation_fn=activation_fn)
212
+ self.norm3 = nn.LayerNorm(dim)
213
+
214
+ # Temp-Attn
215
+ assert unet_use_temporal_attention is not None
216
+ if unet_use_temporal_attention:
217
+ self.attn_temp = CrossAttention(
218
+ query_dim=dim,
219
+ heads=num_attention_heads,
220
+ dim_head=attention_head_dim,
221
+ dropout=dropout,
222
+ bias=attention_bias,
223
+ upcast_attention=upcast_attention,
224
+ )
225
+ nn.init.zeros_(self.attn_temp.to_out[0].weight.data)
226
+ self.norm_temp = AdaLayerNorm(dim, num_embeds_ada_norm) if self.use_ada_layer_norm else nn.LayerNorm(dim)
227
+
228
+ def set_use_memory_efficient_attention_xformers(self, use_memory_efficient_attention_xformers: bool):
229
+ if not is_xformers_available():
230
+ print("Here is how to install it")
231
+ raise ModuleNotFoundError(
232
+ "Refer to https://github.com/facebookresearch/xformers for more information on how to install"
233
+ " xformers",
234
+ name="xformers",
235
+ )
236
+ elif not torch.cuda.is_available():
237
+ raise ValueError(
238
+ "torch.cuda.is_available() should be True but is False. xformers' memory efficient attention is only"
239
+ " available for GPU "
240
+ )
241
+ else:
242
+ try:
243
+ # Make sure we can run the memory efficient attention
244
+ _ = xformers.ops.memory_efficient_attention(
245
+ torch.randn((1, 2, 40), device="cuda"),
246
+ torch.randn((1, 2, 40), device="cuda"),
247
+ torch.randn((1, 2, 40), device="cuda"),
248
+ )
249
+ except Exception as e:
250
+ raise e
251
+ self.attn1._use_memory_efficient_attention_xformers = use_memory_efficient_attention_xformers
252
+ if self.attn2 is not None:
253
+ self.attn2._use_memory_efficient_attention_xformers = use_memory_efficient_attention_xformers
254
+ # self.attn_temp._use_memory_efficient_attention_xformers = use_memory_efficient_attention_xformers
255
+
256
+ def forward(self, hidden_states, encoder_hidden_states=None, timestep=None, attention_mask=None, video_length=None):
257
+ # SparseCausal-Attention
258
+ norm_hidden_states = (
259
+ self.norm1(hidden_states, timestep) if self.use_ada_layer_norm else self.norm1(hidden_states)
260
+ )
261
+
262
+ # if self.only_cross_attention:
263
+ # hidden_states = (
264
+ # self.attn1(norm_hidden_states, encoder_hidden_states, attention_mask=attention_mask) + hidden_states
265
+ # )
266
+ # else:
267
+ # hidden_states = self.attn1(norm_hidden_states, attention_mask=attention_mask, video_length=video_length) + hidden_states
268
+
269
+ # pdb.set_trace()
270
+ if self.unet_use_cross_frame_attention:
271
+ hidden_states = self.attn1(norm_hidden_states, attention_mask=attention_mask, video_length=video_length) + hidden_states
272
+ else:
273
+ hidden_states = self.attn1(norm_hidden_states, attention_mask=attention_mask) + hidden_states
274
+
275
+ if self.attn2 is not None:
276
+ # Cross-Attention
277
+ norm_hidden_states = (
278
+ self.norm2(hidden_states, timestep) if self.use_ada_layer_norm else self.norm2(hidden_states)
279
+ )
280
+ hidden_states = (
281
+ self.attn2(
282
+ norm_hidden_states, encoder_hidden_states=encoder_hidden_states, attention_mask=attention_mask
283
+ )
284
+ + hidden_states
285
+ )
286
+
287
+ # Feed-forward
288
+ hidden_states = self.ff(self.norm3(hidden_states)) + hidden_states
289
+
290
+ # Temporal-Attention
291
+ if self.unet_use_temporal_attention:
292
+ d = hidden_states.shape[1]
293
+ hidden_states = rearrange(hidden_states, "(b f) d c -> (b d) f c", f=video_length)
294
+ norm_hidden_states = (
295
+ self.norm_temp(hidden_states, timestep) if self.use_ada_layer_norm else self.norm_temp(hidden_states)
296
+ )
297
+ hidden_states = self.attn_temp(norm_hidden_states) + hidden_states
298
+ hidden_states = rearrange(hidden_states, "(b d) f c -> (b f) d c", d=d)
299
+
300
+ return hidden_states
animatediff/models/motion_module.py ADDED
@@ -0,0 +1,331 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from dataclasses import dataclass
2
+ from typing import List, Optional, Tuple, Union
3
+
4
+ import torch
5
+ import numpy as np
6
+ import torch.nn.functional as F
7
+ from torch import nn
8
+ import torchvision
9
+
10
+ from diffusers.configuration_utils import ConfigMixin, register_to_config
11
+ from diffusers.modeling_utils import ModelMixin
12
+ from diffusers.utils import BaseOutput
13
+ from diffusers.utils.import_utils import is_xformers_available
14
+ from diffusers.models.attention import CrossAttention, FeedForward
15
+
16
+ from einops import rearrange, repeat
17
+ import math
18
+
19
+
20
+ def zero_module(module):
21
+ # Zero out the parameters of a module and return it.
22
+ for p in module.parameters():
23
+ p.detach().zero_()
24
+ return module
25
+
26
+
27
+ @dataclass
28
+ class TemporalTransformer3DModelOutput(BaseOutput):
29
+ sample: torch.FloatTensor
30
+
31
+
32
+ if is_xformers_available():
33
+ import xformers
34
+ import xformers.ops
35
+ else:
36
+ xformers = None
37
+
38
+
39
+ def get_motion_module(
40
+ in_channels,
41
+ motion_module_type: str,
42
+ motion_module_kwargs: dict
43
+ ):
44
+ if motion_module_type == "Vanilla":
45
+ return VanillaTemporalModule(in_channels=in_channels, **motion_module_kwargs,)
46
+ else:
47
+ raise ValueError
48
+
49
+
50
+ class VanillaTemporalModule(nn.Module):
51
+ def __init__(
52
+ self,
53
+ in_channels,
54
+ num_attention_heads = 8,
55
+ num_transformer_block = 2,
56
+ attention_block_types =( "Temporal_Self", "Temporal_Self" ),
57
+ cross_frame_attention_mode = None,
58
+ temporal_position_encoding = False,
59
+ temporal_position_encoding_max_len = 24,
60
+ temporal_attention_dim_div = 1,
61
+ zero_initialize = True,
62
+ ):
63
+ super().__init__()
64
+
65
+ self.temporal_transformer = TemporalTransformer3DModel(
66
+ in_channels=in_channels,
67
+ num_attention_heads=num_attention_heads,
68
+ attention_head_dim=in_channels // num_attention_heads // temporal_attention_dim_div,
69
+ num_layers=num_transformer_block,
70
+ attention_block_types=attention_block_types,
71
+ cross_frame_attention_mode=cross_frame_attention_mode,
72
+ temporal_position_encoding=temporal_position_encoding,
73
+ temporal_position_encoding_max_len=temporal_position_encoding_max_len,
74
+ )
75
+
76
+ if zero_initialize:
77
+ self.temporal_transformer.proj_out = zero_module(self.temporal_transformer.proj_out)
78
+
79
+ def forward(self, input_tensor, temb, encoder_hidden_states, attention_mask=None, anchor_frame_idx=None):
80
+ hidden_states = input_tensor
81
+ hidden_states = self.temporal_transformer(hidden_states, encoder_hidden_states, attention_mask)
82
+
83
+ output = hidden_states
84
+ return output
85
+
86
+
87
+ class TemporalTransformer3DModel(nn.Module):
88
+ def __init__(
89
+ self,
90
+ in_channels,
91
+ num_attention_heads,
92
+ attention_head_dim,
93
+
94
+ num_layers,
95
+ attention_block_types = ( "Temporal_Self", "Temporal_Self", ),
96
+ dropout = 0.0,
97
+ norm_num_groups = 32,
98
+ cross_attention_dim = 768,
99
+ activation_fn = "geglu",
100
+ attention_bias = False,
101
+ upcast_attention = False,
102
+
103
+ cross_frame_attention_mode = None,
104
+ temporal_position_encoding = False,
105
+ temporal_position_encoding_max_len = 24,
106
+ ):
107
+ super().__init__()
108
+
109
+ inner_dim = num_attention_heads * attention_head_dim
110
+
111
+ self.norm = torch.nn.GroupNorm(num_groups=norm_num_groups, num_channels=in_channels, eps=1e-6, affine=True)
112
+ self.proj_in = nn.Linear(in_channels, inner_dim)
113
+
114
+ self.transformer_blocks = nn.ModuleList(
115
+ [
116
+ TemporalTransformerBlock(
117
+ dim=inner_dim,
118
+ num_attention_heads=num_attention_heads,
119
+ attention_head_dim=attention_head_dim,
120
+ attention_block_types=attention_block_types,
121
+ dropout=dropout,
122
+ norm_num_groups=norm_num_groups,
123
+ cross_attention_dim=cross_attention_dim,
124
+ activation_fn=activation_fn,
125
+ attention_bias=attention_bias,
126
+ upcast_attention=upcast_attention,
127
+ cross_frame_attention_mode=cross_frame_attention_mode,
128
+ temporal_position_encoding=temporal_position_encoding,
129
+ temporal_position_encoding_max_len=temporal_position_encoding_max_len,
130
+ )
131
+ for d in range(num_layers)
132
+ ]
133
+ )
134
+ self.proj_out = nn.Linear(inner_dim, in_channels)
135
+
136
+ def forward(self, hidden_states, encoder_hidden_states=None, attention_mask=None):
137
+ assert hidden_states.dim() == 5, f"Expected hidden_states to have ndim=5, but got ndim={hidden_states.dim()}."
138
+ video_length = hidden_states.shape[2]
139
+ hidden_states = rearrange(hidden_states, "b c f h w -> (b f) c h w")
140
+
141
+ batch, channel, height, weight = hidden_states.shape
142
+ residual = hidden_states
143
+
144
+ hidden_states = self.norm(hidden_states)
145
+ inner_dim = hidden_states.shape[1]
146
+ hidden_states = hidden_states.permute(0, 2, 3, 1).reshape(batch, height * weight, inner_dim)
147
+ hidden_states = self.proj_in(hidden_states)
148
+
149
+ # Transformer Blocks
150
+ for block in self.transformer_blocks:
151
+ hidden_states = block(hidden_states, encoder_hidden_states=encoder_hidden_states, video_length=video_length)
152
+
153
+ # output
154
+ hidden_states = self.proj_out(hidden_states)
155
+ hidden_states = hidden_states.reshape(batch, height, weight, inner_dim).permute(0, 3, 1, 2).contiguous()
156
+
157
+ output = hidden_states + residual
158
+ output = rearrange(output, "(b f) c h w -> b c f h w", f=video_length)
159
+
160
+ return output
161
+
162
+
163
+ class TemporalTransformerBlock(nn.Module):
164
+ def __init__(
165
+ self,
166
+ dim,
167
+ num_attention_heads,
168
+ attention_head_dim,
169
+ attention_block_types = ( "Temporal_Self", "Temporal_Self", ),
170
+ dropout = 0.0,
171
+ norm_num_groups = 32,
172
+ cross_attention_dim = 768,
173
+ activation_fn = "geglu",
174
+ attention_bias = False,
175
+ upcast_attention = False,
176
+ cross_frame_attention_mode = None,
177
+ temporal_position_encoding = False,
178
+ temporal_position_encoding_max_len = 24,
179
+ ):
180
+ super().__init__()
181
+
182
+ attention_blocks = []
183
+ norms = []
184
+
185
+ for block_name in attention_block_types:
186
+ attention_blocks.append(
187
+ VersatileAttention(
188
+ attention_mode=block_name.split("_")[0],
189
+ cross_attention_dim=cross_attention_dim if block_name.endswith("_Cross") else None,
190
+
191
+ query_dim=dim,
192
+ heads=num_attention_heads,
193
+ dim_head=attention_head_dim,
194
+ dropout=dropout,
195
+ bias=attention_bias,
196
+ upcast_attention=upcast_attention,
197
+
198
+ cross_frame_attention_mode=cross_frame_attention_mode,
199
+ temporal_position_encoding=temporal_position_encoding,
200
+ temporal_position_encoding_max_len=temporal_position_encoding_max_len,
201
+ )
202
+ )
203
+ norms.append(nn.LayerNorm(dim))
204
+
205
+ self.attention_blocks = nn.ModuleList(attention_blocks)
206
+ self.norms = nn.ModuleList(norms)
207
+
208
+ self.ff = FeedForward(dim, dropout=dropout, activation_fn=activation_fn)
209
+ self.ff_norm = nn.LayerNorm(dim)
210
+
211
+
212
+ def forward(self, hidden_states, encoder_hidden_states=None, attention_mask=None, video_length=None):
213
+ for attention_block, norm in zip(self.attention_blocks, self.norms):
214
+ norm_hidden_states = norm(hidden_states)
215
+ hidden_states = attention_block(
216
+ norm_hidden_states,
217
+ encoder_hidden_states=encoder_hidden_states if attention_block.is_cross_attention else None,
218
+ video_length=video_length,
219
+ ) + hidden_states
220
+
221
+ hidden_states = self.ff(self.ff_norm(hidden_states)) + hidden_states
222
+
223
+ output = hidden_states
224
+ return output
225
+
226
+
227
+ class PositionalEncoding(nn.Module):
228
+ def __init__(
229
+ self,
230
+ d_model,
231
+ dropout = 0.,
232
+ max_len = 24
233
+ ):
234
+ super().__init__()
235
+ self.dropout = nn.Dropout(p=dropout)
236
+ position = torch.arange(max_len).unsqueeze(1)
237
+ div_term = torch.exp(torch.arange(0, d_model, 2) * (-math.log(10000.0) / d_model))
238
+ pe = torch.zeros(1, max_len, d_model)
239
+ pe[0, :, 0::2] = torch.sin(position * div_term)
240
+ pe[0, :, 1::2] = torch.cos(position * div_term)
241
+ self.register_buffer('pe', pe)
242
+
243
+ def forward(self, x):
244
+ x = x + self.pe[:, :x.size(1)]
245
+ return self.dropout(x)
246
+
247
+
248
+ class VersatileAttention(CrossAttention):
249
+ def __init__(
250
+ self,
251
+ attention_mode = None,
252
+ cross_frame_attention_mode = None,
253
+ temporal_position_encoding = False,
254
+ temporal_position_encoding_max_len = 24,
255
+ *args, **kwargs
256
+ ):
257
+ super().__init__(*args, **kwargs)
258
+ assert attention_mode == "Temporal"
259
+
260
+ self.attention_mode = attention_mode
261
+ self.is_cross_attention = kwargs["cross_attention_dim"] is not None
262
+
263
+ self.pos_encoder = PositionalEncoding(
264
+ kwargs["query_dim"],
265
+ dropout=0.,
266
+ max_len=temporal_position_encoding_max_len
267
+ ) if (temporal_position_encoding and attention_mode == "Temporal") else None
268
+
269
+ def extra_repr(self):
270
+ return f"(Module Info) Attention_Mode: {self.attention_mode}, Is_Cross_Attention: {self.is_cross_attention}"
271
+
272
+ def forward(self, hidden_states, encoder_hidden_states=None, attention_mask=None, video_length=None):
273
+ batch_size, sequence_length, _ = hidden_states.shape
274
+
275
+ if self.attention_mode == "Temporal":
276
+ d = hidden_states.shape[1]
277
+ hidden_states = rearrange(hidden_states, "(b f) d c -> (b d) f c", f=video_length)
278
+
279
+ if self.pos_encoder is not None:
280
+ hidden_states = self.pos_encoder(hidden_states)
281
+
282
+ encoder_hidden_states = repeat(encoder_hidden_states, "b n c -> (b d) n c", d=d) if encoder_hidden_states is not None else encoder_hidden_states
283
+ else:
284
+ raise NotImplementedError
285
+
286
+ encoder_hidden_states = encoder_hidden_states
287
+
288
+ if self.group_norm is not None:
289
+ hidden_states = self.group_norm(hidden_states.transpose(1, 2)).transpose(1, 2)
290
+
291
+ query = self.to_q(hidden_states)
292
+ dim = query.shape[-1]
293
+ query = self.reshape_heads_to_batch_dim(query)
294
+
295
+ if self.added_kv_proj_dim is not None:
296
+ raise NotImplementedError
297
+
298
+ encoder_hidden_states = encoder_hidden_states if encoder_hidden_states is not None else hidden_states
299
+ key = self.to_k(encoder_hidden_states)
300
+ value = self.to_v(encoder_hidden_states)
301
+
302
+ key = self.reshape_heads_to_batch_dim(key)
303
+ value = self.reshape_heads_to_batch_dim(value)
304
+
305
+ if attention_mask is not None:
306
+ if attention_mask.shape[-1] != query.shape[1]:
307
+ target_length = query.shape[1]
308
+ attention_mask = F.pad(attention_mask, (0, target_length), value=0.0)
309
+ attention_mask = attention_mask.repeat_interleave(self.heads, dim=0)
310
+
311
+ # attention, what we cannot get enough of
312
+ if self._use_memory_efficient_attention_xformers:
313
+ hidden_states = self._memory_efficient_attention_xformers(query, key, value, attention_mask)
314
+ # Some versions of xformers return output in fp32, cast it back to the dtype of the input
315
+ hidden_states = hidden_states.to(query.dtype)
316
+ else:
317
+ if self._slice_size is None or query.shape[0] // self._slice_size == 1:
318
+ hidden_states = self._attention(query, key, value, attention_mask)
319
+ else:
320
+ hidden_states = self._sliced_attention(query, key, value, sequence_length, dim, attention_mask)
321
+
322
+ # linear proj
323
+ hidden_states = self.to_out[0](hidden_states)
324
+
325
+ # dropout
326
+ hidden_states = self.to_out[1](hidden_states)
327
+
328
+ if self.attention_mode == "Temporal":
329
+ hidden_states = rearrange(hidden_states, "(b d) f c -> (b f) d c", d=d)
330
+
331
+ return hidden_states
animatediff/models/resnet.py ADDED
@@ -0,0 +1,197 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Adapted from https://github.com/huggingface/diffusers/blob/main/src/diffusers/models/resnet.py
2
+
3
+ import torch
4
+ import torch.nn as nn
5
+ import torch.nn.functional as F
6
+
7
+ from einops import rearrange
8
+
9
+
10
+ class InflatedConv3d(nn.Conv2d):
11
+ def forward(self, x):
12
+ video_length = x.shape[2]
13
+
14
+ x = rearrange(x, "b c f h w -> (b f) c h w")
15
+ x = super().forward(x)
16
+ x = rearrange(x, "(b f) c h w -> b c f h w", f=video_length)
17
+
18
+ return x
19
+
20
+
21
+ class Upsample3D(nn.Module):
22
+ def __init__(self, channels, use_conv=False, use_conv_transpose=False, out_channels=None, name="conv"):
23
+ super().__init__()
24
+ self.channels = channels
25
+ self.out_channels = out_channels or channels
26
+ self.use_conv = use_conv
27
+ self.use_conv_transpose = use_conv_transpose
28
+ self.name = name
29
+
30
+ conv = None
31
+ if use_conv_transpose:
32
+ raise NotImplementedError
33
+ elif use_conv:
34
+ self.conv = InflatedConv3d(self.channels, self.out_channels, 3, padding=1)
35
+
36
+ def forward(self, hidden_states, output_size=None):
37
+ assert hidden_states.shape[1] == self.channels
38
+
39
+ if self.use_conv_transpose:
40
+ raise NotImplementedError
41
+
42
+ # Cast to float32 to as 'upsample_nearest2d_out_frame' op does not support bfloat16
43
+ dtype = hidden_states.dtype
44
+ if dtype == torch.bfloat16:
45
+ hidden_states = hidden_states.to(torch.float32)
46
+
47
+ # upsample_nearest_nhwc fails with large batch sizes. see https://github.com/huggingface/diffusers/issues/984
48
+ if hidden_states.shape[0] >= 64:
49
+ hidden_states = hidden_states.contiguous()
50
+
51
+ # if `output_size` is passed we force the interpolation output
52
+ # size and do not make use of `scale_factor=2`
53
+ if output_size is None:
54
+ hidden_states = F.interpolate(hidden_states, scale_factor=[1.0, 2.0, 2.0], mode="nearest")
55
+ else:
56
+ hidden_states = F.interpolate(hidden_states, size=output_size, mode="nearest")
57
+
58
+ # If the input is bfloat16, we cast back to bfloat16
59
+ if dtype == torch.bfloat16:
60
+ hidden_states = hidden_states.to(dtype)
61
+
62
+ # if self.use_conv:
63
+ # if self.name == "conv":
64
+ # hidden_states = self.conv(hidden_states)
65
+ # else:
66
+ # hidden_states = self.Conv2d_0(hidden_states)
67
+ hidden_states = self.conv(hidden_states)
68
+
69
+ return hidden_states
70
+
71
+
72
+ class Downsample3D(nn.Module):
73
+ def __init__(self, channels, use_conv=False, out_channels=None, padding=1, name="conv"):
74
+ super().__init__()
75
+ self.channels = channels
76
+ self.out_channels = out_channels or channels
77
+ self.use_conv = use_conv
78
+ self.padding = padding
79
+ stride = 2
80
+ self.name = name
81
+
82
+ if use_conv:
83
+ self.conv = InflatedConv3d(self.channels, self.out_channels, 3, stride=stride, padding=padding)
84
+ else:
85
+ raise NotImplementedError
86
+
87
+ def forward(self, hidden_states):
88
+ assert hidden_states.shape[1] == self.channels
89
+ if self.use_conv and self.padding == 0:
90
+ raise NotImplementedError
91
+
92
+ assert hidden_states.shape[1] == self.channels
93
+ hidden_states = self.conv(hidden_states)
94
+
95
+ return hidden_states
96
+
97
+
98
+ class ResnetBlock3D(nn.Module):
99
+ def __init__(
100
+ self,
101
+ *,
102
+ in_channels,
103
+ out_channels=None,
104
+ conv_shortcut=False,
105
+ dropout=0.0,
106
+ temb_channels=512,
107
+ groups=32,
108
+ groups_out=None,
109
+ pre_norm=True,
110
+ eps=1e-6,
111
+ non_linearity="swish",
112
+ time_embedding_norm="default",
113
+ output_scale_factor=1.0,
114
+ use_in_shortcut=None,
115
+ ):
116
+ super().__init__()
117
+ self.pre_norm = pre_norm
118
+ self.pre_norm = True
119
+ self.in_channels = in_channels
120
+ out_channels = in_channels if out_channels is None else out_channels
121
+ self.out_channels = out_channels
122
+ self.use_conv_shortcut = conv_shortcut
123
+ self.time_embedding_norm = time_embedding_norm
124
+ self.output_scale_factor = output_scale_factor
125
+
126
+ if groups_out is None:
127
+ groups_out = groups
128
+
129
+ self.norm1 = torch.nn.GroupNorm(num_groups=groups, num_channels=in_channels, eps=eps, affine=True)
130
+
131
+ self.conv1 = InflatedConv3d(in_channels, out_channels, kernel_size=3, stride=1, padding=1)
132
+
133
+ if temb_channels is not None:
134
+ if self.time_embedding_norm == "default":
135
+ time_emb_proj_out_channels = out_channels
136
+ elif self.time_embedding_norm == "scale_shift":
137
+ time_emb_proj_out_channels = out_channels * 2
138
+ else:
139
+ raise ValueError(f"unknown time_embedding_norm : {self.time_embedding_norm} ")
140
+
141
+ self.time_emb_proj = torch.nn.Linear(temb_channels, time_emb_proj_out_channels)
142
+ else:
143
+ self.time_emb_proj = None
144
+
145
+ self.norm2 = torch.nn.GroupNorm(num_groups=groups_out, num_channels=out_channels, eps=eps, affine=True)
146
+ self.dropout = torch.nn.Dropout(dropout)
147
+ self.conv2 = InflatedConv3d(out_channels, out_channels, kernel_size=3, stride=1, padding=1)
148
+
149
+ if non_linearity == "swish":
150
+ self.nonlinearity = lambda x: F.silu(x)
151
+ elif non_linearity == "mish":
152
+ self.nonlinearity = Mish()
153
+ elif non_linearity == "silu":
154
+ self.nonlinearity = nn.SiLU()
155
+
156
+ self.use_in_shortcut = self.in_channels != self.out_channels if use_in_shortcut is None else use_in_shortcut
157
+
158
+ self.conv_shortcut = None
159
+ if self.use_in_shortcut:
160
+ self.conv_shortcut = InflatedConv3d(in_channels, out_channels, kernel_size=1, stride=1, padding=0)
161
+
162
+ def forward(self, input_tensor, temb):
163
+ hidden_states = input_tensor
164
+
165
+ hidden_states = self.norm1(hidden_states)
166
+ hidden_states = self.nonlinearity(hidden_states)
167
+
168
+ hidden_states = self.conv1(hidden_states)
169
+
170
+ if temb is not None:
171
+ temb = self.time_emb_proj(self.nonlinearity(temb))[:, :, None, None, None]
172
+
173
+ if temb is not None and self.time_embedding_norm == "default":
174
+ hidden_states = hidden_states + temb
175
+
176
+ hidden_states = self.norm2(hidden_states)
177
+
178
+ if temb is not None and self.time_embedding_norm == "scale_shift":
179
+ scale, shift = torch.chunk(temb, 2, dim=1)
180
+ hidden_states = hidden_states * (1 + scale) + shift
181
+
182
+ hidden_states = self.nonlinearity(hidden_states)
183
+
184
+ hidden_states = self.dropout(hidden_states)
185
+ hidden_states = self.conv2(hidden_states)
186
+
187
+ if self.conv_shortcut is not None:
188
+ input_tensor = self.conv_shortcut(input_tensor)
189
+
190
+ output_tensor = (input_tensor + hidden_states) / self.output_scale_factor
191
+
192
+ return output_tensor
193
+
194
+
195
+ class Mish(torch.nn.Module):
196
+ def forward(self, hidden_states):
197
+ return hidden_states * torch.tanh(torch.nn.functional.softplus(hidden_states))
animatediff/models/unet.py ADDED
@@ -0,0 +1,489 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Adapted from https://github.com/huggingface/diffusers/blob/main/src/diffusers/models/unet_2d_condition.py
2
+
3
+ from dataclasses import dataclass
4
+ from typing import List, Optional, Tuple, Union
5
+
6
+ import os
7
+ import json
8
+ import pdb
9
+
10
+ import torch
11
+ import torch.nn as nn
12
+ import torch.utils.checkpoint
13
+
14
+ from diffusers.configuration_utils import ConfigMixin, register_to_config
15
+ from diffusers.modeling_utils import ModelMixin
16
+ from diffusers.utils import BaseOutput, logging
17
+ from diffusers.models.embeddings import TimestepEmbedding, Timesteps
18
+ from .unet_blocks import (
19
+ CrossAttnDownBlock3D,
20
+ CrossAttnUpBlock3D,
21
+ DownBlock3D,
22
+ UNetMidBlock3DCrossAttn,
23
+ UpBlock3D,
24
+ get_down_block,
25
+ get_up_block,
26
+ )
27
+ from .resnet import InflatedConv3d
28
+
29
+
30
+ logger = logging.get_logger(__name__) # pylint: disable=invalid-name
31
+
32
+
33
+ @dataclass
34
+ class UNet3DConditionOutput(BaseOutput):
35
+ sample: torch.FloatTensor
36
+
37
+
38
+ class UNet3DConditionModel(ModelMixin, ConfigMixin):
39
+ _supports_gradient_checkpointing = True
40
+
41
+ @register_to_config
42
+ def __init__(
43
+ self,
44
+ sample_size: Optional[int] = None,
45
+ in_channels: int = 4,
46
+ out_channels: int = 4,
47
+ center_input_sample: bool = False,
48
+ flip_sin_to_cos: bool = True,
49
+ freq_shift: int = 0,
50
+ down_block_types: Tuple[str] = (
51
+ "CrossAttnDownBlock3D",
52
+ "CrossAttnDownBlock3D",
53
+ "CrossAttnDownBlock3D",
54
+ "DownBlock3D",
55
+ ),
56
+ mid_block_type: str = "UNetMidBlock3DCrossAttn",
57
+ up_block_types: Tuple[str] = (
58
+ "UpBlock3D",
59
+ "CrossAttnUpBlock3D",
60
+ "CrossAttnUpBlock3D",
61
+ "CrossAttnUpBlock3D"
62
+ ),
63
+ only_cross_attention: Union[bool, Tuple[bool]] = False,
64
+ block_out_channels: Tuple[int] = (320, 640, 1280, 1280),
65
+ layers_per_block: int = 2,
66
+ downsample_padding: int = 1,
67
+ mid_block_scale_factor: float = 1,
68
+ act_fn: str = "silu",
69
+ norm_num_groups: int = 32,
70
+ norm_eps: float = 1e-5,
71
+ cross_attention_dim: int = 1280,
72
+ attention_head_dim: Union[int, Tuple[int]] = 8,
73
+ dual_cross_attention: bool = False,
74
+ use_linear_projection: bool = False,
75
+ class_embed_type: Optional[str] = None,
76
+ num_class_embeds: Optional[int] = None,
77
+ upcast_attention: bool = False,
78
+ resnet_time_scale_shift: str = "default",
79
+
80
+ # Additional
81
+ use_motion_module = False,
82
+ motion_module_resolutions = ( 1,2,4,8 ),
83
+ motion_module_mid_block = False,
84
+ motion_module_decoder_only = False,
85
+ motion_module_type = None,
86
+ motion_module_kwargs = {},
87
+ unet_use_cross_frame_attention = None,
88
+ unet_use_temporal_attention = None,
89
+ ):
90
+ super().__init__()
91
+
92
+ self.sample_size = sample_size
93
+ time_embed_dim = block_out_channels[0] * 4
94
+
95
+ # input
96
+ self.conv_in = InflatedConv3d(in_channels, block_out_channels[0], kernel_size=3, padding=(1, 1))
97
+
98
+ # time
99
+ self.time_proj = Timesteps(block_out_channels[0], flip_sin_to_cos, freq_shift)
100
+ timestep_input_dim = block_out_channels[0]
101
+
102
+ self.time_embedding = TimestepEmbedding(timestep_input_dim, time_embed_dim)
103
+
104
+ # class embedding
105
+ if class_embed_type is None and num_class_embeds is not None:
106
+ self.class_embedding = nn.Embedding(num_class_embeds, time_embed_dim)
107
+ elif class_embed_type == "timestep":
108
+ self.class_embedding = TimestepEmbedding(timestep_input_dim, time_embed_dim)
109
+ elif class_embed_type == "identity":
110
+ self.class_embedding = nn.Identity(time_embed_dim, time_embed_dim)
111
+ else:
112
+ self.class_embedding = None
113
+
114
+ self.down_blocks = nn.ModuleList([])
115
+ self.mid_block = None
116
+ self.up_blocks = nn.ModuleList([])
117
+
118
+ if isinstance(only_cross_attention, bool):
119
+ only_cross_attention = [only_cross_attention] * len(down_block_types)
120
+
121
+ if isinstance(attention_head_dim, int):
122
+ attention_head_dim = (attention_head_dim,) * len(down_block_types)
123
+
124
+ # down
125
+ output_channel = block_out_channels[0]
126
+ for i, down_block_type in enumerate(down_block_types):
127
+ res = 2 ** i
128
+ input_channel = output_channel
129
+ output_channel = block_out_channels[i]
130
+ is_final_block = i == len(block_out_channels) - 1
131
+
132
+ down_block = get_down_block(
133
+ down_block_type,
134
+ num_layers=layers_per_block,
135
+ in_channels=input_channel,
136
+ out_channels=output_channel,
137
+ temb_channels=time_embed_dim,
138
+ add_downsample=not is_final_block,
139
+ resnet_eps=norm_eps,
140
+ resnet_act_fn=act_fn,
141
+ resnet_groups=norm_num_groups,
142
+ cross_attention_dim=cross_attention_dim,
143
+ attn_num_head_channels=attention_head_dim[i],
144
+ downsample_padding=downsample_padding,
145
+ dual_cross_attention=dual_cross_attention,
146
+ use_linear_projection=use_linear_projection,
147
+ only_cross_attention=only_cross_attention[i],
148
+ upcast_attention=upcast_attention,
149
+ resnet_time_scale_shift=resnet_time_scale_shift,
150
+
151
+ unet_use_cross_frame_attention=unet_use_cross_frame_attention,
152
+ unet_use_temporal_attention=unet_use_temporal_attention,
153
+
154
+ use_motion_module=use_motion_module and (res in motion_module_resolutions) and (not motion_module_decoder_only),
155
+ motion_module_type=motion_module_type,
156
+ motion_module_kwargs=motion_module_kwargs,
157
+ )
158
+ self.down_blocks.append(down_block)
159
+
160
+ # mid
161
+ if mid_block_type == "UNetMidBlock3DCrossAttn":
162
+ self.mid_block = UNetMidBlock3DCrossAttn(
163
+ in_channels=block_out_channels[-1],
164
+ temb_channels=time_embed_dim,
165
+ resnet_eps=norm_eps,
166
+ resnet_act_fn=act_fn,
167
+ output_scale_factor=mid_block_scale_factor,
168
+ resnet_time_scale_shift=resnet_time_scale_shift,
169
+ cross_attention_dim=cross_attention_dim,
170
+ attn_num_head_channels=attention_head_dim[-1],
171
+ resnet_groups=norm_num_groups,
172
+ dual_cross_attention=dual_cross_attention,
173
+ use_linear_projection=use_linear_projection,
174
+ upcast_attention=upcast_attention,
175
+
176
+ unet_use_cross_frame_attention=unet_use_cross_frame_attention,
177
+ unet_use_temporal_attention=unet_use_temporal_attention,
178
+
179
+ use_motion_module=use_motion_module and motion_module_mid_block,
180
+ motion_module_type=motion_module_type,
181
+ motion_module_kwargs=motion_module_kwargs,
182
+ )
183
+ else:
184
+ raise ValueError(f"unknown mid_block_type : {mid_block_type}")
185
+
186
+ # count how many layers upsample the videos
187
+ self.num_upsamplers = 0
188
+
189
+ # up
190
+ reversed_block_out_channels = list(reversed(block_out_channels))
191
+ reversed_attention_head_dim = list(reversed(attention_head_dim))
192
+ only_cross_attention = list(reversed(only_cross_attention))
193
+ output_channel = reversed_block_out_channels[0]
194
+ for i, up_block_type in enumerate(up_block_types):
195
+ res = 2 ** (3 - i)
196
+ is_final_block = i == len(block_out_channels) - 1
197
+
198
+ prev_output_channel = output_channel
199
+ output_channel = reversed_block_out_channels[i]
200
+ input_channel = reversed_block_out_channels[min(i + 1, len(block_out_channels) - 1)]
201
+
202
+ # add upsample block for all BUT final layer
203
+ if not is_final_block:
204
+ add_upsample = True
205
+ self.num_upsamplers += 1
206
+ else:
207
+ add_upsample = False
208
+
209
+ up_block = get_up_block(
210
+ up_block_type,
211
+ num_layers=layers_per_block + 1,
212
+ in_channels=input_channel,
213
+ out_channels=output_channel,
214
+ prev_output_channel=prev_output_channel,
215
+ temb_channels=time_embed_dim,
216
+ add_upsample=add_upsample,
217
+ resnet_eps=norm_eps,
218
+ resnet_act_fn=act_fn,
219
+ resnet_groups=norm_num_groups,
220
+ cross_attention_dim=cross_attention_dim,
221
+ attn_num_head_channels=reversed_attention_head_dim[i],
222
+ dual_cross_attention=dual_cross_attention,
223
+ use_linear_projection=use_linear_projection,
224
+ only_cross_attention=only_cross_attention[i],
225
+ upcast_attention=upcast_attention,
226
+ resnet_time_scale_shift=resnet_time_scale_shift,
227
+
228
+ unet_use_cross_frame_attention=unet_use_cross_frame_attention,
229
+ unet_use_temporal_attention=unet_use_temporal_attention,
230
+
231
+ use_motion_module=use_motion_module and (res in motion_module_resolutions),
232
+ motion_module_type=motion_module_type,
233
+ motion_module_kwargs=motion_module_kwargs,
234
+ )
235
+ self.up_blocks.append(up_block)
236
+ prev_output_channel = output_channel
237
+
238
+ # out
239
+ self.conv_norm_out = nn.GroupNorm(num_channels=block_out_channels[0], num_groups=norm_num_groups, eps=norm_eps)
240
+ self.conv_act = nn.SiLU()
241
+ self.conv_out = InflatedConv3d(block_out_channels[0], out_channels, kernel_size=3, padding=1)
242
+
243
+ def set_attention_slice(self, slice_size):
244
+ r"""
245
+ Enable sliced attention computation.
246
+
247
+ When this option is enabled, the attention module will split the input tensor in slices, to compute attention
248
+ in several steps. This is useful to save some memory in exchange for a small speed decrease.
249
+
250
+ Args:
251
+ slice_size (`str` or `int` or `list(int)`, *optional*, defaults to `"auto"`):
252
+ When `"auto"`, halves the input to the attention heads, so attention will be computed in two steps. If
253
+ `"max"`, maxium amount of memory will be saved by running only one slice at a time. If a number is
254
+ provided, uses as many slices as `attention_head_dim // slice_size`. In this case, `attention_head_dim`
255
+ must be a multiple of `slice_size`.
256
+ """
257
+ sliceable_head_dims = []
258
+
259
+ def fn_recursive_retrieve_slicable_dims(module: torch.nn.Module):
260
+ if hasattr(module, "set_attention_slice"):
261
+ sliceable_head_dims.append(module.sliceable_head_dim)
262
+
263
+ for child in module.children():
264
+ fn_recursive_retrieve_slicable_dims(child)
265
+
266
+ # retrieve number of attention layers
267
+ for module in self.children():
268
+ fn_recursive_retrieve_slicable_dims(module)
269
+
270
+ num_slicable_layers = len(sliceable_head_dims)
271
+
272
+ if slice_size == "auto":
273
+ # half the attention head size is usually a good trade-off between
274
+ # speed and memory
275
+ slice_size = [dim // 2 for dim in sliceable_head_dims]
276
+ elif slice_size == "max":
277
+ # make smallest slice possible
278
+ slice_size = num_slicable_layers * [1]
279
+
280
+ slice_size = num_slicable_layers * [slice_size] if not isinstance(slice_size, list) else slice_size
281
+
282
+ if len(slice_size) != len(sliceable_head_dims):
283
+ raise ValueError(
284
+ f"You have provided {len(slice_size)}, but {self.config} has {len(sliceable_head_dims)} different"
285
+ f" attention layers. Make sure to match `len(slice_size)` to be {len(sliceable_head_dims)}."
286
+ )
287
+
288
+ for i in range(len(slice_size)):
289
+ size = slice_size[i]
290
+ dim = sliceable_head_dims[i]
291
+ if size is not None and size > dim:
292
+ raise ValueError(f"size {size} has to be smaller or equal to {dim}.")
293
+
294
+ # Recursively walk through all the children.
295
+ # Any children which exposes the set_attention_slice method
296
+ # gets the message
297
+ def fn_recursive_set_attention_slice(module: torch.nn.Module, slice_size: List[int]):
298
+ if hasattr(module, "set_attention_slice"):
299
+ module.set_attention_slice(slice_size.pop())
300
+
301
+ for child in module.children():
302
+ fn_recursive_set_attention_slice(child, slice_size)
303
+
304
+ reversed_slice_size = list(reversed(slice_size))
305
+ for module in self.children():
306
+ fn_recursive_set_attention_slice(module, reversed_slice_size)
307
+
308
+ def _set_gradient_checkpointing(self, module, value=False):
309
+ if isinstance(module, (CrossAttnDownBlock3D, DownBlock3D, CrossAttnUpBlock3D, UpBlock3D)):
310
+ module.gradient_checkpointing = value
311
+
312
+ def forward(
313
+ self,
314
+ sample: torch.FloatTensor,
315
+ timestep: Union[torch.Tensor, float, int],
316
+ encoder_hidden_states: torch.Tensor,
317
+ class_labels: Optional[torch.Tensor] = None,
318
+ attention_mask: Optional[torch.Tensor] = None,
319
+ return_dict: bool = True,
320
+ ) -> Union[UNet3DConditionOutput, Tuple]:
321
+ r"""
322
+ Args:
323
+ sample (`torch.FloatTensor`): (batch, channel, height, width) noisy inputs tensor
324
+ timestep (`torch.FloatTensor` or `float` or `int`): (batch) timesteps
325
+ encoder_hidden_states (`torch.FloatTensor`): (batch, sequence_length, feature_dim) encoder hidden states
326
+ return_dict (`bool`, *optional*, defaults to `True`):
327
+ Whether or not to return a [`models.unet_2d_condition.UNet2DConditionOutput`] instead of a plain tuple.
328
+
329
+ Returns:
330
+ [`~models.unet_2d_condition.UNet2DConditionOutput`] or `tuple`:
331
+ [`~models.unet_2d_condition.UNet2DConditionOutput`] if `return_dict` is True, otherwise a `tuple`. When
332
+ returning a tuple, the first element is the sample tensor.
333
+ """
334
+ # By default samples have to be AT least a multiple of the overall upsampling factor.
335
+ # The overall upsampling factor is equal to 2 ** (# num of upsampling layears).
336
+ # However, the upsampling interpolation output size can be forced to fit any upsampling size
337
+ # on the fly if necessary.
338
+ default_overall_up_factor = 2**self.num_upsamplers
339
+
340
+ # upsample size should be forwarded when sample is not a multiple of `default_overall_up_factor`
341
+ forward_upsample_size = False
342
+ upsample_size = None
343
+
344
+ if any(s % default_overall_up_factor != 0 for s in sample.shape[-2:]):
345
+ logger.info("Forward upsample size to force interpolation output size.")
346
+ forward_upsample_size = True
347
+
348
+ # prepare attention_mask
349
+ if attention_mask is not None:
350
+ attention_mask = (1 - attention_mask.to(sample.dtype)) * -10000.0
351
+ attention_mask = attention_mask.unsqueeze(1)
352
+
353
+ # center input if necessary
354
+ if self.config.center_input_sample:
355
+ sample = 2 * sample - 1.0
356
+
357
+ # time
358
+ timesteps = timestep
359
+ if not torch.is_tensor(timesteps):
360
+ # This would be a good case for the `match` statement (Python 3.10+)
361
+ is_mps = sample.device.type == "mps"
362
+ if isinstance(timestep, float):
363
+ dtype = torch.float32 if is_mps else torch.float64
364
+ else:
365
+ dtype = torch.int32 if is_mps else torch.int64
366
+ timesteps = torch.tensor([timesteps], dtype=dtype, device=sample.device)
367
+ elif len(timesteps.shape) == 0:
368
+ timesteps = timesteps[None].to(sample.device)
369
+
370
+ # broadcast to batch dimension in a way that's compatible with ONNX/Core ML
371
+ timesteps = timesteps.expand(sample.shape[0])
372
+
373
+ t_emb = self.time_proj(timesteps)
374
+
375
+ # timesteps does not contain any weights and will always return f32 tensors
376
+ # but time_embedding might actually be running in fp16. so we need to cast here.
377
+ # there might be better ways to encapsulate this.
378
+ t_emb = t_emb.to(dtype=self.dtype)
379
+ emb = self.time_embedding(t_emb)
380
+
381
+ if self.class_embedding is not None:
382
+ if class_labels is None:
383
+ raise ValueError("class_labels should be provided when num_class_embeds > 0")
384
+
385
+ if self.config.class_embed_type == "timestep":
386
+ class_labels = self.time_proj(class_labels)
387
+
388
+ class_emb = self.class_embedding(class_labels).to(dtype=self.dtype)
389
+ emb = emb + class_emb
390
+
391
+ # pre-process
392
+ sample = self.conv_in(sample)
393
+
394
+ # down
395
+ down_block_res_samples = (sample,)
396
+ for downsample_block in self.down_blocks:
397
+ if hasattr(downsample_block, "has_cross_attention") and downsample_block.has_cross_attention:
398
+ sample, res_samples = downsample_block(
399
+ hidden_states=sample,
400
+ temb=emb,
401
+ encoder_hidden_states=encoder_hidden_states,
402
+ attention_mask=attention_mask,
403
+ )
404
+ else:
405
+ sample, res_samples = downsample_block(hidden_states=sample, temb=emb, encoder_hidden_states=encoder_hidden_states)
406
+
407
+ down_block_res_samples += res_samples
408
+
409
+ # mid
410
+ sample = self.mid_block(
411
+ sample, emb, encoder_hidden_states=encoder_hidden_states, attention_mask=attention_mask
412
+ )
413
+
414
+ # up
415
+ for i, upsample_block in enumerate(self.up_blocks):
416
+ is_final_block = i == len(self.up_blocks) - 1
417
+
418
+ res_samples = down_block_res_samples[-len(upsample_block.resnets) :]
419
+ down_block_res_samples = down_block_res_samples[: -len(upsample_block.resnets)]
420
+
421
+ # if we have not reached the final block and need to forward the
422
+ # upsample size, we do it here
423
+ if not is_final_block and forward_upsample_size:
424
+ upsample_size = down_block_res_samples[-1].shape[2:]
425
+
426
+ if hasattr(upsample_block, "has_cross_attention") and upsample_block.has_cross_attention:
427
+ sample = upsample_block(
428
+ hidden_states=sample,
429
+ temb=emb,
430
+ res_hidden_states_tuple=res_samples,
431
+ encoder_hidden_states=encoder_hidden_states,
432
+ upsample_size=upsample_size,
433
+ attention_mask=attention_mask,
434
+ )
435
+ else:
436
+ sample = upsample_block(
437
+ hidden_states=sample, temb=emb, res_hidden_states_tuple=res_samples, upsample_size=upsample_size, encoder_hidden_states=encoder_hidden_states,
438
+ )
439
+
440
+ # post-process
441
+ sample = self.conv_norm_out(sample)
442
+ sample = self.conv_act(sample)
443
+ sample = self.conv_out(sample)
444
+
445
+ if not return_dict:
446
+ return (sample,)
447
+
448
+ return UNet3DConditionOutput(sample=sample)
449
+
450
+ @classmethod
451
+ def from_pretrained_2d(cls, pretrained_model_path, subfolder=None, unet_additional_kwargs=None):
452
+ if subfolder is not None:
453
+ pretrained_model_path = os.path.join(pretrained_model_path, subfolder)
454
+ print(f"loaded temporal unet's pretrained weights from {pretrained_model_path} ...")
455
+
456
+ config_file = os.path.join(pretrained_model_path, 'config.json')
457
+ if not os.path.isfile(config_file):
458
+ raise RuntimeError(f"{config_file} does not exist")
459
+ with open(config_file, "r") as f:
460
+ config = json.load(f)
461
+ config["_class_name"] = cls.__name__
462
+ config["down_block_types"] = [
463
+ "CrossAttnDownBlock3D",
464
+ "CrossAttnDownBlock3D",
465
+ "CrossAttnDownBlock3D",
466
+ "DownBlock3D"
467
+ ]
468
+ config["up_block_types"] = [
469
+ "UpBlock3D",
470
+ "CrossAttnUpBlock3D",
471
+ "CrossAttnUpBlock3D",
472
+ "CrossAttnUpBlock3D"
473
+ ]
474
+
475
+ from diffusers.utils import WEIGHTS_NAME
476
+ model = cls.from_config(config, **unet_additional_kwargs)
477
+ model_file = os.path.join(pretrained_model_path, WEIGHTS_NAME)
478
+ if not os.path.isfile(model_file):
479
+ raise RuntimeError(f"{model_file} does not exist")
480
+ state_dict = torch.load(model_file, map_location="cpu")
481
+
482
+ m, u = model.load_state_dict(state_dict, strict=False)
483
+ print(f"### missing keys: {len(m)}; \n### unexpected keys: {len(u)};")
484
+ # print(f"### missing keys:\n{m}\n### unexpected keys:\n{u}\n")
485
+
486
+ params = [p.numel() if "temporal" in n else 0 for n, p in model.named_parameters()]
487
+ print(f"### Temporal Module Parameters: {sum(params) / 1e6} M")
488
+
489
+ return model
animatediff/models/unet_blocks.py ADDED
@@ -0,0 +1,733 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Adapted from https://github.com/huggingface/diffusers/blob/main/src/diffusers/models/unet_2d_blocks.py
2
+
3
+ import torch
4
+ from torch import nn
5
+
6
+ from .attention import Transformer3DModel
7
+ from .resnet import Downsample3D, ResnetBlock3D, Upsample3D
8
+ from .motion_module import get_motion_module
9
+
10
+ import pdb
11
+
12
+ def get_down_block(
13
+ down_block_type,
14
+ num_layers,
15
+ in_channels,
16
+ out_channels,
17
+ temb_channels,
18
+ add_downsample,
19
+ resnet_eps,
20
+ resnet_act_fn,
21
+ attn_num_head_channels,
22
+ resnet_groups=None,
23
+ cross_attention_dim=None,
24
+ downsample_padding=None,
25
+ dual_cross_attention=False,
26
+ use_linear_projection=False,
27
+ only_cross_attention=False,
28
+ upcast_attention=False,
29
+ resnet_time_scale_shift="default",
30
+
31
+ unet_use_cross_frame_attention=None,
32
+ unet_use_temporal_attention=None,
33
+
34
+ use_motion_module=None,
35
+
36
+ motion_module_type=None,
37
+ motion_module_kwargs=None,
38
+ ):
39
+ down_block_type = down_block_type[7:] if down_block_type.startswith("UNetRes") else down_block_type
40
+ if down_block_type == "DownBlock3D":
41
+ return DownBlock3D(
42
+ num_layers=num_layers,
43
+ in_channels=in_channels,
44
+ out_channels=out_channels,
45
+ temb_channels=temb_channels,
46
+ add_downsample=add_downsample,
47
+ resnet_eps=resnet_eps,
48
+ resnet_act_fn=resnet_act_fn,
49
+ resnet_groups=resnet_groups,
50
+ downsample_padding=downsample_padding,
51
+ resnet_time_scale_shift=resnet_time_scale_shift,
52
+
53
+ use_motion_module=use_motion_module,
54
+ motion_module_type=motion_module_type,
55
+ motion_module_kwargs=motion_module_kwargs,
56
+ )
57
+ elif down_block_type == "CrossAttnDownBlock3D":
58
+ if cross_attention_dim is None:
59
+ raise ValueError("cross_attention_dim must be specified for CrossAttnDownBlock3D")
60
+ return CrossAttnDownBlock3D(
61
+ num_layers=num_layers,
62
+ in_channels=in_channels,
63
+ out_channels=out_channels,
64
+ temb_channels=temb_channels,
65
+ add_downsample=add_downsample,
66
+ resnet_eps=resnet_eps,
67
+ resnet_act_fn=resnet_act_fn,
68
+ resnet_groups=resnet_groups,
69
+ downsample_padding=downsample_padding,
70
+ cross_attention_dim=cross_attention_dim,
71
+ attn_num_head_channels=attn_num_head_channels,
72
+ dual_cross_attention=dual_cross_attention,
73
+ use_linear_projection=use_linear_projection,
74
+ only_cross_attention=only_cross_attention,
75
+ upcast_attention=upcast_attention,
76
+ resnet_time_scale_shift=resnet_time_scale_shift,
77
+
78
+ unet_use_cross_frame_attention=unet_use_cross_frame_attention,
79
+ unet_use_temporal_attention=unet_use_temporal_attention,
80
+
81
+ use_motion_module=use_motion_module,
82
+ motion_module_type=motion_module_type,
83
+ motion_module_kwargs=motion_module_kwargs,
84
+ )
85
+ raise ValueError(f"{down_block_type} does not exist.")
86
+
87
+
88
+ def get_up_block(
89
+ up_block_type,
90
+ num_layers,
91
+ in_channels,
92
+ out_channels,
93
+ prev_output_channel,
94
+ temb_channels,
95
+ add_upsample,
96
+ resnet_eps,
97
+ resnet_act_fn,
98
+ attn_num_head_channels,
99
+ resnet_groups=None,
100
+ cross_attention_dim=None,
101
+ dual_cross_attention=False,
102
+ use_linear_projection=False,
103
+ only_cross_attention=False,
104
+ upcast_attention=False,
105
+ resnet_time_scale_shift="default",
106
+
107
+ unet_use_cross_frame_attention=None,
108
+ unet_use_temporal_attention=None,
109
+
110
+ use_motion_module=None,
111
+ motion_module_type=None,
112
+ motion_module_kwargs=None,
113
+ ):
114
+ up_block_type = up_block_type[7:] if up_block_type.startswith("UNetRes") else up_block_type
115
+ if up_block_type == "UpBlock3D":
116
+ return UpBlock3D(
117
+ num_layers=num_layers,
118
+ in_channels=in_channels,
119
+ out_channels=out_channels,
120
+ prev_output_channel=prev_output_channel,
121
+ temb_channels=temb_channels,
122
+ add_upsample=add_upsample,
123
+ resnet_eps=resnet_eps,
124
+ resnet_act_fn=resnet_act_fn,
125
+ resnet_groups=resnet_groups,
126
+ resnet_time_scale_shift=resnet_time_scale_shift,
127
+
128
+ use_motion_module=use_motion_module,
129
+ motion_module_type=motion_module_type,
130
+ motion_module_kwargs=motion_module_kwargs,
131
+ )
132
+ elif up_block_type == "CrossAttnUpBlock3D":
133
+ if cross_attention_dim is None:
134
+ raise ValueError("cross_attention_dim must be specified for CrossAttnUpBlock3D")
135
+ return CrossAttnUpBlock3D(
136
+ num_layers=num_layers,
137
+ in_channels=in_channels,
138
+ out_channels=out_channels,
139
+ prev_output_channel=prev_output_channel,
140
+ temb_channels=temb_channels,
141
+ add_upsample=add_upsample,
142
+ resnet_eps=resnet_eps,
143
+ resnet_act_fn=resnet_act_fn,
144
+ resnet_groups=resnet_groups,
145
+ cross_attention_dim=cross_attention_dim,
146
+ attn_num_head_channels=attn_num_head_channels,
147
+ dual_cross_attention=dual_cross_attention,
148
+ use_linear_projection=use_linear_projection,
149
+ only_cross_attention=only_cross_attention,
150
+ upcast_attention=upcast_attention,
151
+ resnet_time_scale_shift=resnet_time_scale_shift,
152
+
153
+ unet_use_cross_frame_attention=unet_use_cross_frame_attention,
154
+ unet_use_temporal_attention=unet_use_temporal_attention,
155
+
156
+ use_motion_module=use_motion_module,
157
+ motion_module_type=motion_module_type,
158
+ motion_module_kwargs=motion_module_kwargs,
159
+ )
160
+ raise ValueError(f"{up_block_type} does not exist.")
161
+
162
+
163
+ class UNetMidBlock3DCrossAttn(nn.Module):
164
+ def __init__(
165
+ self,
166
+ in_channels: int,
167
+ temb_channels: int,
168
+ dropout: float = 0.0,
169
+ num_layers: int = 1,
170
+ resnet_eps: float = 1e-6,
171
+ resnet_time_scale_shift: str = "default",
172
+ resnet_act_fn: str = "swish",
173
+ resnet_groups: int = 32,
174
+ resnet_pre_norm: bool = True,
175
+ attn_num_head_channels=1,
176
+ output_scale_factor=1.0,
177
+ cross_attention_dim=1280,
178
+ dual_cross_attention=False,
179
+ use_linear_projection=False,
180
+ upcast_attention=False,
181
+
182
+ unet_use_cross_frame_attention=None,
183
+ unet_use_temporal_attention=None,
184
+
185
+ use_motion_module=None,
186
+
187
+ motion_module_type=None,
188
+ motion_module_kwargs=None,
189
+ ):
190
+ super().__init__()
191
+
192
+ self.has_cross_attention = True
193
+ self.attn_num_head_channels = attn_num_head_channels
194
+ resnet_groups = resnet_groups if resnet_groups is not None else min(in_channels // 4, 32)
195
+
196
+ # there is always at least one resnet
197
+ resnets = [
198
+ ResnetBlock3D(
199
+ in_channels=in_channels,
200
+ out_channels=in_channels,
201
+ temb_channels=temb_channels,
202
+ eps=resnet_eps,
203
+ groups=resnet_groups,
204
+ dropout=dropout,
205
+ time_embedding_norm=resnet_time_scale_shift,
206
+ non_linearity=resnet_act_fn,
207
+ output_scale_factor=output_scale_factor,
208
+ pre_norm=resnet_pre_norm,
209
+ )
210
+ ]
211
+ attentions = []
212
+ motion_modules = []
213
+
214
+ for _ in range(num_layers):
215
+ if dual_cross_attention:
216
+ raise NotImplementedError
217
+ attentions.append(
218
+ Transformer3DModel(
219
+ attn_num_head_channels,
220
+ in_channels // attn_num_head_channels,
221
+ in_channels=in_channels,
222
+ num_layers=1,
223
+ cross_attention_dim=cross_attention_dim,
224
+ norm_num_groups=resnet_groups,
225
+ use_linear_projection=use_linear_projection,
226
+ upcast_attention=upcast_attention,
227
+
228
+ unet_use_cross_frame_attention=unet_use_cross_frame_attention,
229
+ unet_use_temporal_attention=unet_use_temporal_attention,
230
+ )
231
+ )
232
+ motion_modules.append(
233
+ get_motion_module(
234
+ in_channels=in_channels,
235
+ motion_module_type=motion_module_type,
236
+ motion_module_kwargs=motion_module_kwargs,
237
+ ) if use_motion_module else None
238
+ )
239
+ resnets.append(
240
+ ResnetBlock3D(
241
+ in_channels=in_channels,
242
+ out_channels=in_channels,
243
+ temb_channels=temb_channels,
244
+ eps=resnet_eps,
245
+ groups=resnet_groups,
246
+ dropout=dropout,
247
+ time_embedding_norm=resnet_time_scale_shift,
248
+ non_linearity=resnet_act_fn,
249
+ output_scale_factor=output_scale_factor,
250
+ pre_norm=resnet_pre_norm,
251
+ )
252
+ )
253
+
254
+ self.attentions = nn.ModuleList(attentions)
255
+ self.resnets = nn.ModuleList(resnets)
256
+ self.motion_modules = nn.ModuleList(motion_modules)
257
+
258
+ def forward(self, hidden_states, temb=None, encoder_hidden_states=None, attention_mask=None):
259
+ hidden_states = self.resnets[0](hidden_states, temb)
260
+ for attn, resnet, motion_module in zip(self.attentions, self.resnets[1:], self.motion_modules):
261
+ hidden_states = attn(hidden_states, encoder_hidden_states=encoder_hidden_states).sample
262
+ hidden_states = motion_module(hidden_states, temb, encoder_hidden_states=encoder_hidden_states) if motion_module is not None else hidden_states
263
+ hidden_states = resnet(hidden_states, temb)
264
+
265
+ return hidden_states
266
+
267
+
268
+ class CrossAttnDownBlock3D(nn.Module):
269
+ def __init__(
270
+ self,
271
+ in_channels: int,
272
+ out_channels: int,
273
+ temb_channels: int,
274
+ dropout: float = 0.0,
275
+ num_layers: int = 1,
276
+ resnet_eps: float = 1e-6,
277
+ resnet_time_scale_shift: str = "default",
278
+ resnet_act_fn: str = "swish",
279
+ resnet_groups: int = 32,
280
+ resnet_pre_norm: bool = True,
281
+ attn_num_head_channels=1,
282
+ cross_attention_dim=1280,
283
+ output_scale_factor=1.0,
284
+ downsample_padding=1,
285
+ add_downsample=True,
286
+ dual_cross_attention=False,
287
+ use_linear_projection=False,
288
+ only_cross_attention=False,
289
+ upcast_attention=False,
290
+
291
+ unet_use_cross_frame_attention=None,
292
+ unet_use_temporal_attention=None,
293
+
294
+ use_motion_module=None,
295
+
296
+ motion_module_type=None,
297
+ motion_module_kwargs=None,
298
+ ):
299
+ super().__init__()
300
+ resnets = []
301
+ attentions = []
302
+ motion_modules = []
303
+
304
+ self.has_cross_attention = True
305
+ self.attn_num_head_channels = attn_num_head_channels
306
+
307
+ for i in range(num_layers):
308
+ in_channels = in_channels if i == 0 else out_channels
309
+ resnets.append(
310
+ ResnetBlock3D(
311
+ in_channels=in_channels,
312
+ out_channels=out_channels,
313
+ temb_channels=temb_channels,
314
+ eps=resnet_eps,
315
+ groups=resnet_groups,
316
+ dropout=dropout,
317
+ time_embedding_norm=resnet_time_scale_shift,
318
+ non_linearity=resnet_act_fn,
319
+ output_scale_factor=output_scale_factor,
320
+ pre_norm=resnet_pre_norm,
321
+ )
322
+ )
323
+ if dual_cross_attention:
324
+ raise NotImplementedError
325
+ attentions.append(
326
+ Transformer3DModel(
327
+ attn_num_head_channels,
328
+ out_channels // attn_num_head_channels,
329
+ in_channels=out_channels,
330
+ num_layers=1,
331
+ cross_attention_dim=cross_attention_dim,
332
+ norm_num_groups=resnet_groups,
333
+ use_linear_projection=use_linear_projection,
334
+ only_cross_attention=only_cross_attention,
335
+ upcast_attention=upcast_attention,
336
+
337
+ unet_use_cross_frame_attention=unet_use_cross_frame_attention,
338
+ unet_use_temporal_attention=unet_use_temporal_attention,
339
+ )
340
+ )
341
+ motion_modules.append(
342
+ get_motion_module(
343
+ in_channels=out_channels,
344
+ motion_module_type=motion_module_type,
345
+ motion_module_kwargs=motion_module_kwargs,
346
+ ) if use_motion_module else None
347
+ )
348
+
349
+ self.attentions = nn.ModuleList(attentions)
350
+ self.resnets = nn.ModuleList(resnets)
351
+ self.motion_modules = nn.ModuleList(motion_modules)
352
+
353
+ if add_downsample:
354
+ self.downsamplers = nn.ModuleList(
355
+ [
356
+ Downsample3D(
357
+ out_channels, use_conv=True, out_channels=out_channels, padding=downsample_padding, name="op"
358
+ )
359
+ ]
360
+ )
361
+ else:
362
+ self.downsamplers = None
363
+
364
+ self.gradient_checkpointing = False
365
+
366
+ def forward(self, hidden_states, temb=None, encoder_hidden_states=None, attention_mask=None):
367
+ output_states = ()
368
+
369
+ for resnet, attn, motion_module in zip(self.resnets, self.attentions, self.motion_modules):
370
+ if self.training and self.gradient_checkpointing:
371
+
372
+ def create_custom_forward(module, return_dict=None):
373
+ def custom_forward(*inputs):
374
+ if return_dict is not None:
375
+ return module(*inputs, return_dict=return_dict)
376
+ else:
377
+ return module(*inputs)
378
+
379
+ return custom_forward
380
+
381
+ hidden_states = torch.utils.checkpoint.checkpoint(create_custom_forward(resnet), hidden_states, temb)
382
+ hidden_states = torch.utils.checkpoint.checkpoint(
383
+ create_custom_forward(attn, return_dict=False),
384
+ hidden_states,
385
+ encoder_hidden_states,
386
+ )[0]
387
+ if motion_module is not None:
388
+ hidden_states = torch.utils.checkpoint.checkpoint(create_custom_forward(motion_module), hidden_states.requires_grad_(), temb, encoder_hidden_states)
389
+
390
+ else:
391
+ hidden_states = resnet(hidden_states, temb)
392
+ hidden_states = attn(hidden_states, encoder_hidden_states=encoder_hidden_states).sample
393
+
394
+ # add motion module
395
+ hidden_states = motion_module(hidden_states, temb, encoder_hidden_states=encoder_hidden_states) if motion_module is not None else hidden_states
396
+
397
+ output_states += (hidden_states,)
398
+
399
+ if self.downsamplers is not None:
400
+ for downsampler in self.downsamplers:
401
+ hidden_states = downsampler(hidden_states)
402
+
403
+ output_states += (hidden_states,)
404
+
405
+ return hidden_states, output_states
406
+
407
+
408
+ class DownBlock3D(nn.Module):
409
+ def __init__(
410
+ self,
411
+ in_channels: int,
412
+ out_channels: int,
413
+ temb_channels: int,
414
+ dropout: float = 0.0,
415
+ num_layers: int = 1,
416
+ resnet_eps: float = 1e-6,
417
+ resnet_time_scale_shift: str = "default",
418
+ resnet_act_fn: str = "swish",
419
+ resnet_groups: int = 32,
420
+ resnet_pre_norm: bool = True,
421
+ output_scale_factor=1.0,
422
+ add_downsample=True,
423
+ downsample_padding=1,
424
+
425
+ use_motion_module=None,
426
+ motion_module_type=None,
427
+ motion_module_kwargs=None,
428
+ ):
429
+ super().__init__()
430
+ resnets = []
431
+ motion_modules = []
432
+
433
+ for i in range(num_layers):
434
+ in_channels = in_channels if i == 0 else out_channels
435
+ resnets.append(
436
+ ResnetBlock3D(
437
+ in_channels=in_channels,
438
+ out_channels=out_channels,
439
+ temb_channels=temb_channels,
440
+ eps=resnet_eps,
441
+ groups=resnet_groups,
442
+ dropout=dropout,
443
+ time_embedding_norm=resnet_time_scale_shift,
444
+ non_linearity=resnet_act_fn,
445
+ output_scale_factor=output_scale_factor,
446
+ pre_norm=resnet_pre_norm,
447
+ )
448
+ )
449
+ motion_modules.append(
450
+ get_motion_module(
451
+ in_channels=out_channels,
452
+ motion_module_type=motion_module_type,
453
+ motion_module_kwargs=motion_module_kwargs,
454
+ ) if use_motion_module else None
455
+ )
456
+
457
+ self.resnets = nn.ModuleList(resnets)
458
+ self.motion_modules = nn.ModuleList(motion_modules)
459
+
460
+ if add_downsample:
461
+ self.downsamplers = nn.ModuleList(
462
+ [
463
+ Downsample3D(
464
+ out_channels, use_conv=True, out_channels=out_channels, padding=downsample_padding, name="op"
465
+ )
466
+ ]
467
+ )
468
+ else:
469
+ self.downsamplers = None
470
+
471
+ self.gradient_checkpointing = False
472
+
473
+ def forward(self, hidden_states, temb=None, encoder_hidden_states=None):
474
+ output_states = ()
475
+
476
+ for resnet, motion_module in zip(self.resnets, self.motion_modules):
477
+ if self.training and self.gradient_checkpointing:
478
+ def create_custom_forward(module):
479
+ def custom_forward(*inputs):
480
+ return module(*inputs)
481
+
482
+ return custom_forward
483
+
484
+ hidden_states = torch.utils.checkpoint.checkpoint(create_custom_forward(resnet), hidden_states, temb)
485
+ if motion_module is not None:
486
+ hidden_states = torch.utils.checkpoint.checkpoint(create_custom_forward(motion_module), hidden_states.requires_grad_(), temb, encoder_hidden_states)
487
+ else:
488
+ hidden_states = resnet(hidden_states, temb)
489
+
490
+ # add motion module
491
+ hidden_states = motion_module(hidden_states, temb, encoder_hidden_states=encoder_hidden_states) if motion_module is not None else hidden_states
492
+
493
+ output_states += (hidden_states,)
494
+
495
+ if self.downsamplers is not None:
496
+ for downsampler in self.downsamplers:
497
+ hidden_states = downsampler(hidden_states)
498
+
499
+ output_states += (hidden_states,)
500
+
501
+ return hidden_states, output_states
502
+
503
+
504
+ class CrossAttnUpBlock3D(nn.Module):
505
+ def __init__(
506
+ self,
507
+ in_channels: int,
508
+ out_channels: int,
509
+ prev_output_channel: int,
510
+ temb_channels: int,
511
+ dropout: float = 0.0,
512
+ num_layers: int = 1,
513
+ resnet_eps: float = 1e-6,
514
+ resnet_time_scale_shift: str = "default",
515
+ resnet_act_fn: str = "swish",
516
+ resnet_groups: int = 32,
517
+ resnet_pre_norm: bool = True,
518
+ attn_num_head_channels=1,
519
+ cross_attention_dim=1280,
520
+ output_scale_factor=1.0,
521
+ add_upsample=True,
522
+ dual_cross_attention=False,
523
+ use_linear_projection=False,
524
+ only_cross_attention=False,
525
+ upcast_attention=False,
526
+
527
+ unet_use_cross_frame_attention=None,
528
+ unet_use_temporal_attention=None,
529
+
530
+ use_motion_module=None,
531
+
532
+ motion_module_type=None,
533
+ motion_module_kwargs=None,
534
+ ):
535
+ super().__init__()
536
+ resnets = []
537
+ attentions = []
538
+ motion_modules = []
539
+
540
+ self.has_cross_attention = True
541
+ self.attn_num_head_channels = attn_num_head_channels
542
+
543
+ for i in range(num_layers):
544
+ res_skip_channels = in_channels if (i == num_layers - 1) else out_channels
545
+ resnet_in_channels = prev_output_channel if i == 0 else out_channels
546
+
547
+ resnets.append(
548
+ ResnetBlock3D(
549
+ in_channels=resnet_in_channels + res_skip_channels,
550
+ out_channels=out_channels,
551
+ temb_channels=temb_channels,
552
+ eps=resnet_eps,
553
+ groups=resnet_groups,
554
+ dropout=dropout,
555
+ time_embedding_norm=resnet_time_scale_shift,
556
+ non_linearity=resnet_act_fn,
557
+ output_scale_factor=output_scale_factor,
558
+ pre_norm=resnet_pre_norm,
559
+ )
560
+ )
561
+ if dual_cross_attention:
562
+ raise NotImplementedError
563
+ attentions.append(
564
+ Transformer3DModel(
565
+ attn_num_head_channels,
566
+ out_channels // attn_num_head_channels,
567
+ in_channels=out_channels,
568
+ num_layers=1,
569
+ cross_attention_dim=cross_attention_dim,
570
+ norm_num_groups=resnet_groups,
571
+ use_linear_projection=use_linear_projection,
572
+ only_cross_attention=only_cross_attention,
573
+ upcast_attention=upcast_attention,
574
+
575
+ unet_use_cross_frame_attention=unet_use_cross_frame_attention,
576
+ unet_use_temporal_attention=unet_use_temporal_attention,
577
+ )
578
+ )
579
+ motion_modules.append(
580
+ get_motion_module(
581
+ in_channels=out_channels,
582
+ motion_module_type=motion_module_type,
583
+ motion_module_kwargs=motion_module_kwargs,
584
+ ) if use_motion_module else None
585
+ )
586
+
587
+ self.attentions = nn.ModuleList(attentions)
588
+ self.resnets = nn.ModuleList(resnets)
589
+ self.motion_modules = nn.ModuleList(motion_modules)
590
+
591
+ if add_upsample:
592
+ self.upsamplers = nn.ModuleList([Upsample3D(out_channels, use_conv=True, out_channels=out_channels)])
593
+ else:
594
+ self.upsamplers = None
595
+
596
+ self.gradient_checkpointing = False
597
+
598
+ def forward(
599
+ self,
600
+ hidden_states,
601
+ res_hidden_states_tuple,
602
+ temb=None,
603
+ encoder_hidden_states=None,
604
+ upsample_size=None,
605
+ attention_mask=None,
606
+ ):
607
+ for resnet, attn, motion_module in zip(self.resnets, self.attentions, self.motion_modules):
608
+ # pop res hidden states
609
+ res_hidden_states = res_hidden_states_tuple[-1]
610
+ res_hidden_states_tuple = res_hidden_states_tuple[:-1]
611
+ hidden_states = torch.cat([hidden_states, res_hidden_states], dim=1)
612
+
613
+ if self.training and self.gradient_checkpointing:
614
+
615
+ def create_custom_forward(module, return_dict=None):
616
+ def custom_forward(*inputs):
617
+ if return_dict is not None:
618
+ return module(*inputs, return_dict=return_dict)
619
+ else:
620
+ return module(*inputs)
621
+
622
+ return custom_forward
623
+
624
+ hidden_states = torch.utils.checkpoint.checkpoint(create_custom_forward(resnet), hidden_states, temb)
625
+ hidden_states = torch.utils.checkpoint.checkpoint(
626
+ create_custom_forward(attn, return_dict=False),
627
+ hidden_states,
628
+ encoder_hidden_states,
629
+ )[0]
630
+ if motion_module is not None:
631
+ hidden_states = torch.utils.checkpoint.checkpoint(create_custom_forward(motion_module), hidden_states.requires_grad_(), temb, encoder_hidden_states)
632
+
633
+ else:
634
+ hidden_states = resnet(hidden_states, temb)
635
+ hidden_states = attn(hidden_states, encoder_hidden_states=encoder_hidden_states).sample
636
+
637
+ # add motion module
638
+ hidden_states = motion_module(hidden_states, temb, encoder_hidden_states=encoder_hidden_states) if motion_module is not None else hidden_states
639
+
640
+ if self.upsamplers is not None:
641
+ for upsampler in self.upsamplers:
642
+ hidden_states = upsampler(hidden_states, upsample_size)
643
+
644
+ return hidden_states
645
+
646
+
647
+ class UpBlock3D(nn.Module):
648
+ def __init__(
649
+ self,
650
+ in_channels: int,
651
+ prev_output_channel: int,
652
+ out_channels: int,
653
+ temb_channels: int,
654
+ dropout: float = 0.0,
655
+ num_layers: int = 1,
656
+ resnet_eps: float = 1e-6,
657
+ resnet_time_scale_shift: str = "default",
658
+ resnet_act_fn: str = "swish",
659
+ resnet_groups: int = 32,
660
+ resnet_pre_norm: bool = True,
661
+ output_scale_factor=1.0,
662
+ add_upsample=True,
663
+
664
+ use_motion_module=None,
665
+ motion_module_type=None,
666
+ motion_module_kwargs=None,
667
+ ):
668
+ super().__init__()
669
+ resnets = []
670
+ motion_modules = []
671
+
672
+ for i in range(num_layers):
673
+ res_skip_channels = in_channels if (i == num_layers - 1) else out_channels
674
+ resnet_in_channels = prev_output_channel if i == 0 else out_channels
675
+
676
+ resnets.append(
677
+ ResnetBlock3D(
678
+ in_channels=resnet_in_channels + res_skip_channels,
679
+ out_channels=out_channels,
680
+ temb_channels=temb_channels,
681
+ eps=resnet_eps,
682
+ groups=resnet_groups,
683
+ dropout=dropout,
684
+ time_embedding_norm=resnet_time_scale_shift,
685
+ non_linearity=resnet_act_fn,
686
+ output_scale_factor=output_scale_factor,
687
+ pre_norm=resnet_pre_norm,
688
+ )
689
+ )
690
+ motion_modules.append(
691
+ get_motion_module(
692
+ in_channels=out_channels,
693
+ motion_module_type=motion_module_type,
694
+ motion_module_kwargs=motion_module_kwargs,
695
+ ) if use_motion_module else None
696
+ )
697
+
698
+ self.resnets = nn.ModuleList(resnets)
699
+ self.motion_modules = nn.ModuleList(motion_modules)
700
+
701
+ if add_upsample:
702
+ self.upsamplers = nn.ModuleList([Upsample3D(out_channels, use_conv=True, out_channels=out_channels)])
703
+ else:
704
+ self.upsamplers = None
705
+
706
+ self.gradient_checkpointing = False
707
+
708
+ def forward(self, hidden_states, res_hidden_states_tuple, temb=None, upsample_size=None, encoder_hidden_states=None,):
709
+ for resnet, motion_module in zip(self.resnets, self.motion_modules):
710
+ # pop res hidden states
711
+ res_hidden_states = res_hidden_states_tuple[-1]
712
+ res_hidden_states_tuple = res_hidden_states_tuple[:-1]
713
+ hidden_states = torch.cat([hidden_states, res_hidden_states], dim=1)
714
+
715
+ if self.training and self.gradient_checkpointing:
716
+ def create_custom_forward(module):
717
+ def custom_forward(*inputs):
718
+ return module(*inputs)
719
+
720
+ return custom_forward
721
+
722
+ hidden_states = torch.utils.checkpoint.checkpoint(create_custom_forward(resnet), hidden_states, temb)
723
+ if motion_module is not None:
724
+ hidden_states = torch.utils.checkpoint.checkpoint(create_custom_forward(motion_module), hidden_states.requires_grad_(), temb, encoder_hidden_states)
725
+ else:
726
+ hidden_states = resnet(hidden_states, temb)
727
+ hidden_states = motion_module(hidden_states, temb, encoder_hidden_states=encoder_hidden_states) if motion_module is not None else hidden_states
728
+
729
+ if self.upsamplers is not None:
730
+ for upsampler in self.upsamplers:
731
+ hidden_states = upsampler(hidden_states, upsample_size)
732
+
733
+ return hidden_states
animatediff/pipelines/pipeline_animation.py ADDED
@@ -0,0 +1,428 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Adapted from https://github.com/showlab/Tune-A-Video/blob/main/tuneavideo/pipelines/pipeline_tuneavideo.py
2
+
3
+ import inspect
4
+ from typing import Callable, List, Optional, Union
5
+ from dataclasses import dataclass
6
+
7
+ import numpy as np
8
+ import torch
9
+ from tqdm import tqdm
10
+
11
+ from diffusers.utils import is_accelerate_available
12
+ from packaging import version
13
+ from transformers import CLIPTextModel, CLIPTokenizer
14
+
15
+ from diffusers.configuration_utils import FrozenDict
16
+ from diffusers.models import AutoencoderKL
17
+ from diffusers.pipeline_utils import DiffusionPipeline
18
+ from diffusers.schedulers import (
19
+ DDIMScheduler,
20
+ DPMSolverMultistepScheduler,
21
+ EulerAncestralDiscreteScheduler,
22
+ EulerDiscreteScheduler,
23
+ LMSDiscreteScheduler,
24
+ PNDMScheduler,
25
+ )
26
+ from diffusers.utils import deprecate, logging, BaseOutput
27
+
28
+ from einops import rearrange
29
+
30
+ from ..models.unet import UNet3DConditionModel
31
+
32
+
33
+ logger = logging.get_logger(__name__) # pylint: disable=invalid-name
34
+
35
+
36
+ @dataclass
37
+ class AnimationPipelineOutput(BaseOutput):
38
+ videos: Union[torch.Tensor, np.ndarray]
39
+
40
+
41
+ class AnimationPipeline(DiffusionPipeline):
42
+ _optional_components = []
43
+
44
+ def __init__(
45
+ self,
46
+ vae: AutoencoderKL,
47
+ text_encoder: CLIPTextModel,
48
+ tokenizer: CLIPTokenizer,
49
+ unet: UNet3DConditionModel,
50
+ scheduler: Union[
51
+ DDIMScheduler,
52
+ PNDMScheduler,
53
+ LMSDiscreteScheduler,
54
+ EulerDiscreteScheduler,
55
+ EulerAncestralDiscreteScheduler,
56
+ DPMSolverMultistepScheduler,
57
+ ],
58
+ ):
59
+ super().__init__()
60
+
61
+ if hasattr(scheduler.config, "steps_offset") and scheduler.config.steps_offset != 1:
62
+ deprecation_message = (
63
+ f"The configuration file of this scheduler: {scheduler} is outdated. `steps_offset`"
64
+ f" should be set to 1 instead of {scheduler.config.steps_offset}. Please make sure "
65
+ "to update the config accordingly as leaving `steps_offset` might led to incorrect results"
66
+ " in future versions. If you have downloaded this checkpoint from the Hugging Face Hub,"
67
+ " it would be very nice if you could open a Pull request for the `scheduler/scheduler_config.json`"
68
+ " file"
69
+ )
70
+ deprecate("steps_offset!=1", "1.0.0", deprecation_message, standard_warn=False)
71
+ new_config = dict(scheduler.config)
72
+ new_config["steps_offset"] = 1
73
+ scheduler._internal_dict = FrozenDict(new_config)
74
+
75
+ if hasattr(scheduler.config, "clip_sample") and scheduler.config.clip_sample is True:
76
+ deprecation_message = (
77
+ f"The configuration file of this scheduler: {scheduler} has not set the configuration `clip_sample`."
78
+ " `clip_sample` should be set to False in the configuration file. Please make sure to update the"
79
+ " config accordingly as not setting `clip_sample` in the config might lead to incorrect results in"
80
+ " future versions. If you have downloaded this checkpoint from the Hugging Face Hub, it would be very"
81
+ " nice if you could open a Pull request for the `scheduler/scheduler_config.json` file"
82
+ )
83
+ deprecate("clip_sample not set", "1.0.0", deprecation_message, standard_warn=False)
84
+ new_config = dict(scheduler.config)
85
+ new_config["clip_sample"] = False
86
+ scheduler._internal_dict = FrozenDict(new_config)
87
+
88
+ is_unet_version_less_0_9_0 = hasattr(unet.config, "_diffusers_version") and version.parse(
89
+ version.parse(unet.config._diffusers_version).base_version
90
+ ) < version.parse("0.9.0.dev0")
91
+ is_unet_sample_size_less_64 = hasattr(unet.config, "sample_size") and unet.config.sample_size < 64
92
+ if is_unet_version_less_0_9_0 and is_unet_sample_size_less_64:
93
+ deprecation_message = (
94
+ "The configuration file of the unet has set the default `sample_size` to smaller than"
95
+ " 64 which seems highly unlikely. If your checkpoint is a fine-tuned version of any of the"
96
+ " following: \n- CompVis/stable-diffusion-v1-4 \n- CompVis/stable-diffusion-v1-3 \n-"
97
+ " CompVis/stable-diffusion-v1-2 \n- CompVis/stable-diffusion-v1-1 \n- runwayml/stable-diffusion-v1-5"
98
+ " \n- runwayml/stable-diffusion-inpainting \n you should change 'sample_size' to 64 in the"
99
+ " configuration file. Please make sure to update the config accordingly as leaving `sample_size=32`"
100
+ " in the config might lead to incorrect results in future versions. If you have downloaded this"
101
+ " checkpoint from the Hugging Face Hub, it would be very nice if you could open a Pull request for"
102
+ " the `unet/config.json` file"
103
+ )
104
+ deprecate("sample_size<64", "1.0.0", deprecation_message, standard_warn=False)
105
+ new_config = dict(unet.config)
106
+ new_config["sample_size"] = 64
107
+ unet._internal_dict = FrozenDict(new_config)
108
+
109
+ self.register_modules(
110
+ vae=vae,
111
+ text_encoder=text_encoder,
112
+ tokenizer=tokenizer,
113
+ unet=unet,
114
+ scheduler=scheduler,
115
+ )
116
+ self.vae_scale_factor = 2 ** (len(self.vae.config.block_out_channels) - 1)
117
+
118
+ def enable_vae_slicing(self):
119
+ self.vae.enable_slicing()
120
+
121
+ def disable_vae_slicing(self):
122
+ self.vae.disable_slicing()
123
+
124
+ def enable_sequential_cpu_offload(self, gpu_id=0):
125
+ if is_accelerate_available():
126
+ from accelerate import cpu_offload
127
+ else:
128
+ raise ImportError("Please install accelerate via `pip install accelerate`")
129
+
130
+ device = torch.device(f"cuda:{gpu_id}")
131
+
132
+ for cpu_offloaded_model in [self.unet, self.text_encoder, self.vae]:
133
+ if cpu_offloaded_model is not None:
134
+ cpu_offload(cpu_offloaded_model, device)
135
+
136
+
137
+ @property
138
+ def _execution_device(self):
139
+ if self.device != torch.device("meta") or not hasattr(self.unet, "_hf_hook"):
140
+ return self.device
141
+ for module in self.unet.modules():
142
+ if (
143
+ hasattr(module, "_hf_hook")
144
+ and hasattr(module._hf_hook, "execution_device")
145
+ and module._hf_hook.execution_device is not None
146
+ ):
147
+ return torch.device(module._hf_hook.execution_device)
148
+ return self.device
149
+
150
+ def _encode_prompt(self, prompt, device, num_videos_per_prompt, do_classifier_free_guidance, negative_prompt):
151
+ batch_size = len(prompt) if isinstance(prompt, list) else 1
152
+
153
+ text_inputs = self.tokenizer(
154
+ prompt,
155
+ padding="max_length",
156
+ max_length=self.tokenizer.model_max_length,
157
+ truncation=True,
158
+ return_tensors="pt",
159
+ )
160
+ text_input_ids = text_inputs.input_ids
161
+ untruncated_ids = self.tokenizer(prompt, padding="longest", return_tensors="pt").input_ids
162
+
163
+ if untruncated_ids.shape[-1] >= text_input_ids.shape[-1] and not torch.equal(text_input_ids, untruncated_ids):
164
+ removed_text = self.tokenizer.batch_decode(untruncated_ids[:, self.tokenizer.model_max_length - 1 : -1])
165
+ logger.warning(
166
+ "The following part of your input was truncated because CLIP can only handle sequences up to"
167
+ f" {self.tokenizer.model_max_length} tokens: {removed_text}"
168
+ )
169
+
170
+ if hasattr(self.text_encoder.config, "use_attention_mask") and self.text_encoder.config.use_attention_mask:
171
+ attention_mask = text_inputs.attention_mask.to(device)
172
+ else:
173
+ attention_mask = None
174
+
175
+ text_embeddings = self.text_encoder(
176
+ text_input_ids.to(device),
177
+ attention_mask=attention_mask,
178
+ )
179
+ text_embeddings = text_embeddings[0]
180
+
181
+ # duplicate text embeddings for each generation per prompt, using mps friendly method
182
+ bs_embed, seq_len, _ = text_embeddings.shape
183
+ text_embeddings = text_embeddings.repeat(1, num_videos_per_prompt, 1)
184
+ text_embeddings = text_embeddings.view(bs_embed * num_videos_per_prompt, seq_len, -1)
185
+
186
+ # get unconditional embeddings for classifier free guidance
187
+ if do_classifier_free_guidance:
188
+ uncond_tokens: List[str]
189
+ if negative_prompt is None:
190
+ uncond_tokens = [""] * batch_size
191
+ elif type(prompt) is not type(negative_prompt):
192
+ raise TypeError(
193
+ f"`negative_prompt` should be the same type to `prompt`, but got {type(negative_prompt)} !="
194
+ f" {type(prompt)}."
195
+ )
196
+ elif isinstance(negative_prompt, str):
197
+ uncond_tokens = [negative_prompt]
198
+ elif batch_size != len(negative_prompt):
199
+ raise ValueError(
200
+ f"`negative_prompt`: {negative_prompt} has batch size {len(negative_prompt)}, but `prompt`:"
201
+ f" {prompt} has batch size {batch_size}. Please make sure that passed `negative_prompt` matches"
202
+ " the batch size of `prompt`."
203
+ )
204
+ else:
205
+ uncond_tokens = negative_prompt
206
+
207
+ max_length = text_input_ids.shape[-1]
208
+ uncond_input = self.tokenizer(
209
+ uncond_tokens,
210
+ padding="max_length",
211
+ max_length=max_length,
212
+ truncation=True,
213
+ return_tensors="pt",
214
+ )
215
+
216
+ if hasattr(self.text_encoder.config, "use_attention_mask") and self.text_encoder.config.use_attention_mask:
217
+ attention_mask = uncond_input.attention_mask.to(device)
218
+ else:
219
+ attention_mask = None
220
+
221
+ uncond_embeddings = self.text_encoder(
222
+ uncond_input.input_ids.to(device),
223
+ attention_mask=attention_mask,
224
+ )
225
+ uncond_embeddings = uncond_embeddings[0]
226
+
227
+ # duplicate unconditional embeddings for each generation per prompt, using mps friendly method
228
+ seq_len = uncond_embeddings.shape[1]
229
+ uncond_embeddings = uncond_embeddings.repeat(1, num_videos_per_prompt, 1)
230
+ uncond_embeddings = uncond_embeddings.view(batch_size * num_videos_per_prompt, seq_len, -1)
231
+
232
+ # For classifier free guidance, we need to do two forward passes.
233
+ # Here we concatenate the unconditional and text embeddings into a single batch
234
+ # to avoid doing two forward passes
235
+ text_embeddings = torch.cat([uncond_embeddings, text_embeddings])
236
+
237
+ return text_embeddings
238
+
239
+ def decode_latents(self, latents):
240
+ video_length = latents.shape[2]
241
+ latents = 1 / 0.18215 * latents
242
+ latents = rearrange(latents, "b c f h w -> (b f) c h w")
243
+ # video = self.vae.decode(latents).sample
244
+ video = []
245
+ for frame_idx in tqdm(range(latents.shape[0])):
246
+ video.append(self.vae.decode(latents[frame_idx:frame_idx+1]).sample)
247
+ video = torch.cat(video)
248
+ video = rearrange(video, "(b f) c h w -> b c f h w", f=video_length)
249
+ video = (video / 2 + 0.5).clamp(0, 1)
250
+ # we always cast to float32 as this does not cause significant overhead and is compatible with bfloa16
251
+ video = video.cpu().float().numpy()
252
+ return video
253
+
254
+ def prepare_extra_step_kwargs(self, generator, eta):
255
+ # prepare extra kwargs for the scheduler step, since not all schedulers have the same signature
256
+ # eta (η) is only used with the DDIMScheduler, it will be ignored for other schedulers.
257
+ # eta corresponds to η in DDIM paper: https://arxiv.org/abs/2010.02502
258
+ # and should be between [0, 1]
259
+
260
+ accepts_eta = "eta" in set(inspect.signature(self.scheduler.step).parameters.keys())
261
+ extra_step_kwargs = {}
262
+ if accepts_eta:
263
+ extra_step_kwargs["eta"] = eta
264
+
265
+ # check if the scheduler accepts generator
266
+ accepts_generator = "generator" in set(inspect.signature(self.scheduler.step).parameters.keys())
267
+ if accepts_generator:
268
+ extra_step_kwargs["generator"] = generator
269
+ return extra_step_kwargs
270
+
271
+ def check_inputs(self, prompt, height, width, callback_steps):
272
+ if not isinstance(prompt, str) and not isinstance(prompt, list):
273
+ raise ValueError(f"`prompt` has to be of type `str` or `list` but is {type(prompt)}")
274
+
275
+ if height % 8 != 0 or width % 8 != 0:
276
+ raise ValueError(f"`height` and `width` have to be divisible by 8 but are {height} and {width}.")
277
+
278
+ if (callback_steps is None) or (
279
+ callback_steps is not None and (not isinstance(callback_steps, int) or callback_steps <= 0)
280
+ ):
281
+ raise ValueError(
282
+ f"`callback_steps` has to be a positive integer but is {callback_steps} of type"
283
+ f" {type(callback_steps)}."
284
+ )
285
+
286
+ def prepare_latents(self, batch_size, num_channels_latents, video_length, height, width, dtype, device, generator, latents=None):
287
+ shape = (batch_size, num_channels_latents, video_length, height // self.vae_scale_factor, width // self.vae_scale_factor)
288
+ if isinstance(generator, list) and len(generator) != batch_size:
289
+ raise ValueError(
290
+ f"You have passed a list of generators of length {len(generator)}, but requested an effective batch"
291
+ f" size of {batch_size}. Make sure the batch size matches the length of the generators."
292
+ )
293
+ if latents is None:
294
+ rand_device = "cpu" if device.type == "mps" else device
295
+
296
+ if isinstance(generator, list):
297
+ shape = shape
298
+ # shape = (1,) + shape[1:]
299
+ latents = [
300
+ torch.randn(shape, generator=generator[i], device=rand_device, dtype=dtype)
301
+ for i in range(batch_size)
302
+ ]
303
+ latents = torch.cat(latents, dim=0).to(device)
304
+ else:
305
+ latents = torch.randn(shape, generator=generator, device=rand_device, dtype=dtype).to(device)
306
+ else:
307
+ if latents.shape != shape:
308
+ raise ValueError(f"Unexpected latents shape, got {latents.shape}, expected {shape}")
309
+ latents = latents.to(device)
310
+
311
+ # scale the initial noise by the standard deviation required by the scheduler
312
+ latents = latents * self.scheduler.init_noise_sigma
313
+ return latents
314
+
315
+ @torch.no_grad()
316
+ def __call__(
317
+ self,
318
+ prompt: Union[str, List[str]],
319
+ video_length: Optional[int],
320
+ height: Optional[int] = None,
321
+ width: Optional[int] = None,
322
+ num_inference_steps: int = 50,
323
+ guidance_scale: float = 7.5,
324
+ negative_prompt: Optional[Union[str, List[str]]] = None,
325
+ num_videos_per_prompt: Optional[int] = 1,
326
+ eta: float = 0.0,
327
+ generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None,
328
+ latents: Optional[torch.FloatTensor] = None,
329
+ output_type: Optional[str] = "tensor",
330
+ return_dict: bool = True,
331
+ callback: Optional[Callable[[int, int, torch.FloatTensor], None]] = None,
332
+ callback_steps: Optional[int] = 1,
333
+ **kwargs,
334
+ ):
335
+ # Default height and width to unet
336
+ height = height or self.unet.config.sample_size * self.vae_scale_factor
337
+ width = width or self.unet.config.sample_size * self.vae_scale_factor
338
+
339
+ # Check inputs. Raise error if not correct
340
+ self.check_inputs(prompt, height, width, callback_steps)
341
+
342
+ # Define call parameters
343
+ # batch_size = 1 if isinstance(prompt, str) else len(prompt)
344
+ batch_size = 1
345
+ if latents is not None:
346
+ batch_size = latents.shape[0]
347
+ if isinstance(prompt, list):
348
+ batch_size = len(prompt)
349
+
350
+ device = self._execution_device
351
+ # here `guidance_scale` is defined analog to the guidance weight `w` of equation (2)
352
+ # of the Imagen paper: https://arxiv.org/pdf/2205.11487.pdf . `guidance_scale = 1`
353
+ # corresponds to doing no classifier free guidance.
354
+ do_classifier_free_guidance = guidance_scale > 1.0
355
+
356
+ # Encode input prompt
357
+ prompt = prompt if isinstance(prompt, list) else [prompt] * batch_size
358
+ if negative_prompt is not None:
359
+ negative_prompt = negative_prompt if isinstance(negative_prompt, list) else [negative_prompt] * batch_size
360
+ text_embeddings = self._encode_prompt(
361
+ prompt, device, num_videos_per_prompt, do_classifier_free_guidance, negative_prompt
362
+ )
363
+
364
+ # Prepare timesteps
365
+ self.scheduler.set_timesteps(num_inference_steps, device=device)
366
+ timesteps = self.scheduler.timesteps
367
+
368
+ # Prepare latent variables
369
+ num_channels_latents = self.unet.in_channels
370
+ latents = self.prepare_latents(
371
+ batch_size * num_videos_per_prompt,
372
+ num_channels_latents,
373
+ video_length,
374
+ height,
375
+ width,
376
+ text_embeddings.dtype,
377
+ device,
378
+ generator,
379
+ latents,
380
+ )
381
+ latents_dtype = latents.dtype
382
+
383
+ # Prepare extra step kwargs.
384
+ extra_step_kwargs = self.prepare_extra_step_kwargs(generator, eta)
385
+
386
+ # Denoising loop
387
+ num_warmup_steps = len(timesteps) - num_inference_steps * self.scheduler.order
388
+ with self.progress_bar(total=num_inference_steps) as progress_bar:
389
+ for i, t in enumerate(timesteps):
390
+ # expand the latents if we are doing classifier free guidance
391
+ latent_model_input = torch.cat([latents] * 2) if do_classifier_free_guidance else latents
392
+ latent_model_input = self.scheduler.scale_model_input(latent_model_input, t)
393
+
394
+ # predict the noise residual
395
+ noise_pred = self.unet(latent_model_input, t, encoder_hidden_states=text_embeddings).sample.to(dtype=latents_dtype)
396
+ # noise_pred = []
397
+ # import pdb
398
+ # pdb.set_trace()
399
+ # for batch_idx in range(latent_model_input.shape[0]):
400
+ # noise_pred_single = self.unet(latent_model_input[batch_idx:batch_idx+1], t, encoder_hidden_states=text_embeddings[batch_idx:batch_idx+1]).sample.to(dtype=latents_dtype)
401
+ # noise_pred.append(noise_pred_single)
402
+ # noise_pred = torch.cat(noise_pred)
403
+
404
+ # perform guidance
405
+ if do_classifier_free_guidance:
406
+ noise_pred_uncond, noise_pred_text = noise_pred.chunk(2)
407
+ noise_pred = noise_pred_uncond + guidance_scale * (noise_pred_text - noise_pred_uncond)
408
+
409
+ # compute the previous noisy sample x_t -> x_t-1
410
+ latents = self.scheduler.step(noise_pred, t, latents, **extra_step_kwargs).prev_sample
411
+
412
+ # call the callback, if provided
413
+ if i == len(timesteps) - 1 or ((i + 1) > num_warmup_steps and (i + 1) % self.scheduler.order == 0):
414
+ progress_bar.update()
415
+ if callback is not None and i % callback_steps == 0:
416
+ callback(i, t, latents)
417
+
418
+ # Post-processing
419
+ video = self.decode_latents(latents)
420
+
421
+ # Convert to tensor
422
+ if output_type == "tensor":
423
+ video = torch.from_numpy(video)
424
+
425
+ if not return_dict:
426
+ return video
427
+
428
+ return AnimationPipelineOutput(videos=video)
animatediff/utils/convert_from_ckpt.py ADDED
@@ -0,0 +1,959 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # coding=utf-8
2
+ # Copyright 2023 The HuggingFace Inc. team.
3
+ #
4
+ # Licensed under the Apache License, Version 2.0 (the "License");
5
+ # you may not use this file except in compliance with the License.
6
+ # You may obtain a copy of the License at
7
+ #
8
+ # http://www.apache.org/licenses/LICENSE-2.0
9
+ #
10
+ # Unless required by applicable law or agreed to in writing, software
11
+ # distributed under the License is distributed on an "AS IS" BASIS,
12
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13
+ # See the License for the specific language governing permissions and
14
+ # limitations under the License.
15
+ """ Conversion script for the Stable Diffusion checkpoints."""
16
+
17
+ import re
18
+ from io import BytesIO
19
+ from typing import Optional
20
+
21
+ import requests
22
+ import torch
23
+ from transformers import (
24
+ AutoFeatureExtractor,
25
+ BertTokenizerFast,
26
+ CLIPImageProcessor,
27
+ CLIPTextModel,
28
+ CLIPTextModelWithProjection,
29
+ CLIPTokenizer,
30
+ CLIPVisionConfig,
31
+ CLIPVisionModelWithProjection,
32
+ )
33
+
34
+ from diffusers.models import (
35
+ AutoencoderKL,
36
+ PriorTransformer,
37
+ UNet2DConditionModel,
38
+ )
39
+ from diffusers.schedulers import (
40
+ DDIMScheduler,
41
+ DDPMScheduler,
42
+ DPMSolverMultistepScheduler,
43
+ EulerAncestralDiscreteScheduler,
44
+ EulerDiscreteScheduler,
45
+ HeunDiscreteScheduler,
46
+ LMSDiscreteScheduler,
47
+ PNDMScheduler,
48
+ UnCLIPScheduler,
49
+ )
50
+ from diffusers.utils.import_utils import BACKENDS_MAPPING
51
+
52
+
53
+ def shave_segments(path, n_shave_prefix_segments=1):
54
+ """
55
+ Removes segments. Positive values shave the first segments, negative shave the last segments.
56
+ """
57
+ if n_shave_prefix_segments >= 0:
58
+ return ".".join(path.split(".")[n_shave_prefix_segments:])
59
+ else:
60
+ return ".".join(path.split(".")[:n_shave_prefix_segments])
61
+
62
+
63
+ def renew_resnet_paths(old_list, n_shave_prefix_segments=0):
64
+ """
65
+ Updates paths inside resnets to the new naming scheme (local renaming)
66
+ """
67
+ mapping = []
68
+ for old_item in old_list:
69
+ new_item = old_item.replace("in_layers.0", "norm1")
70
+ new_item = new_item.replace("in_layers.2", "conv1")
71
+
72
+ new_item = new_item.replace("out_layers.0", "norm2")
73
+ new_item = new_item.replace("out_layers.3", "conv2")
74
+
75
+ new_item = new_item.replace("emb_layers.1", "time_emb_proj")
76
+ new_item = new_item.replace("skip_connection", "conv_shortcut")
77
+
78
+ new_item = shave_segments(new_item, n_shave_prefix_segments=n_shave_prefix_segments)
79
+
80
+ mapping.append({"old": old_item, "new": new_item})
81
+
82
+ return mapping
83
+
84
+
85
+ def renew_vae_resnet_paths(old_list, n_shave_prefix_segments=0):
86
+ """
87
+ Updates paths inside resnets to the new naming scheme (local renaming)
88
+ """
89
+ mapping = []
90
+ for old_item in old_list:
91
+ new_item = old_item
92
+
93
+ new_item = new_item.replace("nin_shortcut", "conv_shortcut")
94
+ new_item = shave_segments(new_item, n_shave_prefix_segments=n_shave_prefix_segments)
95
+
96
+ mapping.append({"old": old_item, "new": new_item})
97
+
98
+ return mapping
99
+
100
+
101
+ def renew_attention_paths(old_list, n_shave_prefix_segments=0):
102
+ """
103
+ Updates paths inside attentions to the new naming scheme (local renaming)
104
+ """
105
+ mapping = []
106
+ for old_item in old_list:
107
+ new_item = old_item
108
+
109
+ # new_item = new_item.replace('norm.weight', 'group_norm.weight')
110
+ # new_item = new_item.replace('norm.bias', 'group_norm.bias')
111
+
112
+ # new_item = new_item.replace('proj_out.weight', 'proj_attn.weight')
113
+ # new_item = new_item.replace('proj_out.bias', 'proj_attn.bias')
114
+
115
+ # new_item = shave_segments(new_item, n_shave_prefix_segments=n_shave_prefix_segments)
116
+
117
+ mapping.append({"old": old_item, "new": new_item})
118
+
119
+ return mapping
120
+
121
+
122
+ def renew_vae_attention_paths(old_list, n_shave_prefix_segments=0):
123
+ """
124
+ Updates paths inside attentions to the new naming scheme (local renaming)
125
+ """
126
+ mapping = []
127
+ for old_item in old_list:
128
+ new_item = old_item
129
+
130
+ new_item = new_item.replace("norm.weight", "group_norm.weight")
131
+ new_item = new_item.replace("norm.bias", "group_norm.bias")
132
+
133
+ new_item = new_item.replace("q.weight", "query.weight")
134
+ new_item = new_item.replace("q.bias", "query.bias")
135
+
136
+ new_item = new_item.replace("k.weight", "key.weight")
137
+ new_item = new_item.replace("k.bias", "key.bias")
138
+
139
+ new_item = new_item.replace("v.weight", "value.weight")
140
+ new_item = new_item.replace("v.bias", "value.bias")
141
+
142
+ new_item = new_item.replace("proj_out.weight", "proj_attn.weight")
143
+ new_item = new_item.replace("proj_out.bias", "proj_attn.bias")
144
+
145
+ new_item = shave_segments(new_item, n_shave_prefix_segments=n_shave_prefix_segments)
146
+
147
+ mapping.append({"old": old_item, "new": new_item})
148
+
149
+ return mapping
150
+
151
+
152
+ def assign_to_checkpoint(
153
+ paths, checkpoint, old_checkpoint, attention_paths_to_split=None, additional_replacements=None, config=None
154
+ ):
155
+ """
156
+ This does the final conversion step: take locally converted weights and apply a global renaming to them. It splits
157
+ attention layers, and takes into account additional replacements that may arise.
158
+
159
+ Assigns the weights to the new checkpoint.
160
+ """
161
+ assert isinstance(paths, list), "Paths should be a list of dicts containing 'old' and 'new' keys."
162
+
163
+ # Splits the attention layers into three variables.
164
+ if attention_paths_to_split is not None:
165
+ for path, path_map in attention_paths_to_split.items():
166
+ old_tensor = old_checkpoint[path]
167
+ channels = old_tensor.shape[0] // 3
168
+
169
+ target_shape = (-1, channels) if len(old_tensor.shape) == 3 else (-1)
170
+
171
+ num_heads = old_tensor.shape[0] // config["num_head_channels"] // 3
172
+
173
+ old_tensor = old_tensor.reshape((num_heads, 3 * channels // num_heads) + old_tensor.shape[1:])
174
+ query, key, value = old_tensor.split(channels // num_heads, dim=1)
175
+
176
+ checkpoint[path_map["query"]] = query.reshape(target_shape)
177
+ checkpoint[path_map["key"]] = key.reshape(target_shape)
178
+ checkpoint[path_map["value"]] = value.reshape(target_shape)
179
+
180
+ for path in paths:
181
+ new_path = path["new"]
182
+
183
+ # These have already been assigned
184
+ if attention_paths_to_split is not None and new_path in attention_paths_to_split:
185
+ continue
186
+
187
+ # Global renaming happens here
188
+ new_path = new_path.replace("middle_block.0", "mid_block.resnets.0")
189
+ new_path = new_path.replace("middle_block.1", "mid_block.attentions.0")
190
+ new_path = new_path.replace("middle_block.2", "mid_block.resnets.1")
191
+
192
+ if additional_replacements is not None:
193
+ for replacement in additional_replacements:
194
+ new_path = new_path.replace(replacement["old"], replacement["new"])
195
+
196
+ # proj_attn.weight has to be converted from conv 1D to linear
197
+ if "proj_attn.weight" in new_path:
198
+ checkpoint[new_path] = old_checkpoint[path["old"]][:, :, 0]
199
+ else:
200
+ checkpoint[new_path] = old_checkpoint[path["old"]]
201
+
202
+
203
+ def conv_attn_to_linear(checkpoint):
204
+ keys = list(checkpoint.keys())
205
+ attn_keys = ["query.weight", "key.weight", "value.weight"]
206
+ for key in keys:
207
+ if ".".join(key.split(".")[-2:]) in attn_keys:
208
+ if checkpoint[key].ndim > 2:
209
+ checkpoint[key] = checkpoint[key][:, :, 0, 0]
210
+ elif "proj_attn.weight" in key:
211
+ if checkpoint[key].ndim > 2:
212
+ checkpoint[key] = checkpoint[key][:, :, 0]
213
+
214
+
215
+ def create_unet_diffusers_config(original_config, image_size: int, controlnet=False):
216
+ """
217
+ Creates a config for the diffusers based on the config of the LDM model.
218
+ """
219
+ if controlnet:
220
+ unet_params = original_config.model.params.control_stage_config.params
221
+ else:
222
+ unet_params = original_config.model.params.unet_config.params
223
+
224
+ vae_params = original_config.model.params.first_stage_config.params.ddconfig
225
+
226
+ block_out_channels = [unet_params.model_channels * mult for mult in unet_params.channel_mult]
227
+
228
+ down_block_types = []
229
+ resolution = 1
230
+ for i in range(len(block_out_channels)):
231
+ block_type = "CrossAttnDownBlock2D" if resolution in unet_params.attention_resolutions else "DownBlock2D"
232
+ down_block_types.append(block_type)
233
+ if i != len(block_out_channels) - 1:
234
+ resolution *= 2
235
+
236
+ up_block_types = []
237
+ for i in range(len(block_out_channels)):
238
+ block_type = "CrossAttnUpBlock2D" if resolution in unet_params.attention_resolutions else "UpBlock2D"
239
+ up_block_types.append(block_type)
240
+ resolution //= 2
241
+
242
+ vae_scale_factor = 2 ** (len(vae_params.ch_mult) - 1)
243
+
244
+ head_dim = unet_params.num_heads if "num_heads" in unet_params else None
245
+ use_linear_projection = (
246
+ unet_params.use_linear_in_transformer if "use_linear_in_transformer" in unet_params else False
247
+ )
248
+ if use_linear_projection:
249
+ # stable diffusion 2-base-512 and 2-768
250
+ if head_dim is None:
251
+ head_dim = [5, 10, 20, 20]
252
+
253
+ class_embed_type = None
254
+ projection_class_embeddings_input_dim = None
255
+
256
+ if "num_classes" in unet_params:
257
+ if unet_params.num_classes == "sequential":
258
+ class_embed_type = "projection"
259
+ assert "adm_in_channels" in unet_params
260
+ projection_class_embeddings_input_dim = unet_params.adm_in_channels
261
+ else:
262
+ raise NotImplementedError(f"Unknown conditional unet num_classes config: {unet_params.num_classes}")
263
+
264
+ config = {
265
+ "sample_size": image_size // vae_scale_factor,
266
+ "in_channels": unet_params.in_channels,
267
+ "down_block_types": tuple(down_block_types),
268
+ "block_out_channels": tuple(block_out_channels),
269
+ "layers_per_block": unet_params.num_res_blocks,
270
+ "cross_attention_dim": unet_params.context_dim,
271
+ "attention_head_dim": head_dim,
272
+ "use_linear_projection": use_linear_projection,
273
+ "class_embed_type": class_embed_type,
274
+ "projection_class_embeddings_input_dim": projection_class_embeddings_input_dim,
275
+ }
276
+
277
+ if not controlnet:
278
+ config["out_channels"] = unet_params.out_channels
279
+ config["up_block_types"] = tuple(up_block_types)
280
+
281
+ return config
282
+
283
+
284
+ def create_vae_diffusers_config(original_config, image_size: int):
285
+ """
286
+ Creates a config for the diffusers based on the config of the LDM model.
287
+ """
288
+ vae_params = original_config.model.params.first_stage_config.params.ddconfig
289
+ _ = original_config.model.params.first_stage_config.params.embed_dim
290
+
291
+ block_out_channels = [vae_params.ch * mult for mult in vae_params.ch_mult]
292
+ down_block_types = ["DownEncoderBlock2D"] * len(block_out_channels)
293
+ up_block_types = ["UpDecoderBlock2D"] * len(block_out_channels)
294
+
295
+ config = {
296
+ "sample_size": image_size,
297
+ "in_channels": vae_params.in_channels,
298
+ "out_channels": vae_params.out_ch,
299
+ "down_block_types": tuple(down_block_types),
300
+ "up_block_types": tuple(up_block_types),
301
+ "block_out_channels": tuple(block_out_channels),
302
+ "latent_channels": vae_params.z_channels,
303
+ "layers_per_block": vae_params.num_res_blocks,
304
+ }
305
+ return config
306
+
307
+
308
+ def create_diffusers_schedular(original_config):
309
+ schedular = DDIMScheduler(
310
+ num_train_timesteps=original_config.model.params.timesteps,
311
+ beta_start=original_config.model.params.linear_start,
312
+ beta_end=original_config.model.params.linear_end,
313
+ beta_schedule="scaled_linear",
314
+ )
315
+ return schedular
316
+
317
+
318
+ def create_ldm_bert_config(original_config):
319
+ bert_params = original_config.model.parms.cond_stage_config.params
320
+ config = LDMBertConfig(
321
+ d_model=bert_params.n_embed,
322
+ encoder_layers=bert_params.n_layer,
323
+ encoder_ffn_dim=bert_params.n_embed * 4,
324
+ )
325
+ return config
326
+
327
+
328
+ def convert_ldm_unet_checkpoint(checkpoint, config, path=None, extract_ema=False, controlnet=False):
329
+ """
330
+ Takes a state dict and a config, and returns a converted checkpoint.
331
+ """
332
+
333
+ # extract state_dict for UNet
334
+ unet_state_dict = {}
335
+ keys = list(checkpoint.keys())
336
+
337
+ if controlnet:
338
+ unet_key = "control_model."
339
+ else:
340
+ unet_key = "model.diffusion_model."
341
+
342
+ # at least a 100 parameters have to start with `model_ema` in order for the checkpoint to be EMA
343
+ if sum(k.startswith("model_ema") for k in keys) > 100 and extract_ema:
344
+ print(f"Checkpoint {path} has both EMA and non-EMA weights.")
345
+ print(
346
+ "In this conversion only the EMA weights are extracted. If you want to instead extract the non-EMA"
347
+ " weights (useful to continue fine-tuning), please make sure to remove the `--extract_ema` flag."
348
+ )
349
+ for key in keys:
350
+ if key.startswith("model.diffusion_model"):
351
+ flat_ema_key = "model_ema." + "".join(key.split(".")[1:])
352
+ unet_state_dict[key.replace(unet_key, "")] = checkpoint.pop(flat_ema_key)
353
+ else:
354
+ if sum(k.startswith("model_ema") for k in keys) > 100:
355
+ print(
356
+ "In this conversion only the non-EMA weights are extracted. If you want to instead extract the EMA"
357
+ " weights (usually better for inference), please make sure to add the `--extract_ema` flag."
358
+ )
359
+
360
+ for key in keys:
361
+ if key.startswith(unet_key):
362
+ unet_state_dict[key.replace(unet_key, "")] = checkpoint.pop(key)
363
+
364
+ new_checkpoint = {}
365
+
366
+ new_checkpoint["time_embedding.linear_1.weight"] = unet_state_dict["time_embed.0.weight"]
367
+ new_checkpoint["time_embedding.linear_1.bias"] = unet_state_dict["time_embed.0.bias"]
368
+ new_checkpoint["time_embedding.linear_2.weight"] = unet_state_dict["time_embed.2.weight"]
369
+ new_checkpoint["time_embedding.linear_2.bias"] = unet_state_dict["time_embed.2.bias"]
370
+
371
+ if config["class_embed_type"] is None:
372
+ # No parameters to port
373
+ ...
374
+ elif config["class_embed_type"] == "timestep" or config["class_embed_type"] == "projection":
375
+ new_checkpoint["class_embedding.linear_1.weight"] = unet_state_dict["label_emb.0.0.weight"]
376
+ new_checkpoint["class_embedding.linear_1.bias"] = unet_state_dict["label_emb.0.0.bias"]
377
+ new_checkpoint["class_embedding.linear_2.weight"] = unet_state_dict["label_emb.0.2.weight"]
378
+ new_checkpoint["class_embedding.linear_2.bias"] = unet_state_dict["label_emb.0.2.bias"]
379
+ else:
380
+ raise NotImplementedError(f"Not implemented `class_embed_type`: {config['class_embed_type']}")
381
+
382
+ new_checkpoint["conv_in.weight"] = unet_state_dict["input_blocks.0.0.weight"]
383
+ new_checkpoint["conv_in.bias"] = unet_state_dict["input_blocks.0.0.bias"]
384
+
385
+ if not controlnet:
386
+ new_checkpoint["conv_norm_out.weight"] = unet_state_dict["out.0.weight"]
387
+ new_checkpoint["conv_norm_out.bias"] = unet_state_dict["out.0.bias"]
388
+ new_checkpoint["conv_out.weight"] = unet_state_dict["out.2.weight"]
389
+ new_checkpoint["conv_out.bias"] = unet_state_dict["out.2.bias"]
390
+
391
+ # Retrieves the keys for the input blocks only
392
+ num_input_blocks = len({".".join(layer.split(".")[:2]) for layer in unet_state_dict if "input_blocks" in layer})
393
+ input_blocks = {
394
+ layer_id: [key for key in unet_state_dict if f"input_blocks.{layer_id}" in key]
395
+ for layer_id in range(num_input_blocks)
396
+ }
397
+
398
+ # Retrieves the keys for the middle blocks only
399
+ num_middle_blocks = len({".".join(layer.split(".")[:2]) for layer in unet_state_dict if "middle_block" in layer})
400
+ middle_blocks = {
401
+ layer_id: [key for key in unet_state_dict if f"middle_block.{layer_id}" in key]
402
+ for layer_id in range(num_middle_blocks)
403
+ }
404
+
405
+ # Retrieves the keys for the output blocks only
406
+ num_output_blocks = len({".".join(layer.split(".")[:2]) for layer in unet_state_dict if "output_blocks" in layer})
407
+ output_blocks = {
408
+ layer_id: [key for key in unet_state_dict if f"output_blocks.{layer_id}" in key]
409
+ for layer_id in range(num_output_blocks)
410
+ }
411
+
412
+ for i in range(1, num_input_blocks):
413
+ block_id = (i - 1) // (config["layers_per_block"] + 1)
414
+ layer_in_block_id = (i - 1) % (config["layers_per_block"] + 1)
415
+
416
+ resnets = [
417
+ key for key in input_blocks[i] if f"input_blocks.{i}.0" in key and f"input_blocks.{i}.0.op" not in key
418
+ ]
419
+ attentions = [key for key in input_blocks[i] if f"input_blocks.{i}.1" in key]
420
+
421
+ if f"input_blocks.{i}.0.op.weight" in unet_state_dict:
422
+ new_checkpoint[f"down_blocks.{block_id}.downsamplers.0.conv.weight"] = unet_state_dict.pop(
423
+ f"input_blocks.{i}.0.op.weight"
424
+ )
425
+ new_checkpoint[f"down_blocks.{block_id}.downsamplers.0.conv.bias"] = unet_state_dict.pop(
426
+ f"input_blocks.{i}.0.op.bias"
427
+ )
428
+
429
+ paths = renew_resnet_paths(resnets)
430
+ meta_path = {"old": f"input_blocks.{i}.0", "new": f"down_blocks.{block_id}.resnets.{layer_in_block_id}"}
431
+ assign_to_checkpoint(
432
+ paths, new_checkpoint, unet_state_dict, additional_replacements=[meta_path], config=config
433
+ )
434
+
435
+ if len(attentions):
436
+ paths = renew_attention_paths(attentions)
437
+ meta_path = {"old": f"input_blocks.{i}.1", "new": f"down_blocks.{block_id}.attentions.{layer_in_block_id}"}
438
+ assign_to_checkpoint(
439
+ paths, new_checkpoint, unet_state_dict, additional_replacements=[meta_path], config=config
440
+ )
441
+
442
+ resnet_0 = middle_blocks[0]
443
+ attentions = middle_blocks[1]
444
+ resnet_1 = middle_blocks[2]
445
+
446
+ resnet_0_paths = renew_resnet_paths(resnet_0)
447
+ assign_to_checkpoint(resnet_0_paths, new_checkpoint, unet_state_dict, config=config)
448
+
449
+ resnet_1_paths = renew_resnet_paths(resnet_1)
450
+ assign_to_checkpoint(resnet_1_paths, new_checkpoint, unet_state_dict, config=config)
451
+
452
+ attentions_paths = renew_attention_paths(attentions)
453
+ meta_path = {"old": "middle_block.1", "new": "mid_block.attentions.0"}
454
+ assign_to_checkpoint(
455
+ attentions_paths, new_checkpoint, unet_state_dict, additional_replacements=[meta_path], config=config
456
+ )
457
+
458
+ for i in range(num_output_blocks):
459
+ block_id = i // (config["layers_per_block"] + 1)
460
+ layer_in_block_id = i % (config["layers_per_block"] + 1)
461
+ output_block_layers = [shave_segments(name, 2) for name in output_blocks[i]]
462
+ output_block_list = {}
463
+
464
+ for layer in output_block_layers:
465
+ layer_id, layer_name = layer.split(".")[0], shave_segments(layer, 1)
466
+ if layer_id in output_block_list:
467
+ output_block_list[layer_id].append(layer_name)
468
+ else:
469
+ output_block_list[layer_id] = [layer_name]
470
+
471
+ if len(output_block_list) > 1:
472
+ resnets = [key for key in output_blocks[i] if f"output_blocks.{i}.0" in key]
473
+ attentions = [key for key in output_blocks[i] if f"output_blocks.{i}.1" in key]
474
+
475
+ resnet_0_paths = renew_resnet_paths(resnets)
476
+ paths = renew_resnet_paths(resnets)
477
+
478
+ meta_path = {"old": f"output_blocks.{i}.0", "new": f"up_blocks.{block_id}.resnets.{layer_in_block_id}"}
479
+ assign_to_checkpoint(
480
+ paths, new_checkpoint, unet_state_dict, additional_replacements=[meta_path], config=config
481
+ )
482
+
483
+ output_block_list = {k: sorted(v) for k, v in output_block_list.items()}
484
+ if ["conv.bias", "conv.weight"] in output_block_list.values():
485
+ index = list(output_block_list.values()).index(["conv.bias", "conv.weight"])
486
+ new_checkpoint[f"up_blocks.{block_id}.upsamplers.0.conv.weight"] = unet_state_dict[
487
+ f"output_blocks.{i}.{index}.conv.weight"
488
+ ]
489
+ new_checkpoint[f"up_blocks.{block_id}.upsamplers.0.conv.bias"] = unet_state_dict[
490
+ f"output_blocks.{i}.{index}.conv.bias"
491
+ ]
492
+
493
+ # Clear attentions as they have been attributed above.
494
+ if len(attentions) == 2:
495
+ attentions = []
496
+
497
+ if len(attentions):
498
+ paths = renew_attention_paths(attentions)
499
+ meta_path = {
500
+ "old": f"output_blocks.{i}.1",
501
+ "new": f"up_blocks.{block_id}.attentions.{layer_in_block_id}",
502
+ }
503
+ assign_to_checkpoint(
504
+ paths, new_checkpoint, unet_state_dict, additional_replacements=[meta_path], config=config
505
+ )
506
+ else:
507
+ resnet_0_paths = renew_resnet_paths(output_block_layers, n_shave_prefix_segments=1)
508
+ for path in resnet_0_paths:
509
+ old_path = ".".join(["output_blocks", str(i), path["old"]])
510
+ new_path = ".".join(["up_blocks", str(block_id), "resnets", str(layer_in_block_id), path["new"]])
511
+
512
+ new_checkpoint[new_path] = unet_state_dict[old_path]
513
+
514
+ if controlnet:
515
+ # conditioning embedding
516
+
517
+ orig_index = 0
518
+
519
+ new_checkpoint["controlnet_cond_embedding.conv_in.weight"] = unet_state_dict.pop(
520
+ f"input_hint_block.{orig_index}.weight"
521
+ )
522
+ new_checkpoint["controlnet_cond_embedding.conv_in.bias"] = unet_state_dict.pop(
523
+ f"input_hint_block.{orig_index}.bias"
524
+ )
525
+
526
+ orig_index += 2
527
+
528
+ diffusers_index = 0
529
+
530
+ while diffusers_index < 6:
531
+ new_checkpoint[f"controlnet_cond_embedding.blocks.{diffusers_index}.weight"] = unet_state_dict.pop(
532
+ f"input_hint_block.{orig_index}.weight"
533
+ )
534
+ new_checkpoint[f"controlnet_cond_embedding.blocks.{diffusers_index}.bias"] = unet_state_dict.pop(
535
+ f"input_hint_block.{orig_index}.bias"
536
+ )
537
+ diffusers_index += 1
538
+ orig_index += 2
539
+
540
+ new_checkpoint["controlnet_cond_embedding.conv_out.weight"] = unet_state_dict.pop(
541
+ f"input_hint_block.{orig_index}.weight"
542
+ )
543
+ new_checkpoint["controlnet_cond_embedding.conv_out.bias"] = unet_state_dict.pop(
544
+ f"input_hint_block.{orig_index}.bias"
545
+ )
546
+
547
+ # down blocks
548
+ for i in range(num_input_blocks):
549
+ new_checkpoint[f"controlnet_down_blocks.{i}.weight"] = unet_state_dict.pop(f"zero_convs.{i}.0.weight")
550
+ new_checkpoint[f"controlnet_down_blocks.{i}.bias"] = unet_state_dict.pop(f"zero_convs.{i}.0.bias")
551
+
552
+ # mid block
553
+ new_checkpoint["controlnet_mid_block.weight"] = unet_state_dict.pop("middle_block_out.0.weight")
554
+ new_checkpoint["controlnet_mid_block.bias"] = unet_state_dict.pop("middle_block_out.0.bias")
555
+
556
+ return new_checkpoint
557
+
558
+
559
+ def convert_ldm_vae_checkpoint(checkpoint, config):
560
+ # extract state dict for VAE
561
+ vae_state_dict = {}
562
+ vae_key = "first_stage_model."
563
+ keys = list(checkpoint.keys())
564
+ for key in keys:
565
+ if key.startswith(vae_key):
566
+ vae_state_dict[key.replace(vae_key, "")] = checkpoint.get(key)
567
+
568
+ new_checkpoint = {}
569
+
570
+ new_checkpoint["encoder.conv_in.weight"] = vae_state_dict["encoder.conv_in.weight"]
571
+ new_checkpoint["encoder.conv_in.bias"] = vae_state_dict["encoder.conv_in.bias"]
572
+ new_checkpoint["encoder.conv_out.weight"] = vae_state_dict["encoder.conv_out.weight"]
573
+ new_checkpoint["encoder.conv_out.bias"] = vae_state_dict["encoder.conv_out.bias"]
574
+ new_checkpoint["encoder.conv_norm_out.weight"] = vae_state_dict["encoder.norm_out.weight"]
575
+ new_checkpoint["encoder.conv_norm_out.bias"] = vae_state_dict["encoder.norm_out.bias"]
576
+
577
+ new_checkpoint["decoder.conv_in.weight"] = vae_state_dict["decoder.conv_in.weight"]
578
+ new_checkpoint["decoder.conv_in.bias"] = vae_state_dict["decoder.conv_in.bias"]
579
+ new_checkpoint["decoder.conv_out.weight"] = vae_state_dict["decoder.conv_out.weight"]
580
+ new_checkpoint["decoder.conv_out.bias"] = vae_state_dict["decoder.conv_out.bias"]
581
+ new_checkpoint["decoder.conv_norm_out.weight"] = vae_state_dict["decoder.norm_out.weight"]
582
+ new_checkpoint["decoder.conv_norm_out.bias"] = vae_state_dict["decoder.norm_out.bias"]
583
+
584
+ new_checkpoint["quant_conv.weight"] = vae_state_dict["quant_conv.weight"]
585
+ new_checkpoint["quant_conv.bias"] = vae_state_dict["quant_conv.bias"]
586
+ new_checkpoint["post_quant_conv.weight"] = vae_state_dict["post_quant_conv.weight"]
587
+ new_checkpoint["post_quant_conv.bias"] = vae_state_dict["post_quant_conv.bias"]
588
+
589
+ # Retrieves the keys for the encoder down blocks only
590
+ num_down_blocks = len({".".join(layer.split(".")[:3]) for layer in vae_state_dict if "encoder.down" in layer})
591
+ down_blocks = {
592
+ layer_id: [key for key in vae_state_dict if f"down.{layer_id}" in key] for layer_id in range(num_down_blocks)
593
+ }
594
+
595
+ # Retrieves the keys for the decoder up blocks only
596
+ num_up_blocks = len({".".join(layer.split(".")[:3]) for layer in vae_state_dict if "decoder.up" in layer})
597
+ up_blocks = {
598
+ layer_id: [key for key in vae_state_dict if f"up.{layer_id}" in key] for layer_id in range(num_up_blocks)
599
+ }
600
+
601
+ for i in range(num_down_blocks):
602
+ resnets = [key for key in down_blocks[i] if f"down.{i}" in key and f"down.{i}.downsample" not in key]
603
+
604
+ if f"encoder.down.{i}.downsample.conv.weight" in vae_state_dict:
605
+ new_checkpoint[f"encoder.down_blocks.{i}.downsamplers.0.conv.weight"] = vae_state_dict.pop(
606
+ f"encoder.down.{i}.downsample.conv.weight"
607
+ )
608
+ new_checkpoint[f"encoder.down_blocks.{i}.downsamplers.0.conv.bias"] = vae_state_dict.pop(
609
+ f"encoder.down.{i}.downsample.conv.bias"
610
+ )
611
+
612
+ paths = renew_vae_resnet_paths(resnets)
613
+ meta_path = {"old": f"down.{i}.block", "new": f"down_blocks.{i}.resnets"}
614
+ assign_to_checkpoint(paths, new_checkpoint, vae_state_dict, additional_replacements=[meta_path], config=config)
615
+
616
+ mid_resnets = [key for key in vae_state_dict if "encoder.mid.block" in key]
617
+ num_mid_res_blocks = 2
618
+ for i in range(1, num_mid_res_blocks + 1):
619
+ resnets = [key for key in mid_resnets if f"encoder.mid.block_{i}" in key]
620
+
621
+ paths = renew_vae_resnet_paths(resnets)
622
+ meta_path = {"old": f"mid.block_{i}", "new": f"mid_block.resnets.{i - 1}"}
623
+ assign_to_checkpoint(paths, new_checkpoint, vae_state_dict, additional_replacements=[meta_path], config=config)
624
+
625
+ mid_attentions = [key for key in vae_state_dict if "encoder.mid.attn" in key]
626
+ paths = renew_vae_attention_paths(mid_attentions)
627
+ meta_path = {"old": "mid.attn_1", "new": "mid_block.attentions.0"}
628
+ assign_to_checkpoint(paths, new_checkpoint, vae_state_dict, additional_replacements=[meta_path], config=config)
629
+ conv_attn_to_linear(new_checkpoint)
630
+
631
+ for i in range(num_up_blocks):
632
+ block_id = num_up_blocks - 1 - i
633
+ resnets = [
634
+ key for key in up_blocks[block_id] if f"up.{block_id}" in key and f"up.{block_id}.upsample" not in key
635
+ ]
636
+
637
+ if f"decoder.up.{block_id}.upsample.conv.weight" in vae_state_dict:
638
+ new_checkpoint[f"decoder.up_blocks.{i}.upsamplers.0.conv.weight"] = vae_state_dict[
639
+ f"decoder.up.{block_id}.upsample.conv.weight"
640
+ ]
641
+ new_checkpoint[f"decoder.up_blocks.{i}.upsamplers.0.conv.bias"] = vae_state_dict[
642
+ f"decoder.up.{block_id}.upsample.conv.bias"
643
+ ]
644
+
645
+ paths = renew_vae_resnet_paths(resnets)
646
+ meta_path = {"old": f"up.{block_id}.block", "new": f"up_blocks.{i}.resnets"}
647
+ assign_to_checkpoint(paths, new_checkpoint, vae_state_dict, additional_replacements=[meta_path], config=config)
648
+
649
+ mid_resnets = [key for key in vae_state_dict if "decoder.mid.block" in key]
650
+ num_mid_res_blocks = 2
651
+ for i in range(1, num_mid_res_blocks + 1):
652
+ resnets = [key for key in mid_resnets if f"decoder.mid.block_{i}" in key]
653
+
654
+ paths = renew_vae_resnet_paths(resnets)
655
+ meta_path = {"old": f"mid.block_{i}", "new": f"mid_block.resnets.{i - 1}"}
656
+ assign_to_checkpoint(paths, new_checkpoint, vae_state_dict, additional_replacements=[meta_path], config=config)
657
+
658
+ mid_attentions = [key for key in vae_state_dict if "decoder.mid.attn" in key]
659
+ paths = renew_vae_attention_paths(mid_attentions)
660
+ meta_path = {"old": "mid.attn_1", "new": "mid_block.attentions.0"}
661
+ assign_to_checkpoint(paths, new_checkpoint, vae_state_dict, additional_replacements=[meta_path], config=config)
662
+ conv_attn_to_linear(new_checkpoint)
663
+ return new_checkpoint
664
+
665
+
666
+ def convert_ldm_bert_checkpoint(checkpoint, config):
667
+ def _copy_attn_layer(hf_attn_layer, pt_attn_layer):
668
+ hf_attn_layer.q_proj.weight.data = pt_attn_layer.to_q.weight
669
+ hf_attn_layer.k_proj.weight.data = pt_attn_layer.to_k.weight
670
+ hf_attn_layer.v_proj.weight.data = pt_attn_layer.to_v.weight
671
+
672
+ hf_attn_layer.out_proj.weight = pt_attn_layer.to_out.weight
673
+ hf_attn_layer.out_proj.bias = pt_attn_layer.to_out.bias
674
+
675
+ def _copy_linear(hf_linear, pt_linear):
676
+ hf_linear.weight = pt_linear.weight
677
+ hf_linear.bias = pt_linear.bias
678
+
679
+ def _copy_layer(hf_layer, pt_layer):
680
+ # copy layer norms
681
+ _copy_linear(hf_layer.self_attn_layer_norm, pt_layer[0][0])
682
+ _copy_linear(hf_layer.final_layer_norm, pt_layer[1][0])
683
+
684
+ # copy attn
685
+ _copy_attn_layer(hf_layer.self_attn, pt_layer[0][1])
686
+
687
+ # copy MLP
688
+ pt_mlp = pt_layer[1][1]
689
+ _copy_linear(hf_layer.fc1, pt_mlp.net[0][0])
690
+ _copy_linear(hf_layer.fc2, pt_mlp.net[2])
691
+
692
+ def _copy_layers(hf_layers, pt_layers):
693
+ for i, hf_layer in enumerate(hf_layers):
694
+ if i != 0:
695
+ i += i
696
+ pt_layer = pt_layers[i : i + 2]
697
+ _copy_layer(hf_layer, pt_layer)
698
+
699
+ hf_model = LDMBertModel(config).eval()
700
+
701
+ # copy embeds
702
+ hf_model.model.embed_tokens.weight = checkpoint.transformer.token_emb.weight
703
+ hf_model.model.embed_positions.weight.data = checkpoint.transformer.pos_emb.emb.weight
704
+
705
+ # copy layer norm
706
+ _copy_linear(hf_model.model.layer_norm, checkpoint.transformer.norm)
707
+
708
+ # copy hidden layers
709
+ _copy_layers(hf_model.model.layers, checkpoint.transformer.attn_layers.layers)
710
+
711
+ _copy_linear(hf_model.to_logits, checkpoint.transformer.to_logits)
712
+
713
+ return hf_model
714
+
715
+
716
+ def convert_ldm_clip_checkpoint(checkpoint):
717
+ text_model = CLIPTextModel.from_pretrained("openai/clip-vit-large-patch14")
718
+ keys = list(checkpoint.keys())
719
+
720
+ text_model_dict = {}
721
+
722
+ for key in keys:
723
+ if key.startswith("cond_stage_model.transformer"):
724
+ text_model_dict[key[len("cond_stage_model.transformer.") :]] = checkpoint[key]
725
+
726
+ text_model.load_state_dict(text_model_dict)
727
+
728
+ return text_model
729
+
730
+
731
+ textenc_conversion_lst = [
732
+ ("cond_stage_model.model.positional_embedding", "text_model.embeddings.position_embedding.weight"),
733
+ ("cond_stage_model.model.token_embedding.weight", "text_model.embeddings.token_embedding.weight"),
734
+ ("cond_stage_model.model.ln_final.weight", "text_model.final_layer_norm.weight"),
735
+ ("cond_stage_model.model.ln_final.bias", "text_model.final_layer_norm.bias"),
736
+ ]
737
+ textenc_conversion_map = {x[0]: x[1] for x in textenc_conversion_lst}
738
+
739
+ textenc_transformer_conversion_lst = [
740
+ # (stable-diffusion, HF Diffusers)
741
+ ("resblocks.", "text_model.encoder.layers."),
742
+ ("ln_1", "layer_norm1"),
743
+ ("ln_2", "layer_norm2"),
744
+ (".c_fc.", ".fc1."),
745
+ (".c_proj.", ".fc2."),
746
+ (".attn", ".self_attn"),
747
+ ("ln_final.", "transformer.text_model.final_layer_norm."),
748
+ ("token_embedding.weight", "transformer.text_model.embeddings.token_embedding.weight"),
749
+ ("positional_embedding", "transformer.text_model.embeddings.position_embedding.weight"),
750
+ ]
751
+ protected = {re.escape(x[0]): x[1] for x in textenc_transformer_conversion_lst}
752
+ textenc_pattern = re.compile("|".join(protected.keys()))
753
+
754
+
755
+ def convert_paint_by_example_checkpoint(checkpoint):
756
+ config = CLIPVisionConfig.from_pretrained("openai/clip-vit-large-patch14")
757
+ model = PaintByExampleImageEncoder(config)
758
+
759
+ keys = list(checkpoint.keys())
760
+
761
+ text_model_dict = {}
762
+
763
+ for key in keys:
764
+ if key.startswith("cond_stage_model.transformer"):
765
+ text_model_dict[key[len("cond_stage_model.transformer.") :]] = checkpoint[key]
766
+
767
+ # load clip vision
768
+ model.model.load_state_dict(text_model_dict)
769
+
770
+ # load mapper
771
+ keys_mapper = {
772
+ k[len("cond_stage_model.mapper.res") :]: v
773
+ for k, v in checkpoint.items()
774
+ if k.startswith("cond_stage_model.mapper")
775
+ }
776
+
777
+ MAPPING = {
778
+ "attn.c_qkv": ["attn1.to_q", "attn1.to_k", "attn1.to_v"],
779
+ "attn.c_proj": ["attn1.to_out.0"],
780
+ "ln_1": ["norm1"],
781
+ "ln_2": ["norm3"],
782
+ "mlp.c_fc": ["ff.net.0.proj"],
783
+ "mlp.c_proj": ["ff.net.2"],
784
+ }
785
+
786
+ mapped_weights = {}
787
+ for key, value in keys_mapper.items():
788
+ prefix = key[: len("blocks.i")]
789
+ suffix = key.split(prefix)[-1].split(".")[-1]
790
+ name = key.split(prefix)[-1].split(suffix)[0][1:-1]
791
+ mapped_names = MAPPING[name]
792
+
793
+ num_splits = len(mapped_names)
794
+ for i, mapped_name in enumerate(mapped_names):
795
+ new_name = ".".join([prefix, mapped_name, suffix])
796
+ shape = value.shape[0] // num_splits
797
+ mapped_weights[new_name] = value[i * shape : (i + 1) * shape]
798
+
799
+ model.mapper.load_state_dict(mapped_weights)
800
+
801
+ # load final layer norm
802
+ model.final_layer_norm.load_state_dict(
803
+ {
804
+ "bias": checkpoint["cond_stage_model.final_ln.bias"],
805
+ "weight": checkpoint["cond_stage_model.final_ln.weight"],
806
+ }
807
+ )
808
+
809
+ # load final proj
810
+ model.proj_out.load_state_dict(
811
+ {
812
+ "bias": checkpoint["proj_out.bias"],
813
+ "weight": checkpoint["proj_out.weight"],
814
+ }
815
+ )
816
+
817
+ # load uncond vector
818
+ model.uncond_vector.data = torch.nn.Parameter(checkpoint["learnable_vector"])
819
+ return model
820
+
821
+
822
+ def convert_open_clip_checkpoint(checkpoint):
823
+ text_model = CLIPTextModel.from_pretrained("stabilityai/stable-diffusion-2", subfolder="text_encoder")
824
+
825
+ keys = list(checkpoint.keys())
826
+
827
+ text_model_dict = {}
828
+
829
+ if "cond_stage_model.model.text_projection" in checkpoint:
830
+ d_model = int(checkpoint["cond_stage_model.model.text_projection"].shape[0])
831
+ else:
832
+ d_model = 1024
833
+
834
+ text_model_dict["text_model.embeddings.position_ids"] = text_model.text_model.embeddings.get_buffer("position_ids")
835
+
836
+ for key in keys:
837
+ if "resblocks.23" in key: # Diffusers drops the final layer and only uses the penultimate layer
838
+ continue
839
+ if key in textenc_conversion_map:
840
+ text_model_dict[textenc_conversion_map[key]] = checkpoint[key]
841
+ if key.startswith("cond_stage_model.model.transformer."):
842
+ new_key = key[len("cond_stage_model.model.transformer.") :]
843
+ if new_key.endswith(".in_proj_weight"):
844
+ new_key = new_key[: -len(".in_proj_weight")]
845
+ new_key = textenc_pattern.sub(lambda m: protected[re.escape(m.group(0))], new_key)
846
+ text_model_dict[new_key + ".q_proj.weight"] = checkpoint[key][:d_model, :]
847
+ text_model_dict[new_key + ".k_proj.weight"] = checkpoint[key][d_model : d_model * 2, :]
848
+ text_model_dict[new_key + ".v_proj.weight"] = checkpoint[key][d_model * 2 :, :]
849
+ elif new_key.endswith(".in_proj_bias"):
850
+ new_key = new_key[: -len(".in_proj_bias")]
851
+ new_key = textenc_pattern.sub(lambda m: protected[re.escape(m.group(0))], new_key)
852
+ text_model_dict[new_key + ".q_proj.bias"] = checkpoint[key][:d_model]
853
+ text_model_dict[new_key + ".k_proj.bias"] = checkpoint[key][d_model : d_model * 2]
854
+ text_model_dict[new_key + ".v_proj.bias"] = checkpoint[key][d_model * 2 :]
855
+ else:
856
+ new_key = textenc_pattern.sub(lambda m: protected[re.escape(m.group(0))], new_key)
857
+
858
+ text_model_dict[new_key] = checkpoint[key]
859
+
860
+ text_model.load_state_dict(text_model_dict)
861
+
862
+ return text_model
863
+
864
+
865
+ def stable_unclip_image_encoder(original_config):
866
+ """
867
+ Returns the image processor and clip image encoder for the img2img unclip pipeline.
868
+
869
+ We currently know of two types of stable unclip models which separately use the clip and the openclip image
870
+ encoders.
871
+ """
872
+
873
+ image_embedder_config = original_config.model.params.embedder_config
874
+
875
+ sd_clip_image_embedder_class = image_embedder_config.target
876
+ sd_clip_image_embedder_class = sd_clip_image_embedder_class.split(".")[-1]
877
+
878
+ if sd_clip_image_embedder_class == "ClipImageEmbedder":
879
+ clip_model_name = image_embedder_config.params.model
880
+
881
+ if clip_model_name == "ViT-L/14":
882
+ feature_extractor = CLIPImageProcessor()
883
+ image_encoder = CLIPVisionModelWithProjection.from_pretrained("openai/clip-vit-large-patch14")
884
+ else:
885
+ raise NotImplementedError(f"Unknown CLIP checkpoint name in stable diffusion checkpoint {clip_model_name}")
886
+
887
+ elif sd_clip_image_embedder_class == "FrozenOpenCLIPImageEmbedder":
888
+ feature_extractor = CLIPImageProcessor()
889
+ image_encoder = CLIPVisionModelWithProjection.from_pretrained("laion/CLIP-ViT-H-14-laion2B-s32B-b79K")
890
+ else:
891
+ raise NotImplementedError(
892
+ f"Unknown CLIP image embedder class in stable diffusion checkpoint {sd_clip_image_embedder_class}"
893
+ )
894
+
895
+ return feature_extractor, image_encoder
896
+
897
+
898
+ def stable_unclip_image_noising_components(
899
+ original_config, clip_stats_path: Optional[str] = None, device: Optional[str] = None
900
+ ):
901
+ """
902
+ Returns the noising components for the img2img and txt2img unclip pipelines.
903
+
904
+ Converts the stability noise augmentor into
905
+ 1. a `StableUnCLIPImageNormalizer` for holding the CLIP stats
906
+ 2. a `DDPMScheduler` for holding the noise schedule
907
+
908
+ If the noise augmentor config specifies a clip stats path, the `clip_stats_path` must be provided.
909
+ """
910
+ noise_aug_config = original_config.model.params.noise_aug_config
911
+ noise_aug_class = noise_aug_config.target
912
+ noise_aug_class = noise_aug_class.split(".")[-1]
913
+
914
+ if noise_aug_class == "CLIPEmbeddingNoiseAugmentation":
915
+ noise_aug_config = noise_aug_config.params
916
+ embedding_dim = noise_aug_config.timestep_dim
917
+ max_noise_level = noise_aug_config.noise_schedule_config.timesteps
918
+ beta_schedule = noise_aug_config.noise_schedule_config.beta_schedule
919
+
920
+ image_normalizer = StableUnCLIPImageNormalizer(embedding_dim=embedding_dim)
921
+ image_noising_scheduler = DDPMScheduler(num_train_timesteps=max_noise_level, beta_schedule=beta_schedule)
922
+
923
+ if "clip_stats_path" in noise_aug_config:
924
+ if clip_stats_path is None:
925
+ raise ValueError("This stable unclip config requires a `clip_stats_path`")
926
+
927
+ clip_mean, clip_std = torch.load(clip_stats_path, map_location=device)
928
+ clip_mean = clip_mean[None, :]
929
+ clip_std = clip_std[None, :]
930
+
931
+ clip_stats_state_dict = {
932
+ "mean": clip_mean,
933
+ "std": clip_std,
934
+ }
935
+
936
+ image_normalizer.load_state_dict(clip_stats_state_dict)
937
+ else:
938
+ raise NotImplementedError(f"Unknown noise augmentor class: {noise_aug_class}")
939
+
940
+ return image_normalizer, image_noising_scheduler
941
+
942
+
943
+ def convert_controlnet_checkpoint(
944
+ checkpoint, original_config, checkpoint_path, image_size, upcast_attention, extract_ema
945
+ ):
946
+ ctrlnet_config = create_unet_diffusers_config(original_config, image_size=image_size, controlnet=True)
947
+ ctrlnet_config["upcast_attention"] = upcast_attention
948
+
949
+ ctrlnet_config.pop("sample_size")
950
+
951
+ controlnet_model = ControlNetModel(**ctrlnet_config)
952
+
953
+ converted_ctrl_checkpoint = convert_ldm_unet_checkpoint(
954
+ checkpoint, ctrlnet_config, path=checkpoint_path, extract_ema=extract_ema, controlnet=True
955
+ )
956
+
957
+ controlnet_model.load_state_dict(converted_ctrl_checkpoint)
958
+
959
+ return controlnet_model
animatediff/utils/convert_lora_safetensor_to_diffusers.py ADDED
@@ -0,0 +1,128 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # coding=utf-8
2
+ # Copyright 2023, Haofan Wang, Qixun Wang, All rights reserved.
3
+ #
4
+ # Licensed under the Apache License, Version 2.0 (the "License");
5
+ # you may not use this file except in compliance with the License.
6
+ # You may obtain a copy of the License at
7
+ #
8
+ # http://www.apache.org/licenses/LICENSE-2.0
9
+ #
10
+ # Unless required by applicable law or agreed to in writing, software
11
+ # distributed under the License is distributed on an "AS IS" BASIS,
12
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13
+ # See the License for the specific language governing permissions and
14
+ # limitations under the License.
15
+
16
+ """ Conversion script for the LoRA's safetensors checkpoints. """
17
+
18
+ import argparse
19
+
20
+ import torch
21
+ from safetensors.torch import load_file
22
+
23
+ from diffusers import StableDiffusionPipeline
24
+ import pdb
25
+
26
+ def convert_lora(pipeline, state_dict, LORA_PREFIX_UNET="lora_unet", LORA_PREFIX_TEXT_ENCODER="lora_te", alpha=0.6):
27
+ # load base model
28
+ # pipeline = StableDiffusionPipeline.from_pretrained(base_model_path, torch_dtype=torch.float32)
29
+
30
+ # load LoRA weight from .safetensors
31
+ # state_dict = load_file(checkpoint_path)
32
+
33
+ visited = []
34
+
35
+ # directly update weight in diffusers model
36
+ for key in state_dict:
37
+ # it is suggested to print out the key, it usually will be something like below
38
+ # "lora_te_text_model_encoder_layers_0_self_attn_k_proj.lora_down.weight"
39
+
40
+ # as we have set the alpha beforehand, so just skip
41
+ if ".alpha" in key or key in visited:
42
+ continue
43
+
44
+ if "text" in key:
45
+ layer_infos = key.split(".")[0].split(LORA_PREFIX_TEXT_ENCODER + "_")[-1].split("_")
46
+ curr_layer = pipeline.text_encoder
47
+ else:
48
+ layer_infos = key.split(".")[0].split(LORA_PREFIX_UNET + "_")[-1].split("_")
49
+ curr_layer = pipeline.unet
50
+
51
+ # find the target layer
52
+ temp_name = layer_infos.pop(0)
53
+ while len(layer_infos) > -1:
54
+ try:
55
+ curr_layer = curr_layer.__getattr__(temp_name)
56
+ if len(layer_infos) > 0:
57
+ temp_name = layer_infos.pop(0)
58
+ elif len(layer_infos) == 0:
59
+ break
60
+ except Exception:
61
+ if len(temp_name) > 0:
62
+ temp_name += "_" + layer_infos.pop(0)
63
+ else:
64
+ temp_name = layer_infos.pop(0)
65
+
66
+ pair_keys = []
67
+ if "lora_down" in key:
68
+ pair_keys.append(key.replace("lora_down", "lora_up"))
69
+ pair_keys.append(key)
70
+ else:
71
+ pair_keys.append(key)
72
+ pair_keys.append(key.replace("lora_up", "lora_down"))
73
+
74
+ # update weight
75
+ if len(state_dict[pair_keys[0]].shape) == 4:
76
+ weight_up = state_dict[pair_keys[0]].squeeze(3).squeeze(2).to(torch.float32)
77
+ weight_down = state_dict[pair_keys[1]].squeeze(3).squeeze(2).to(torch.float32)
78
+ curr_layer.weight.data += alpha * torch.mm(weight_up, weight_down).unsqueeze(2).unsqueeze(3).to(curr_layer.weight.data.device)
79
+ else:
80
+ weight_up = state_dict[pair_keys[0]].to(torch.float32)
81
+ weight_down = state_dict[pair_keys[1]].to(torch.float32)
82
+ curr_layer.weight.data += alpha * torch.mm(weight_up, weight_down).to(curr_layer.weight.data.device)
83
+
84
+ # update visited list
85
+ for item in pair_keys:
86
+ visited.append(item)
87
+
88
+ return pipeline
89
+
90
+
91
+ if __name__ == "__main__":
92
+ parser = argparse.ArgumentParser()
93
+
94
+ parser.add_argument(
95
+ "--base_model_path", default=None, type=str, required=True, help="Path to the base model in diffusers format."
96
+ )
97
+ parser.add_argument(
98
+ "--checkpoint_path", default=None, type=str, required=True, help="Path to the checkpoint to convert."
99
+ )
100
+ parser.add_argument("--dump_path", default=None, type=str, required=True, help="Path to the output model.")
101
+ parser.add_argument(
102
+ "--lora_prefix_unet", default="lora_unet", type=str, help="The prefix of UNet weight in safetensors"
103
+ )
104
+ parser.add_argument(
105
+ "--lora_prefix_text_encoder",
106
+ default="lora_te",
107
+ type=str,
108
+ help="The prefix of text encoder weight in safetensors",
109
+ )
110
+ parser.add_argument("--alpha", default=0.75, type=float, help="The merging ratio in W = W0 + alpha * deltaW")
111
+ parser.add_argument(
112
+ "--to_safetensors", action="store_true", help="Whether to store pipeline in safetensors format or not."
113
+ )
114
+ parser.add_argument("--device", type=str, help="Device to use (e.g. cpu, cuda:0, cuda:1, etc.)")
115
+
116
+ args = parser.parse_args()
117
+
118
+ base_model_path = args.base_model_path
119
+ checkpoint_path = args.checkpoint_path
120
+ dump_path = args.dump_path
121
+ lora_prefix_unet = args.lora_prefix_unet
122
+ lora_prefix_text_encoder = args.lora_prefix_text_encoder
123
+ alpha = args.alpha
124
+
125
+ pipe = convert(base_model_path, checkpoint_path, lora_prefix_unet, lora_prefix_text_encoder, alpha)
126
+
127
+ pipe = pipe.to(args.device)
128
+ pipe.save_pretrained(args.dump_path, safe_serialization=args.to_safetensors)
animatediff/utils/util.py ADDED
@@ -0,0 +1,84 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import os
2
+ import imageio
3
+ import numpy as np
4
+ from typing import Union
5
+
6
+ import torch
7
+ import torchvision
8
+
9
+ from tqdm import tqdm
10
+ from einops import rearrange
11
+
12
+
13
+ def save_videos_grid(videos: torch.Tensor, path: str, rescale=False, n_rows=6, fps=8):
14
+ videos = rearrange(videos, "b c t h w -> t b c h w")
15
+ outputs = []
16
+ for x in videos:
17
+ x = torchvision.utils.make_grid(x, nrow=n_rows)
18
+ x = x.transpose(0, 1).transpose(1, 2).squeeze(-1)
19
+ if rescale:
20
+ x = (x + 1.0) / 2.0 # -1,1 -> 0,1
21
+ x = (x * 255).numpy().astype(np.uint8)
22
+ outputs.append(x)
23
+
24
+ os.makedirs(os.path.dirname(path), exist_ok=True)
25
+ imageio.mimsave(path, outputs, fps=fps)
26
+
27
+
28
+ # DDIM Inversion
29
+ @torch.no_grad()
30
+ def init_prompt(prompt, pipeline):
31
+ uncond_input = pipeline.tokenizer(
32
+ [""], padding="max_length", max_length=pipeline.tokenizer.model_max_length,
33
+ return_tensors="pt"
34
+ )
35
+ uncond_embeddings = pipeline.text_encoder(uncond_input.input_ids.to(pipeline.device))[0]
36
+ text_input = pipeline.tokenizer(
37
+ [prompt],
38
+ padding="max_length",
39
+ max_length=pipeline.tokenizer.model_max_length,
40
+ truncation=True,
41
+ return_tensors="pt",
42
+ )
43
+ text_embeddings = pipeline.text_encoder(text_input.input_ids.to(pipeline.device))[0]
44
+ context = torch.cat([uncond_embeddings, text_embeddings])
45
+
46
+ return context
47
+
48
+
49
+ def next_step(model_output: Union[torch.FloatTensor, np.ndarray], timestep: int,
50
+ sample: Union[torch.FloatTensor, np.ndarray], ddim_scheduler):
51
+ timestep, next_timestep = min(
52
+ timestep - ddim_scheduler.config.num_train_timesteps // ddim_scheduler.num_inference_steps, 999), timestep
53
+ alpha_prod_t = ddim_scheduler.alphas_cumprod[timestep] if timestep >= 0 else ddim_scheduler.final_alpha_cumprod
54
+ alpha_prod_t_next = ddim_scheduler.alphas_cumprod[next_timestep]
55
+ beta_prod_t = 1 - alpha_prod_t
56
+ next_original_sample = (sample - beta_prod_t ** 0.5 * model_output) / alpha_prod_t ** 0.5
57
+ next_sample_direction = (1 - alpha_prod_t_next) ** 0.5 * model_output
58
+ next_sample = alpha_prod_t_next ** 0.5 * next_original_sample + next_sample_direction
59
+ return next_sample
60
+
61
+
62
+ def get_noise_pred_single(latents, t, context, unet):
63
+ noise_pred = unet(latents, t, encoder_hidden_states=context)["sample"]
64
+ return noise_pred
65
+
66
+
67
+ @torch.no_grad()
68
+ def ddim_loop(pipeline, ddim_scheduler, latent, num_inv_steps, prompt):
69
+ context = init_prompt(prompt, pipeline)
70
+ uncond_embeddings, cond_embeddings = context.chunk(2)
71
+ all_latent = [latent]
72
+ latent = latent.clone().detach()
73
+ for i in tqdm(range(num_inv_steps)):
74
+ t = ddim_scheduler.timesteps[len(ddim_scheduler.timesteps) - i - 1]
75
+ noise_pred = get_noise_pred_single(latent, t, cond_embeddings, pipeline.unet)
76
+ latent = next_step(noise_pred, t, latent, ddim_scheduler)
77
+ all_latent.append(latent)
78
+ return all_latent
79
+
80
+
81
+ @torch.no_grad()
82
+ def ddim_inversion(pipeline, ddim_scheduler, video_latent, num_inv_steps, prompt=""):
83
+ ddim_latents = ddim_loop(pipeline, ddim_scheduler, video_latent, num_inv_steps, prompt)
84
+ return ddim_latents
app.py ADDED
@@ -0,0 +1,294 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import os
2
+ import torch
3
+ import random
4
+
5
+ import gradio as gr
6
+ from glob import glob
7
+ from omegaconf import OmegaConf
8
+ from safetensors import safe_open
9
+
10
+ from diffusers import AutoencoderKL
11
+ from diffusers import EulerDiscreteScheduler, DDIMScheduler
12
+ from diffusers.utils.import_utils import is_xformers_available
13
+ from transformers import CLIPTextModel, CLIPTokenizer
14
+
15
+ from animatediff.models.unet import UNet3DConditionModel
16
+ from animatediff.pipelines.pipeline_animation import AnimationPipeline
17
+ from animatediff.utils.util import save_videos_grid
18
+ from animatediff.utils.convert_from_ckpt import convert_ldm_unet_checkpoint, convert_ldm_clip_checkpoint, convert_ldm_vae_checkpoint
19
+
20
+
21
+ pretrained_model_path = "models/StableDiffusion/stable-diffusion-v1-5"
22
+ inference_config_path = "configs/inference/inference.yaml"
23
+
24
+ css = """
25
+ .toolbutton {
26
+ margin-buttom: 0em 0em 0em 0em;
27
+ max-width: 2.5em;
28
+ min-width: 2.5em !important;
29
+ height: 2.5em;
30
+ }
31
+ """
32
+
33
+ examples = [
34
+ # 1-ToonYou
35
+ [
36
+ "toonyou_beta3.safetensors",
37
+ "mm_sd_v14.ckpt",
38
+ "masterpiece, best quality, 1girl, solo, cherry blossoms, hanami, pink flower, white flower, spring season, wisteria, petals, flower, plum blossoms, outdoors, falling petals, white hair, black eyes",
39
+ "worst quality, low quality, nsfw, logo",
40
+ 512, 512, "13204175718326964000"
41
+ ],
42
+ # 2-Lyriel
43
+ [
44
+ "lyriel_v16.safetensors",
45
+ "mm_sd_v15.ckpt",
46
+ "A forbidden castle high up in the mountains, pixel art, intricate details2, hdr, intricate details, hyperdetailed5, natural skin texture, hyperrealism, soft light, sharp, game art, key visual, surreal",
47
+ "3d, cartoon, anime, sketches, worst quality, low quality, normal quality, lowres, normal quality, monochrome, grayscale, skin spots, acnes, skin blemishes, bad anatomy, girl, loli, young, large breasts, red eyes, muscular",
48
+ 512, 512, "6681501646976930000"
49
+ ],
50
+ # 3-RCNZ
51
+ [
52
+ "rcnzCartoon3d_v10.safetensors",
53
+ "mm_sd_v14.ckpt",
54
+ "Jane Eyre with headphones, natural skin texture,4mm,k textures, soft cinematic light, adobe lightroom, photolab, hdr, intricate, elegant, highly detailed, sharp focus, cinematic look, soothing tones, insane details, intricate details, hyperdetailed, low contrast, soft cinematic light, dim colors, exposure blend, hdr, faded",
55
+ "deformed, distorted, disfigured, poorly drawn, bad anatomy, wrong anatomy, extra limb, missing limb, floating limbs, mutated hands and fingers, disconnected limbs, mutation, mutated, ugly, disgusting, blurry, amputation",
56
+ 512, 512, "2416282124261060"
57
+ ],
58
+ # 4-MajicMix
59
+ [
60
+ "majicmixRealistic_v5Preview.safetensors",
61
+ "mm_sd_v14.ckpt",
62
+ "1girl, offshoulder, light smile, shiny skin best quality, masterpiece, photorealistic",
63
+ "bad hand, worst quality, low quality, normal quality, lowres, bad anatomy, bad hands, watermark, moles",
64
+ 512, 512, "7132772652786303"
65
+ ],
66
+ # 5-RealisticVision
67
+ [
68
+ "realisticVisionV20_v20.safetensors",
69
+ "mm_sd_v15.ckpt",
70
+ "photo of coastline, rocks, storm weather, wind, waves, lightning, 8k uhd, dslr, soft lighting, high quality, film grain, Fujifilm XT3",
71
+ "blur, haze, deformed iris, deformed pupils, semi-realistic, cgi, 3d, render, sketch, cartoon, drawing, anime, mutated hands and fingers, deformed, distorted, disfigured, poorly drawn, bad anatomy, wrong anatomy, extra limb, missing limb, floating limbs, disconnected limbs, mutation, mutated, ugly, disgusting, amputation",
72
+ 512, 512, "1490157606650685400"
73
+ ]
74
+ ]
75
+
76
+ # clean unrelated ckpts
77
+ # ckpts = [
78
+ # "realisticVisionV40_v20Novae.safetensors",
79
+ # "majicmixRealistic_v5Preview.safetensors",
80
+ # "rcnzCartoon3d_v10.safetensors",
81
+ # "lyriel_v16.safetensors",
82
+ # "toonyou_beta3.safetensors"
83
+ # ]
84
+
85
+ # for path in glob(os.path.join("models", "DreamBooth_LoRA", "*.safetensors")):
86
+ # for ckpt in ckpts:
87
+ # if path.endswith(ckpt): break
88
+ # else:
89
+ # print(f"### Cleaning {path} ...")
90
+ # os.system(f"rm -rf {path}")
91
+
92
+ # os.system(f"rm -rf {os.path.join('models', 'DreamBooth_LoRA', '*.safetensors')}")
93
+
94
+ # os.system(f"bash download_bashscripts/1-ToonYou.sh")
95
+ # os.system(f"bash download_bashscripts/2-Lyriel.sh")
96
+ # os.system(f"bash download_bashscripts/3-RcnzCartoon.sh")
97
+ # os.system(f"bash download_bashscripts/4-MajicMix.sh")
98
+ # os.system(f"bash download_bashscripts/5-RealisticVision.sh")
99
+
100
+ # clean Grdio cache
101
+ print(f"### Cleaning cached examples ...")
102
+ os.system(f"rm -rf gradio_cached_examples/")
103
+
104
+
105
+ class AnimateController:
106
+ def __init__(self):
107
+
108
+ # config dirs
109
+ self.basedir = os.getcwd()
110
+ self.stable_diffusion_dir = os.path.join(self.basedir, "models", "StableDiffusion")
111
+ self.motion_module_dir = os.path.join(self.basedir, "models", "Motion_Module")
112
+ self.personalized_model_dir = os.path.join(self.basedir, "models", "DreamBooth_LoRA")
113
+ self.savedir = os.path.join(self.basedir, "samples")
114
+ os.makedirs(self.savedir, exist_ok=True)
115
+
116
+ self.base_model_list = []
117
+ self.motion_module_list = []
118
+
119
+ self.selected_base_model = None
120
+ self.selected_motion_module = None
121
+
122
+ self.refresh_motion_module()
123
+ self.refresh_personalized_model()
124
+
125
+ # config models
126
+ self.inference_config = OmegaConf.load(inference_config_path)
127
+
128
+ self.tokenizer = CLIPTokenizer.from_pretrained(pretrained_model_path, subfolder="tokenizer")
129
+ self.text_encoder = CLIPTextModel.from_pretrained(pretrained_model_path, subfolder="text_encoder").cuda()
130
+ self.vae = AutoencoderKL.from_pretrained(pretrained_model_path, subfolder="vae").cuda()
131
+ self.unet = UNet3DConditionModel.from_pretrained_2d(pretrained_model_path, subfolder="unet", unet_additional_kwargs=OmegaConf.to_container(self.inference_config.unet_additional_kwargs)).cuda()
132
+
133
+ self.update_base_model(self.base_model_list[0])
134
+ self.update_motion_module(self.motion_module_list[0])
135
+
136
+
137
+ def refresh_motion_module(self):
138
+ motion_module_list = glob(os.path.join(self.motion_module_dir, "*.ckpt"))
139
+ self.motion_module_list = [os.path.basename(p) for p in motion_module_list]
140
+
141
+ def refresh_personalized_model(self):
142
+ base_model_list = glob(os.path.join(self.personalized_model_dir, "*.safetensors"))
143
+ self.base_model_list = [os.path.basename(p) for p in base_model_list]
144
+
145
+
146
+ def update_base_model(self, base_model_dropdown):
147
+ self.selected_base_model = base_model_dropdown
148
+
149
+ base_model_dropdown = os.path.join(self.personalized_model_dir, base_model_dropdown)
150
+ base_model_state_dict = {}
151
+ with safe_open(base_model_dropdown, framework="pt", device="cpu") as f:
152
+ for key in f.keys(): base_model_state_dict[key] = f.get_tensor(key)
153
+
154
+ converted_vae_checkpoint = convert_ldm_vae_checkpoint(base_model_state_dict, self.vae.config)
155
+ self.vae.load_state_dict(converted_vae_checkpoint)
156
+
157
+ converted_unet_checkpoint = convert_ldm_unet_checkpoint(base_model_state_dict, self.unet.config)
158
+ self.unet.load_state_dict(converted_unet_checkpoint, strict=False)
159
+
160
+ self.text_encoder = convert_ldm_clip_checkpoint(base_model_state_dict)
161
+ return gr.Dropdown.update()
162
+
163
+ def update_motion_module(self, motion_module_dropdown):
164
+ self.selected_motion_module = motion_module_dropdown
165
+
166
+ motion_module_dropdown = os.path.join(self.motion_module_dir, motion_module_dropdown)
167
+ motion_module_state_dict = torch.load(motion_module_dropdown, map_location="cpu")
168
+ _, unexpected = self.unet.load_state_dict(motion_module_state_dict, strict=False)
169
+ assert len(unexpected) == 0
170
+ return gr.Dropdown.update()
171
+
172
+
173
+ def animate(
174
+ self,
175
+ base_model_dropdown,
176
+ motion_module_dropdown,
177
+ prompt_textbox,
178
+ negative_prompt_textbox,
179
+ width_slider,
180
+ height_slider,
181
+ seed_textbox,
182
+ ):
183
+ if self.selected_base_model != base_model_dropdown: self.update_base_model(base_model_dropdown)
184
+ if self.selected_motion_module != motion_module_dropdown: self.update_motion_module(motion_module_dropdown)
185
+
186
+ if is_xformers_available(): self.unet.enable_xformers_memory_efficient_attention()
187
+
188
+ pipeline = AnimationPipeline(
189
+ vae=self.vae, text_encoder=self.text_encoder, tokenizer=self.tokenizer, unet=self.unet,
190
+ scheduler=DDIMScheduler(**OmegaConf.to_container(self.inference_config.noise_scheduler_kwargs))
191
+ ).to("cuda")
192
+
193
+ if int(seed_textbox) > 0: seed = int(seed_textbox)
194
+ else: seed = random.randint(1, 1e16)
195
+ torch.manual_seed(int(seed))
196
+
197
+ assert seed == torch.initial_seed()
198
+ print(f"### seed: {seed}")
199
+
200
+ generator = torch.Generator(device="cuda")
201
+ generator.manual_seed(seed)
202
+
203
+ sample = pipeline(
204
+ prompt_textbox,
205
+ negative_prompt = negative_prompt_textbox,
206
+ num_inference_steps = 25,
207
+ guidance_scale = 8.,
208
+ width = width_slider,
209
+ height = height_slider,
210
+ video_length = 16,
211
+ generator = generator,
212
+ ).videos
213
+
214
+ save_sample_path = os.path.join(self.savedir, f"sample.mp4")
215
+ save_videos_grid(sample, save_sample_path)
216
+
217
+ json_config = {
218
+ "prompt": prompt_textbox,
219
+ "n_prompt": negative_prompt_textbox,
220
+ "width": width_slider,
221
+ "height": height_slider,
222
+ "seed": seed,
223
+ "base_model": base_model_dropdown,
224
+ "motion_module": motion_module_dropdown,
225
+ }
226
+ return gr.Video.update(value=save_sample_path), gr.Json.update(value=json_config)
227
+
228
+
229
+ controller = AnimateController()
230
+
231
+
232
+ def ui():
233
+ with gr.Blocks(css=css) as demo:
234
+ gr.Markdown(
235
+ """
236
+ # AnimateDiff: Animate Your Personalized Text-to-Image Diffusion Models without Specific Tuning
237
+ Yuwei Guo, Ceyuan Yang*, Anyi Rao, Yaohui Wang, Yu Qiao, Dahua Lin, Bo Dai (*Corresponding Author)<br>
238
+ [Arxiv Report](https://arxiv.org/abs/2307.04725) | [Project Page](https://animatediff.github.io/) | [Github](https://github.com/guoyww/animatediff/)
239
+ """
240
+ )
241
+ gr.Markdown(
242
+ """
243
+ ### Quick Start
244
+ 1. Select desired `Base DreamBooth Model`.
245
+ 2. Select `Motion Module` from `mm_sd_v14.ckpt` and `mm_sd_v15.ckpt`. We recommend trying both of them for the best results.
246
+ 3. Provide `Prompt` and `Negative Prompt` for each model. You are encouraged to refer to each model's webpage on CivitAI to learn how to write prompts for them. Below are the DreamBooth models in this demo. Click to visit their homepage.
247
+ - [`toonyou_beta3.safetensors`](https://civitai.com/models/30240?modelVersionId=78775)
248
+ - [`lyriel_v16.safetensors`](https://civitai.com/models/22922/lyriel)
249
+ - [`rcnzCartoon3d_v10.safetensors`](https://civitai.com/models/66347?modelVersionId=71009)
250
+ - [`majicmixRealistic_v5Preview.safetensors`](https://civitai.com/models/43331?modelVersionId=79068)
251
+ - [`realisticVisionV20_v20.safetensors`](https://civitai.com/models/4201?modelVersionId=29460)
252
+ 4. Click `Generate`, wait for ~1 min, and enjoy.
253
+ """
254
+ )
255
+ with gr.Row():
256
+ with gr.Column():
257
+ base_model_dropdown = gr.Dropdown( label="Base DreamBooth Model", choices=controller.base_model_list, value=controller.base_model_list[0], interactive=True )
258
+ motion_module_dropdown = gr.Dropdown( label="Motion Module", choices=controller.motion_module_list, value=controller.motion_module_list[0], interactive=True )
259
+
260
+ base_model_dropdown.change(fn=controller.update_base_model, inputs=[base_model_dropdown], outputs=[base_model_dropdown])
261
+ motion_module_dropdown.change(fn=controller.update_motion_module, inputs=[motion_module_dropdown], outputs=[motion_module_dropdown])
262
+
263
+ prompt_textbox = gr.Textbox( label="Prompt", lines=3 )
264
+ negative_prompt_textbox = gr.Textbox( label="Negative Prompt", lines=3, value="worst quality, low quality, nsfw, logo")
265
+
266
+ with gr.Accordion("Advance", open=False):
267
+ with gr.Row():
268
+ width_slider = gr.Slider( label="Width", value=512, minimum=256, maximum=1024, step=64 )
269
+ height_slider = gr.Slider( label="Height", value=512, minimum=256, maximum=1024, step=64 )
270
+ with gr.Row():
271
+ seed_textbox = gr.Textbox( label="Seed", value=-1)
272
+ seed_button = gr.Button(value="\U0001F3B2", elem_classes="toolbutton")
273
+ seed_button.click(fn=lambda: gr.Textbox.update(value=random.randint(1, 1e16)), inputs=[], outputs=[seed_textbox])
274
+
275
+ generate_button = gr.Button( value="Generate", variant='primary' )
276
+
277
+ with gr.Column():
278
+ result_video = gr.Video( label="Generated Animation", interactive=False )
279
+ json_config = gr.Json( label="Config", value=None )
280
+
281
+ inputs = [base_model_dropdown, motion_module_dropdown, prompt_textbox, negative_prompt_textbox, width_slider, height_slider, seed_textbox]
282
+ outputs = [result_video, json_config]
283
+
284
+ generate_button.click( fn=controller.animate, inputs=inputs, outputs=outputs )
285
+
286
+ gr.Examples( fn=controller.animate, examples=examples, inputs=inputs, outputs=outputs, cache_examples=True )
287
+
288
+ return demo
289
+
290
+
291
+ if __name__ == "__main__":
292
+ demo = ui()
293
+ demo.queue(max_size=20)
294
+ demo.launch()
configs/inference/inference.yaml ADDED
@@ -0,0 +1,26 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ unet_additional_kwargs:
2
+ unet_use_cross_frame_attention: false
3
+ unet_use_temporal_attention: false
4
+ use_motion_module: true
5
+ motion_module_resolutions:
6
+ - 1
7
+ - 2
8
+ - 4
9
+ - 8
10
+ motion_module_mid_block: false
11
+ motion_module_decoder_only: false
12
+ motion_module_type: Vanilla
13
+ motion_module_kwargs:
14
+ num_attention_heads: 8
15
+ num_transformer_block: 1
16
+ attention_block_types:
17
+ - Temporal_Self
18
+ - Temporal_Self
19
+ temporal_position_encoding: true
20
+ temporal_position_encoding_max_len: 24
21
+ temporal_attention_dim_div: 1
22
+
23
+ noise_scheduler_kwargs:
24
+ beta_start: 0.00085
25
+ beta_end: 0.012
26
+ beta_schedule: "linear"
configs/prompts/1-ToonYou.yaml ADDED
@@ -0,0 +1,22 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ ToonYou:
2
+ base: ""
3
+ path: "models/DreamBooth_LoRA/toonyou_beta3.safetensors"
4
+ motion_module:
5
+ - "models/Motion_Module/mm_sd_v14.ckpt"
6
+ - "models/Motion_Module/mm_sd_v15.ckpt"
7
+
8
+ seed: [10788741199826055526, 6520604954829636163, 6519455744612555650, 16372571278361863751]
9
+ steps: 25
10
+ guidance_scale: 7.5
11
+
12
+ prompt:
13
+ - "best quality, masterpiece, 1girl, looking at viewer, blurry background, upper body, contemporary, dress"
14
+ - "masterpiece, best quality, 1girl, solo, cherry blossoms, hanami, pink flower, white flower, spring season, wisteria, petals, flower, plum blossoms, outdoors, falling petals, white hair, black eyes,"
15
+ - "best quality, masterpiece, 1boy, formal, abstract, looking at viewer, masculine, marble pattern"
16
+ - "best quality, masterpiece, 1girl, cloudy sky, dandelion, contrapposto, alternate hairstyle,"
17
+
18
+ n_prompt:
19
+ - ""
20
+ - "badhandv4,easynegative,ng_deepnegative_v1_75t,verybadimagenegative_v1.3, bad-artist, bad_prompt_version2-neg, teeth"
21
+ - ""
22
+ - ""
configs/prompts/2-Lyriel.yaml ADDED
@@ -0,0 +1,22 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ Lyriel:
2
+ base: ""
3
+ path: "models/DreamBooth_LoRA/lyriel_v16.safetensors"
4
+ motion_module:
5
+ - "models/Motion_Module/mm_sd_v14.ckpt"
6
+ - "models/Motion_Module/mm_sd_v15.ckpt"
7
+
8
+ seed: [10917152860782582783, 6399018107401806238, 15875751942533906793, 6653196880059936551]
9
+ steps: 25
10
+ guidance_scale: 7.5
11
+
12
+ prompt:
13
+ - "dark shot, epic realistic, portrait of halo, sunglasses, blue eyes, tartan scarf, white hair by atey ghailan, by greg rutkowski, by greg tocchini, by james gilleard, by joe fenton, by kaethe butcher, gradient yellow, black, brown and magenta color scheme, grunge aesthetic!!! graffiti tag wall background, art by greg rutkowski and artgerm, soft cinematic light, adobe lightroom, photolab, hdr, intricate, highly detailed, depth of field, faded, neutral colors, hdr, muted colors, hyperdetailed, artstation, cinematic, warm lights, dramatic light, intricate details, complex background, rutkowski, teal and orange"
14
+ - "A forbidden castle high up in the mountains, pixel art, intricate details2, hdr, intricate details, hyperdetailed5, natural skin texture, hyperrealism, soft light, sharp, game art, key visual, surreal"
15
+ - "dark theme, medieval portrait of a man sharp features, grim, cold stare, dark colors, Volumetric lighting, baroque oil painting by Greg Rutkowski, Artgerm, WLOP, Alphonse Mucha dynamic lighting hyperdetailed intricately detailed, hdr, muted colors, complex background, hyperrealism, hyperdetailed, amandine van ray"
16
+ - "As I have gone alone in there and with my treasures bold, I can keep my secret where and hint of riches new and old. Begin it where warm waters halt and take it in a canyon down, not far but too far to walk, put in below the home of brown."
17
+
18
+ n_prompt:
19
+ - "3d, cartoon, lowres, bad anatomy, bad hands, text, error, missing fingers, extra digit, fewer digits, cropped, worst quality, low quality, normal quality, jpeg artifacts, signature, watermark, username, blurry, artist name, young, loli, elf, 3d, illustration"
20
+ - "3d, cartoon, anime, sketches, worst quality, low quality, normal quality, lowres, normal quality, monochrome, grayscale, skin spots, acnes, skin blemishes, bad anatomy, girl, loli, young, large breasts, red eyes, muscular"
21
+ - "dof, grayscale, black and white, bw, 3d, cartoon, anime, sketches, worst quality, low quality, normal quality, lowres, normal quality, monochrome, grayscale, skin spots, acnes, skin blemishes, bad anatomy, girl, loli, young, large breasts, red eyes, muscular,badhandsv5-neg, By bad artist -neg 1, monochrome"
22
+ - "holding an item, cowboy, hat, cartoon, 3d, disfigured, bad art, deformed,extra limbs,close up,b&w, wierd colors, blurry, duplicate, morbid, mutilated, [out of frame], extra fingers, mutated hands, poorly drawn hands, poorly drawn face, mutation, deformed, ugly, blurry, bad anatomy, bad proportions, extra limbs, cloned face, disfigured, out of frame, ugly, extra limbs, bad anatomy, gross proportions, malformed limbs, missing arms, missing legs, extra arms, extra legs, mutated hands, fused fingers, too many fingers, long neck, Photoshop, video game, ugly, tiling, poorly drawn hands, poorly drawn feet, poorly drawn face, out of frame, mutation, mutated, extra limbs, extra legs, extra arms, disfigured, deformed, cross-eye, body out of frame, blurry, bad art, bad anatomy, 3d render"
configs/prompts/3-RcnzCartoon.yaml ADDED
@@ -0,0 +1,22 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ RcnzCartoon:
2
+ base: ""
3
+ path: "models/DreamBooth_LoRA/rcnzCartoon3d_v10.safetensors"
4
+ motion_module:
5
+ - "models/Motion_Module/mm_sd_v14.ckpt"
6
+ - "models/Motion_Module/mm_sd_v15.ckpt"
7
+
8
+ seed: [16931037867122267877, 2094308009433392066, 4292543217695451092, 15572665120852309890]
9
+ steps: 25
10
+ guidance_scale: 7.5
11
+
12
+ prompt:
13
+ - "Jane Eyre with headphones, natural skin texture,4mm,k textures, soft cinematic light, adobe lightroom, photolab, hdr, intricate, elegant, highly detailed, sharp focus, cinematic look, soothing tones, insane details, intricate details, hyperdetailed, low contrast, soft cinematic light, dim colors, exposure blend, hdr, faded"
14
+ - "close up Portrait photo of muscular bearded guy in a worn mech suit, light bokeh, intricate, steel metal [rust], elegant, sharp focus, photo by greg rutkowski, soft lighting, vibrant colors, masterpiece, streets, detailed face"
15
+ - "absurdres, photorealistic, masterpiece, a 30 year old man with gold framed, aviator reading glasses and a black hooded jacket and a beard, professional photo, a character portrait, altermodern, detailed eyes, detailed lips, detailed face, grey eyes"
16
+ - "a golden labrador, warm vibrant colours, natural lighting, dappled lighting, diffused lighting, absurdres, highres,k, uhd, hdr, rtx, unreal, octane render, RAW photo, photorealistic, global illumination, subsurface scattering"
17
+
18
+ n_prompt:
19
+ - "deformed, distorted, disfigured, poorly drawn, bad anatomy, wrong anatomy, extra limb, missing limb, floating limbs, mutated hands and fingers, disconnected limbs, mutation, mutated, ugly, disgusting, blurry, amputation"
20
+ - "nude, cross eyed, tongue, open mouth, inside, 3d, cartoon, anime, sketches, worst quality, low quality, normal quality, lowres, normal quality, monochrome, grayscale, skin spots, acnes, skin blemishes, bad anatomy, red eyes, muscular"
21
+ - "easynegative, cartoon, anime, sketches, necklace, earrings worst quality, low quality, normal quality, bad anatomy, bad hands, shiny skin, error, missing fingers, extra digit, fewer digits, jpeg artifacts, signature, watermark, username, blurry, chubby, anorectic, bad eyes, old, wrinkled skin, red skin, photograph By bad artist -neg, big eyes, muscular face,"
22
+ - "beard, EasyNegative, lowres, chromatic aberration, depth of field, motion blur, blurry, bokeh, bad quality, worst quality, multiple arms, badhand"
configs/prompts/4-MajicMix.yaml ADDED
@@ -0,0 +1,22 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ MajicMix:
2
+ base: ""
3
+ path: "models/DreamBooth_LoRA/majicmixRealistic_v5Preview.safetensors"
4
+ motion_module:
5
+ - "models/Motion_Module/mm_sd_v14.ckpt"
6
+ - "models/Motion_Module/mm_sd_v15.ckpt"
7
+
8
+ seed: [1572448948722921032, 1099474677988590681, 6488833139725635347, 18339859844376517918]
9
+ steps: 25
10
+ guidance_scale: 7.5
11
+
12
+ prompt:
13
+ - "1girl, offshoulder, light smile, shiny skin best quality, masterpiece, photorealistic"
14
+ - "best quality, masterpiece, photorealistic, 1boy, 50 years old beard, dramatic lighting"
15
+ - "best quality, masterpiece, photorealistic, 1girl, light smile, shirt with collars, waist up, dramatic lighting, from below"
16
+ - "male, man, beard, bodybuilder, skinhead,cold face, tough guy, cowboyshot, tattoo, french windows, luxury hotel masterpiece, best quality, photorealistic"
17
+
18
+ n_prompt:
19
+ - "ng_deepnegative_v1_75t, badhandv4, worst quality, low quality, normal quality, lowres, bad anatomy, bad hands, watermark, moles"
20
+ - "nsfw, ng_deepnegative_v1_75t,badhandv4, worst quality, low quality, normal quality, lowres,watermark, monochrome"
21
+ - "nsfw, ng_deepnegative_v1_75t,badhandv4, worst quality, low quality, normal quality, lowres,watermark, monochrome"
22
+ - "nude, nsfw, ng_deepnegative_v1_75t, badhandv4, worst quality, low quality, normal quality, lowres, bad anatomy, bad hands, monochrome, grayscale watermark, moles, people"
configs/prompts/5-RealisticVision.yaml ADDED
@@ -0,0 +1,22 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ RealisticVision:
2
+ base: ""
3
+ path: "models/DreamBooth_LoRA/realisticVisionV20_v20.safetensors"
4
+ motion_module:
5
+ - "models/Motion_Module/mm_sd_v14.ckpt"
6
+ - "models/Motion_Module/mm_sd_v15.ckpt"
7
+
8
+ seed: [5658137986800322009, 12099779162349365895, 10499524853910852697, 16768009035333711932]
9
+ steps: 25
10
+ guidance_scale: 7.5
11
+
12
+ prompt:
13
+ - "b&w photo of 42 y.o man in black clothes, bald, face, half body, body, high detailed skin, skin pores, coastline, overcast weather, wind, waves, 8k uhd, dslr, soft lighting, high quality, film grain, Fujifilm XT3"
14
+ - "close up photo of a rabbit, forest, haze, halation, bloom, dramatic atmosphere, centred, rule of thirds, 200mm 1.4f macro shot"
15
+ - "photo of coastline, rocks, storm weather, wind, waves, lightning, 8k uhd, dslr, soft lighting, high quality, film grain, Fujifilm XT3"
16
+ - "night, b&w photo of old house, post apocalypse, forest, storm weather, wind, rocks, 8k uhd, dslr, soft lighting, high quality, film grain"
17
+
18
+ n_prompt:
19
+ - "semi-realistic, cgi, 3d, render, sketch, cartoon, drawing, anime, text, close up, cropped, out of frame, worst quality, low quality, jpeg artifacts, ugly, duplicate, morbid, mutilated, extra fingers, mutated hands, poorly drawn hands, poorly drawn face, mutation, deformed, blurry, dehydrated, bad anatomy, bad proportions, extra limbs, cloned face, disfigured, gross proportions, malformed limbs, missing arms, missing legs, extra arms, extra legs, fused fingers, too many fingers, long neck"
20
+ - "semi-realistic, cgi, 3d, render, sketch, cartoon, drawing, anime, text, close up, cropped, out of frame, worst quality, low quality, jpeg artifacts, ugly, duplicate, morbid, mutilated, extra fingers, mutated hands, poorly drawn hands, poorly drawn face, mutation, deformed, blurry, dehydrated, bad anatomy, bad proportions, extra limbs, cloned face, disfigured, gross proportions, malformed limbs, missing arms, missing legs, extra arms, extra legs, fused fingers, too many fingers, long neck"
21
+ - "blur, haze, deformed iris, deformed pupils, semi-realistic, cgi, 3d, render, sketch, cartoon, drawing, anime, mutated hands and fingers, deformed, distorted, disfigured, poorly drawn, bad anatomy, wrong anatomy, extra limb, missing limb, floating limbs, disconnected limbs, mutation, mutated, ugly, disgusting, amputation"
22
+ - "blur, haze, deformed iris, deformed pupils, semi-realistic, cgi, 3d, render, sketch, cartoon, drawing, anime, art, mutated hands and fingers, deformed, distorted, disfigured, poorly drawn, bad anatomy, wrong anatomy, extra limb, missing limb, floating limbs, disconnected limbs, mutation, mutated, ugly, disgusting, amputation"
configs/prompts/6-Tusun.yaml ADDED
@@ -0,0 +1,20 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ Tusun:
2
+ base: "models/DreamBooth_LoRA/moonfilm_reality20.safetensors"
3
+ path: "models/DreamBooth_LoRA/TUSUN.safetensors"
4
+ motion_module:
5
+ - "models/Motion_Module/mm_sd_v14.ckpt"
6
+ - "models/Motion_Module/mm_sd_v15.ckpt"
7
+
8
+ seed: [10154078483724687116, 2664393535095473805, 4231566096207622938, 1713349740448094493]
9
+ steps: 25
10
+ guidance_scale: 7.5
11
+ lora_alpha: 0.6
12
+
13
+ prompt:
14
+ - "tusuncub with its mouth open, blurry, open mouth, fangs, photo background, looking at viewer, tongue, full body, solo, cute and lovely, Beautiful and realistic eye details, perfect anatomy, Nonsense, pure background, Centered-Shot, realistic photo, photograph, 4k, hyper detailed, DSLR, 24 Megapixels, 8mm Lens, Full Frame, film grain, Global Illumination, studio Lighting, Award Winning Photography, diffuse reflection, ray tracing"
15
+ - "cute tusun with a blurry background, black background, simple background, signature, face, solo, cute and lovely, Beautiful and realistic eye details, perfect anatomy, Nonsense, pure background, Centered-Shot, realistic photo, photograph, 4k, hyper detailed, DSLR, 24 Megapixels, 8mm Lens, Full Frame, film grain, Global Illumination, studio Lighting, Award Winning Photography, diffuse reflection, ray tracing"
16
+ - "cut tusuncub walking in the snow, blurry, looking at viewer, depth of field, blurry background, full body, solo, cute and lovely, Beautiful and realistic eye details, perfect anatomy, Nonsense, pure background, Centered-Shot, realistic photo, photograph, 4k, hyper detailed, DSLR, 24 Megapixels, 8mm Lens, Full Frame, film grain, Global Illumination, studio Lighting, Award Winning Photography, diffuse reflection, ray tracing"
17
+ - "character design, cyberpunk tusun kitten wearing astronaut suit, sci-fic, realistic eye color and details, fluffy, big head, science fiction, communist ideology, Cyborg, fantasy, intense angle, soft lighting, photograph, 4k, hyper detailed, portrait wallpaper, realistic, photo-realistic, DSLR, 24 Megapixels, Full Frame, vibrant details, octane render, finely detail, best quality, incredibly absurdres, robotic parts, rim light, vibrant details, luxurious cyberpunk, hyperrealistic, cable electric wires, microchip, full body"
18
+
19
+ n_prompt:
20
+ - "worst quality, low quality, deformed, distorted, disfigured, bad eyes, bad anatomy, disconnected limbs, wrong body proportions, low quality, worst quality, text, watermark, signatre, logo, illustration, painting, cartoons, ugly, easy_negative"
configs/prompts/7-FilmVelvia.yaml ADDED
@@ -0,0 +1,23 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ FilmVelvia:
2
+ base: "models/DreamBooth_LoRA/majicmixRealistic_v4.safetensors"
3
+ path: "models/DreamBooth_LoRA/FilmVelvia2.safetensors"
4
+ motion_module:
5
+ - "models/Motion_Module/mm_sd_v14.ckpt"
6
+ - "models/Motion_Module/mm_sd_v15.ckpt"
7
+
8
+ seed: [358675358833372813, 3519455280971923743, 11684545350557985081, 8696855302100399877]
9
+ steps: 25
10
+ guidance_scale: 7.5
11
+ lora_alpha: 0.6
12
+
13
+ prompt:
14
+ - "a woman standing on the side of a road at night,girl, long hair, motor vehicle, car, looking at viewer, ground vehicle, night, hands in pockets, blurry background, coat, black hair, parted lips, bokeh, jacket, brown hair, outdoors, red lips, upper body, artist name"
15
+ - ", dark shot,0mm, portrait quality of a arab man worker,boy, wasteland that stands out vividly against the background of the desert, barren landscape, closeup, moles skin, soft light, sharp, exposure blend, medium shot, bokeh, hdr, high contrast, cinematic, teal and orange5, muted colors, dim colors, soothing tones, low saturation, hyperdetailed, noir"
16
+ - "fashion photography portrait of 1girl, offshoulder, fluffy short hair, soft light, rim light, beautiful shadow, low key, photorealistic, raw photo, natural skin texture, realistic eye and face details, hyperrealism, ultra high res, 4K, Best quality, masterpiece, necklace, cleavage, in the dark"
17
+ - "In this lighthearted portrait, a woman is dressed as a fierce warrior, armed with an arsenal of paintbrushes and palette knives. Her war paint is composed of thick, vibrant strokes of color, and her armor is made of paint tubes and paint-splattered canvases. She stands victoriously atop a mountain of conquered blank canvases, with a beautiful, colorful landscape behind her, symbolizing the power of art and creativity. bust Portrait, close-up, Bright and transparent scene lighting, "
18
+
19
+ n_prompt:
20
+ - "cartoon, anime, sketches,worst quality, low quality, deformed, distorted, disfigured, bad eyes, wrong lips, weird mouth, bad teeth, mutated hands and fingers, bad anatomy, wrong anatomy, amputation, extra limb, missing limb, floating limbs, disconnected limbs, mutation, ugly, disgusting, bad_pictures, negative_hand-neg"
21
+ - "cartoon, anime, sketches,worst quality, low quality, deformed, distorted, disfigured, bad eyes, wrong lips, weird mouth, bad teeth, mutated hands and fingers, bad anatomy, wrong anatomy, amputation, extra limb, missing limb, floating limbs, disconnected limbs, mutation, ugly, disgusting, bad_pictures, negative_hand-neg"
22
+ - "wrong white balance, dark, cartoon, anime, sketches,worst quality, low quality, deformed, distorted, disfigured, bad eyes, wrong lips, weird mouth, bad teeth, mutated hands and fingers, bad anatomy, wrong anatomy, amputation, extra limb, missing limb, floating limbs, disconnected limbs, mutation, ugly, disgusting, bad_pictures, negative_hand-neg"
23
+ - "wrong white balance, dark, cartoon, anime, sketches,worst quality, low quality, deformed, distorted, disfigured, bad eyes, wrong lips, weird mouth, bad teeth, mutated hands and fingers, bad anatomy, wrong anatomy, amputation, extra limb, missing limb, floating limbs, disconnected limbs, mutation, ugly, disgusting, bad_pictures, negative_hand-neg"
configs/prompts/8-GhibliBackground.yaml ADDED
@@ -0,0 +1,20 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ GhibliBackground:
2
+ base: "models/DreamBooth_LoRA/CounterfeitV30_25.safetensors"
3
+ path: "models/DreamBooth_LoRA/lora_Ghibli_n3.safetensors"
4
+ motion_module:
5
+ - "models/Motion_Module/mm_sd_v14.ckpt"
6
+ - "models/Motion_Module/mm_sd_v15.ckpt"
7
+
8
+ seed: [8775748474469046618, 5893874876080607656, 11911465742147695752, 12437784838692000640]
9
+ steps: 25
10
+ guidance_scale: 7.5
11
+ lora_alpha: 1.0
12
+
13
+ prompt:
14
+ - "best quality,single build,architecture, blue_sky, building,cloudy_sky, day, fantasy, fence, field, house, build,architecture,landscape, moss, outdoors, overgrown, path, river, road, rock, scenery, sky, sword, tower, tree, waterfall"
15
+ - "black_border, building, city, day, fantasy, ice, landscape, letterboxed, mountain, ocean, outdoors, planet, scenery, ship, snow, snowing, water, watercraft, waterfall, winter"
16
+ - ",mysterious sea area, fantasy,build,concept"
17
+ - "Tomb Raider,Scenography,Old building"
18
+
19
+ n_prompt:
20
+ - "easynegative,bad_construction,bad_structure,bad_wail,bad_windows,blurry,cloned_window,cropped,deformed,disfigured,error,extra_windows,extra_chimney,extra_door,extra_structure,extra_frame,fewer_digits,fused_structure,gross_proportions,jpeg_artifacts,long_roof,low_quality,structure_limbs,missing_windows,missing_doors,missing_roofs,mutated_structure,mutation,normal_quality,out_of_frame,owres,poorly_drawn_structure,poorly_drawn_house,signature,text,too_many_windows,ugly,username,uta,watermark,worst_quality"
download_bashscripts/0-MotionModule.sh ADDED
@@ -0,0 +1,2 @@
 
 
 
1
+ gdown 1RqkQuGPaCO5sGZ6V6KZ-jUWmsRu48Kdq -O models/Motion_Module/
2
+ gdown 1ql0g_Ys4UCz2RnokYlBjyOYPbttbIpbu -O models/Motion_Module/
download_bashscripts/1-ToonYou.sh ADDED
@@ -0,0 +1,2 @@
 
 
 
1
+ #!/bin/bash
2
+ wget https://civitai.com/api/download/models/78775 -P models/DreamBooth_LoRA/ --content-disposition --no-check-certificate
download_bashscripts/2-Lyriel.sh ADDED
@@ -0,0 +1,2 @@
 
 
 
1
+ #!/bin/bash
2
+ wget https://civitai.com/api/download/models/72396 -P models/DreamBooth_LoRA/ --content-disposition --no-check-certificate
download_bashscripts/3-RcnzCartoon.sh ADDED
@@ -0,0 +1,2 @@
 
 
 
1
+ #!/bin/bash
2
+ wget https://civitai.com/api/download/models/71009 -P models/DreamBooth_LoRA/ --content-disposition --no-check-certificate
download_bashscripts/4-MajicMix.sh ADDED
@@ -0,0 +1,2 @@
 
 
 
1
+ #!/bin/bash
2
+ wget https://civitai.com/api/download/models/79068 -P models/DreamBooth_LoRA/ --content-disposition --no-check-certificate
download_bashscripts/5-RealisticVision.sh ADDED
@@ -0,0 +1,2 @@
 
 
 
1
+ #!/bin/bash
2
+ wget https://civitai.com/api/download/models/29460 -P models/DreamBooth_LoRA/ --content-disposition --no-check-certificate
download_bashscripts/6-Tusun.sh ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ #!/bin/bash
2
+ wget https://civitai.com/api/download/models/97261 -P models/DreamBooth_LoRA/ --content-disposition --no-check-certificate
3
+ wget https://civitai.com/api/download/models/50705 -P models/DreamBooth_LoRA/ --content-disposition --no-check-certificate
download_bashscripts/7-FilmVelvia.sh ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ #!/bin/bash
2
+ wget https://civitai.com/api/download/models/90115 -P models/DreamBooth_LoRA/ --content-disposition --no-check-certificate
3
+ wget https://civitai.com/api/download/models/92475 -P models/DreamBooth_LoRA/ --content-disposition --no-check-certificate
download_bashscripts/8-GhibliBackground.sh ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ #!/bin/bash
2
+ wget https://civitai.com/api/download/models/102828 -P models/DreamBooth_LoRA/ --content-disposition --no-check-certificate
3
+ wget https://civitai.com/api/download/models/57618 -P models/DreamBooth_LoRA/ --content-disposition --no-check-certificate
models/DreamBooth_LoRA/Put personalized T2I checkpoints here.txt ADDED
File without changes
models/DreamBooth_LoRA/lyriel_v16.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:ec6f68ea6388951d53d6c8178c22aecfd1b4fedfe31f5a5814ddd7638c4eff37
3
+ size 2132625432
models/DreamBooth_LoRA/majicmixRealistic_v5Preview.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:a38fa861a24f4f4c6e0f68289101e645dd9ca1e93e1049cc8a4f2a77513fad52
3
+ size 2400040290
models/DreamBooth_LoRA/rcnzCartoon3d_v10.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:a6b4c0392d7486bfa4fd1a31c7b7d2679f743f8ea8d9f219c82b5c33db31ddb9
3
+ size 2132625644
models/DreamBooth_LoRA/realisticVisionV20_v20.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:c0d1994c73d784a17a5b335ae8bda02dcc8dd2fc5f5dbf55169d5aab385e53f2
3
+ size 2132650523
models/DreamBooth_LoRA/toonyou_beta3.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:52768d2bc4e73af920df3466d0806d9a40727dab0869fc7aa2f3bc82214bb071
3
+ size 2132626252
models/Motion_Module/Put motion module checkpoints here.txt ADDED
File without changes
models/Motion_Module/mm_sd_v14.ckpt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:aa7fd8a200a89031edd84487e2a757c5315460eca528fa70d4b3885c399bffd5
3
+ size 1672133921
models/Motion_Module/mm_sd_v15.ckpt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:cf16ea656cb16124990c8e2c70a29c793f9841f3a2223073fac8bd89ebd9b69a
3
+ size 1672133921
models/StableDiffusion/Put diffusers stable-diffusion-v1-5 repo here.txt ADDED
File without changes
models/StableDiffusion/stable-diffusion-v1-5/.gitattributes ADDED
@@ -0,0 +1,35 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ *.7z filter=lfs diff=lfs merge=lfs -text
2
+ *.arrow filter=lfs diff=lfs merge=lfs -text
3
+ *.bin filter=lfs diff=lfs merge=lfs -text
4
+ *.bz2 filter=lfs diff=lfs merge=lfs -text
5
+ *.ftz filter=lfs diff=lfs merge=lfs -text
6
+ *.gz filter=lfs diff=lfs merge=lfs -text
7
+ *.h5 filter=lfs diff=lfs merge=lfs -text
8
+ *.joblib filter=lfs diff=lfs merge=lfs -text
9
+ *.lfs.* filter=lfs diff=lfs merge=lfs -text
10
+ *.mlmodel filter=lfs diff=lfs merge=lfs -text
11
+ *.model filter=lfs diff=lfs merge=lfs -text
12
+ *.msgpack filter=lfs diff=lfs merge=lfs -text
13
+ *.npy filter=lfs diff=lfs merge=lfs -text
14
+ *.npz filter=lfs diff=lfs merge=lfs -text
15
+ *.onnx filter=lfs diff=lfs merge=lfs -text
16
+ *.ot filter=lfs diff=lfs merge=lfs -text
17
+ *.parquet filter=lfs diff=lfs merge=lfs -text
18
+ *.pb filter=lfs diff=lfs merge=lfs -text
19
+ *.pickle filter=lfs diff=lfs merge=lfs -text
20
+ *.pkl filter=lfs diff=lfs merge=lfs -text
21
+ *.pt filter=lfs diff=lfs merge=lfs -text
22
+ *.pth filter=lfs diff=lfs merge=lfs -text
23
+ *.rar filter=lfs diff=lfs merge=lfs -text
24
+ *.safetensors filter=lfs diff=lfs merge=lfs -text
25
+ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
26
+ *.tar.* filter=lfs diff=lfs merge=lfs -text
27
+ *.tflite filter=lfs diff=lfs merge=lfs -text
28
+ *.tgz filter=lfs diff=lfs merge=lfs -text
29
+ *.wasm filter=lfs diff=lfs merge=lfs -text
30
+ *.xz filter=lfs diff=lfs merge=lfs -text
31
+ *.zip filter=lfs diff=lfs merge=lfs -text
32
+ *.zst filter=lfs diff=lfs merge=lfs -text
33
+ *tfevents* filter=lfs diff=lfs merge=lfs -text
34
+ v1-5-pruned-emaonly.ckpt filter=lfs diff=lfs merge=lfs -text
35
+ v1-5-pruned.ckpt filter=lfs diff=lfs merge=lfs -text
models/StableDiffusion/stable-diffusion-v1-5/README.md ADDED
@@ -0,0 +1,207 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ ---
2
+ license: creativeml-openrail-m
3
+ tags:
4
+ - stable-diffusion
5
+ - stable-diffusion-diffusers
6
+ - text-to-image
7
+ inference: true
8
+ extra_gated_prompt: |-
9
+ This model is open access and available to all, with a CreativeML OpenRAIL-M license further specifying rights and usage.
10
+ The CreativeML OpenRAIL License specifies:
11
+
12
+ 1. You can't use the model to deliberately produce nor share illegal or harmful outputs or content
13
+ 2. CompVis claims no rights on the outputs you generate, you are free to use them and are accountable for their use which must not go against the provisions set in the license
14
+ 3. You may re-distribute the weights and use the model commercially and/or as a service. If you do, please be aware you have to include the same use restrictions as the ones in the license and share a copy of the CreativeML OpenRAIL-M to all your users (please read the license entirely and carefully)
15
+ Please read the full license carefully here: https://huggingface.co/spaces/CompVis/stable-diffusion-license
16
+
17
+ extra_gated_heading: Please read the LICENSE to access this model
18
+ ---
19
+
20
+ # Stable Diffusion v1-5 Model Card
21
+
22
+ Stable Diffusion is a latent text-to-image diffusion model capable of generating photo-realistic images given any text input.
23
+ For more information about how Stable Diffusion functions, please have a look at [🤗's Stable Diffusion blog](https://huggingface.co/blog/stable_diffusion).
24
+
25
+ The **Stable-Diffusion-v1-5** checkpoint was initialized with the weights of the [Stable-Diffusion-v1-2](https:/steps/huggingface.co/CompVis/stable-diffusion-v1-2)
26
+ checkpoint and subsequently fine-tuned on 595k steps at resolution 512x512 on "laion-aesthetics v2 5+" and 10% dropping of the text-conditioning to improve [classifier-free guidance sampling](https://arxiv.org/abs/2207.12598).
27
+
28
+ You can use this both with the [🧨Diffusers library](https://github.com/huggingface/diffusers) and the [RunwayML GitHub repository](https://github.com/runwayml/stable-diffusion).
29
+
30
+ ### Diffusers
31
+ ```py
32
+ from diffusers import StableDiffusionPipeline
33
+ import torch
34
+
35
+ model_id = "runwayml/stable-diffusion-v1-5"
36
+ pipe = StableDiffusionPipeline.from_pretrained(model_id, torch_dtype=torch.float16)
37
+ pipe = pipe.to("cuda")
38
+
39
+ prompt = "a photo of an astronaut riding a horse on mars"
40
+ image = pipe(prompt).images[0]
41
+
42
+ image.save("astronaut_rides_horse.png")
43
+ ```
44
+ For more detailed instructions, use-cases and examples in JAX follow the instructions [here](https://github.com/huggingface/diffusers#text-to-image-generation-with-stable-diffusion)
45
+
46
+ ### Original GitHub Repository
47
+
48
+ 1. Download the weights
49
+ - [v1-5-pruned-emaonly.ckpt](https://huggingface.co/runwayml/stable-diffusion-v1-5/resolve/main/v1-5-pruned-emaonly.ckpt) - 4.27GB, ema-only weight. uses less VRAM - suitable for inference
50
+ - [v1-5-pruned.ckpt](https://huggingface.co/runwayml/stable-diffusion-v1-5/resolve/main/v1-5-pruned.ckpt) - 7.7GB, ema+non-ema weights. uses more VRAM - suitable for fine-tuning
51
+
52
+ 2. Follow instructions [here](https://github.com/runwayml/stable-diffusion).
53
+
54
+ ## Model Details
55
+ - **Developed by:** Robin Rombach, Patrick Esser
56
+ - **Model type:** Diffusion-based text-to-image generation model
57
+ - **Language(s):** English
58
+ - **License:** [The CreativeML OpenRAIL M license](https://huggingface.co/spaces/CompVis/stable-diffusion-license) is an [Open RAIL M license](https://www.licenses.ai/blog/2022/8/18/naming-convention-of-responsible-ai-licenses), adapted from the work that [BigScience](https://bigscience.huggingface.co/) and [the RAIL Initiative](https://www.licenses.ai/) are jointly carrying in the area of responsible AI licensing. See also [the article about the BLOOM Open RAIL license](https://bigscience.huggingface.co/blog/the-bigscience-rail-license) on which our license is based.
59
+ - **Model Description:** This is a model that can be used to generate and modify images based on text prompts. It is a [Latent Diffusion Model](https://arxiv.org/abs/2112.10752) that uses a fixed, pretrained text encoder ([CLIP ViT-L/14](https://arxiv.org/abs/2103.00020)) as suggested in the [Imagen paper](https://arxiv.org/abs/2205.11487).
60
+ - **Resources for more information:** [GitHub Repository](https://github.com/CompVis/stable-diffusion), [Paper](https://arxiv.org/abs/2112.10752).
61
+ - **Cite as:**
62
+
63
+ @InProceedings{Rombach_2022_CVPR,
64
+ author = {Rombach, Robin and Blattmann, Andreas and Lorenz, Dominik and Esser, Patrick and Ommer, Bj\"orn},
65
+ title = {High-Resolution Image Synthesis With Latent Diffusion Models},
66
+ booktitle = {Proceedings of the IEEE/CVF Conference on Computer Vision and Pattern Recognition (CVPR)},
67
+ month = {June},
68
+ year = {2022},
69
+ pages = {10684-10695}
70
+ }
71
+
72
+ # Uses
73
+
74
+ ## Direct Use
75
+ The model is intended for research purposes only. Possible research areas and
76
+ tasks include
77
+
78
+ - Safe deployment of models which have the potential to generate harmful content.
79
+ - Probing and understanding the limitations and biases of generative models.
80
+ - Generation of artworks and use in design and other artistic processes.
81
+ - Applications in educational or creative tools.
82
+ - Research on generative models.
83
+
84
+ Excluded uses are described below.
85
+
86
+ ### Misuse, Malicious Use, and Out-of-Scope Use
87
+ _Note: This section is taken from the [DALLE-MINI model card](https://huggingface.co/dalle-mini/dalle-mini), but applies in the same way to Stable Diffusion v1_.
88
+
89
+
90
+ The model should not be used to intentionally create or disseminate images that create hostile or alienating environments for people. This includes generating images that people would foreseeably find disturbing, distressing, or offensive; or content that propagates historical or current stereotypes.
91
+
92
+ #### Out-of-Scope Use
93
+ The model was not trained to be factual or true representations of people or events, and therefore using the model to generate such content is out-of-scope for the abilities of this model.
94
+
95
+ #### Misuse and Malicious Use
96
+ Using the model to generate content that is cruel to individuals is a misuse of this model. This includes, but is not limited to:
97
+
98
+ - Generating demeaning, dehumanizing, or otherwise harmful representations of people or their environments, cultures, religions, etc.
99
+ - Intentionally promoting or propagating discriminatory content or harmful stereotypes.
100
+ - Impersonating individuals without their consent.
101
+ - Sexual content without consent of the people who might see it.
102
+ - Mis- and disinformation
103
+ - Representations of egregious violence and gore
104
+ - Sharing of copyrighted or licensed material in violation of its terms of use.
105
+ - Sharing content that is an alteration of copyrighted or licensed material in violation of its terms of use.
106
+
107
+ ## Limitations and Bias
108
+
109
+ ### Limitations
110
+
111
+ - The model does not achieve perfect photorealism
112
+ - The model cannot render legible text
113
+ - The model does not perform well on more difficult tasks which involve compositionality, such as rendering an image corresponding to “A red cube on top of a blue sphere”
114
+ - Faces and people in general may not be generated properly.
115
+ - The model was trained mainly with English captions and will not work as well in other languages.
116
+ - The autoencoding part of the model is lossy
117
+ - The model was trained on a large-scale dataset
118
+ [LAION-5B](https://laion.ai/blog/laion-5b/) which contains adult material
119
+ and is not fit for product use without additional safety mechanisms and
120
+ considerations.
121
+ - No additional measures were used to deduplicate the dataset. As a result, we observe some degree of memorization for images that are duplicated in the training data.
122
+ The training data can be searched at [https://rom1504.github.io/clip-retrieval/](https://rom1504.github.io/clip-retrieval/) to possibly assist in the detection of memorized images.
123
+
124
+ ### Bias
125
+
126
+ While the capabilities of image generation models are impressive, they can also reinforce or exacerbate social biases.
127
+ Stable Diffusion v1 was trained on subsets of [LAION-2B(en)](https://laion.ai/blog/laion-5b/),
128
+ which consists of images that are primarily limited to English descriptions.
129
+ Texts and images from communities and cultures that use other languages are likely to be insufficiently accounted for.
130
+ This affects the overall output of the model, as white and western cultures are often set as the default. Further, the
131
+ ability of the model to generate content with non-English prompts is significantly worse than with English-language prompts.
132
+
133
+ ### Safety Module
134
+
135
+ The intended use of this model is with the [Safety Checker](https://github.com/huggingface/diffusers/blob/main/src/diffusers/pipelines/stable_diffusion/safety_checker.py) in Diffusers.
136
+ This checker works by checking model outputs against known hard-coded NSFW concepts.
137
+ The concepts are intentionally hidden to reduce the likelihood of reverse-engineering this filter.
138
+ Specifically, the checker compares the class probability of harmful concepts in the embedding space of the `CLIPTextModel` *after generation* of the images.
139
+ The concepts are passed into the model with the generated image and compared to a hand-engineered weight for each NSFW concept.
140
+
141
+
142
+ ## Training
143
+
144
+ **Training Data**
145
+ The model developers used the following dataset for training the model:
146
+
147
+ - LAION-2B (en) and subsets thereof (see next section)
148
+
149
+ **Training Procedure**
150
+ Stable Diffusion v1-5 is a latent diffusion model which combines an autoencoder with a diffusion model that is trained in the latent space of the autoencoder. During training,
151
+
152
+ - Images are encoded through an encoder, which turns images into latent representations. The autoencoder uses a relative downsampling factor of 8 and maps images of shape H x W x 3 to latents of shape H/f x W/f x 4
153
+ - Text prompts are encoded through a ViT-L/14 text-encoder.
154
+ - The non-pooled output of the text encoder is fed into the UNet backbone of the latent diffusion model via cross-attention.
155
+ - The loss is a reconstruction objective between the noise that was added to the latent and the prediction made by the UNet.
156
+
157
+ Currently six Stable Diffusion checkpoints are provided, which were trained as follows.
158
+ - [`stable-diffusion-v1-1`](https://huggingface.co/CompVis/stable-diffusion-v1-1): 237,000 steps at resolution `256x256` on [laion2B-en](https://huggingface.co/datasets/laion/laion2B-en).
159
+ 194,000 steps at resolution `512x512` on [laion-high-resolution](https://huggingface.co/datasets/laion/laion-high-resolution) (170M examples from LAION-5B with resolution `>= 1024x1024`).
160
+ - [`stable-diffusion-v1-2`](https://huggingface.co/CompVis/stable-diffusion-v1-2): Resumed from `stable-diffusion-v1-1`.
161
+ 515,000 steps at resolution `512x512` on "laion-improved-aesthetics" (a subset of laion2B-en,
162
+ filtered to images with an original size `>= 512x512`, estimated aesthetics score `> 5.0`, and an estimated watermark probability `< 0.5`. The watermark estimate is from the LAION-5B metadata, the aesthetics score is estimated using an [improved aesthetics estimator](https://github.com/christophschuhmann/improved-aesthetic-predictor)).
163
+ - [`stable-diffusion-v1-3`](https://huggingface.co/CompVis/stable-diffusion-v1-3): Resumed from `stable-diffusion-v1-2` - 195,000 steps at resolution `512x512` on "laion-improved-aesthetics" and 10 % dropping of the text-conditioning to improve [classifier-free guidance sampling](https://arxiv.org/abs/2207.12598).
164
+ - [`stable-diffusion-v1-4`](https://huggingface.co/CompVis/stable-diffusion-v1-4) Resumed from `stable-diffusion-v1-2` - 225,000 steps at resolution `512x512` on "laion-aesthetics v2 5+" and 10 % dropping of the text-conditioning to improve [classifier-free guidance sampling](https://arxiv.org/abs/2207.12598).
165
+ - [`stable-diffusion-v1-5`](https://huggingface.co/runwayml/stable-diffusion-v1-5) Resumed from `stable-diffusion-v1-2` - 595,000 steps at resolution `512x512` on "laion-aesthetics v2 5+" and 10 % dropping of the text-conditioning to improve [classifier-free guidance sampling](https://arxiv.org/abs/2207.12598).
166
+ - [`stable-diffusion-inpainting`](https://huggingface.co/runwayml/stable-diffusion-inpainting) Resumed from `stable-diffusion-v1-5` - then 440,000 steps of inpainting training at resolution 512x512 on “laion-aesthetics v2 5+” and 10% dropping of the text-conditioning. For inpainting, the UNet has 5 additional input channels (4 for the encoded masked-image and 1 for the mask itself) whose weights were zero-initialized after restoring the non-inpainting checkpoint. During training, we generate synthetic masks and in 25% mask everything.
167
+
168
+ - **Hardware:** 32 x 8 x A100 GPUs
169
+ - **Optimizer:** AdamW
170
+ - **Gradient Accumulations**: 2
171
+ - **Batch:** 32 x 8 x 2 x 4 = 2048
172
+ - **Learning rate:** warmup to 0.0001 for 10,000 steps and then kept constant
173
+
174
+ ## Evaluation Results
175
+ Evaluations with different classifier-free guidance scales (1.5, 2.0, 3.0, 4.0,
176
+ 5.0, 6.0, 7.0, 8.0) and 50 PNDM/PLMS sampling
177
+ steps show the relative improvements of the checkpoints:
178
+
179
+ ![pareto](https://huggingface.co/CompVis/stable-diffusion/resolve/main/v1-1-to-v1-5.png)
180
+
181
+ Evaluated using 50 PLMS steps and 10000 random prompts from the COCO2017 validation set, evaluated at 512x512 resolution. Not optimized for FID scores.
182
+ ## Environmental Impact
183
+
184
+ **Stable Diffusion v1** **Estimated Emissions**
185
+ Based on that information, we estimate the following CO2 emissions using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). The hardware, runtime, cloud provider, and compute region were utilized to estimate the carbon impact.
186
+
187
+ - **Hardware Type:** A100 PCIe 40GB
188
+ - **Hours used:** 150000
189
+ - **Cloud Provider:** AWS
190
+ - **Compute Region:** US-east
191
+ - **Carbon Emitted (Power consumption x Time x Carbon produced based on location of power grid):** 11250 kg CO2 eq.
192
+
193
+
194
+ ## Citation
195
+
196
+ ```bibtex
197
+ @InProceedings{Rombach_2022_CVPR,
198
+ author = {Rombach, Robin and Blattmann, Andreas and Lorenz, Dominik and Esser, Patrick and Ommer, Bj\"orn},
199
+ title = {High-Resolution Image Synthesis With Latent Diffusion Models},
200
+ booktitle = {Proceedings of the IEEE/CVF Conference on Computer Vision and Pattern Recognition (CVPR)},
201
+ month = {June},
202
+ year = {2022},
203
+ pages = {10684-10695}
204
+ }
205
+ ```
206
+
207
+ *This model card was written by: Robin Rombach and Patrick Esser and is based on the [DALL-E Mini model card](https://huggingface.co/dalle-mini/dalle-mini).*
models/StableDiffusion/stable-diffusion-v1-5/feature_extractor/preprocessor_config.json ADDED
@@ -0,0 +1,20 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "crop_size": 224,
3
+ "do_center_crop": true,
4
+ "do_convert_rgb": true,
5
+ "do_normalize": true,
6
+ "do_resize": true,
7
+ "feature_extractor_type": "CLIPFeatureExtractor",
8
+ "image_mean": [
9
+ 0.48145466,
10
+ 0.4578275,
11
+ 0.40821073
12
+ ],
13
+ "image_std": [
14
+ 0.26862954,
15
+ 0.26130258,
16
+ 0.27577711
17
+ ],
18
+ "resample": 3,
19
+ "size": 224
20
+ }
models/StableDiffusion/stable-diffusion-v1-5/model_index.json ADDED
@@ -0,0 +1,32 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "_class_name": "StableDiffusionPipeline",
3
+ "_diffusers_version": "0.6.0",
4
+ "feature_extractor": [
5
+ "transformers",
6
+ "CLIPImageProcessor"
7
+ ],
8
+ "safety_checker": [
9
+ "stable_diffusion",
10
+ "StableDiffusionSafetyChecker"
11
+ ],
12
+ "scheduler": [
13
+ "diffusers",
14
+ "PNDMScheduler"
15
+ ],
16
+ "text_encoder": [
17
+ "transformers",
18
+ "CLIPTextModel"
19
+ ],
20
+ "tokenizer": [
21
+ "transformers",
22
+ "CLIPTokenizer"
23
+ ],
24
+ "unet": [
25
+ "diffusers",
26
+ "UNet2DConditionModel"
27
+ ],
28
+ "vae": [
29
+ "diffusers",
30
+ "AutoencoderKL"
31
+ ]
32
+ }
models/StableDiffusion/stable-diffusion-v1-5/safety_checker/config.json ADDED
@@ -0,0 +1,175 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "_commit_hash": "4bb648a606ef040e7685bde262611766a5fdd67b",
3
+ "_name_or_path": "CompVis/stable-diffusion-safety-checker",
4
+ "architectures": [
5
+ "StableDiffusionSafetyChecker"
6
+ ],
7
+ "initializer_factor": 1.0,
8
+ "logit_scale_init_value": 2.6592,
9
+ "model_type": "clip",
10
+ "projection_dim": 768,
11
+ "text_config": {
12
+ "_name_or_path": "",
13
+ "add_cross_attention": false,
14
+ "architectures": null,
15
+ "attention_dropout": 0.0,
16
+ "bad_words_ids": null,
17
+ "bos_token_id": 0,
18
+ "chunk_size_feed_forward": 0,
19
+ "cross_attention_hidden_size": null,
20
+ "decoder_start_token_id": null,
21
+ "diversity_penalty": 0.0,
22
+ "do_sample": false,
23
+ "dropout": 0.0,
24
+ "early_stopping": false,
25
+ "encoder_no_repeat_ngram_size": 0,
26
+ "eos_token_id": 2,
27
+ "exponential_decay_length_penalty": null,
28
+ "finetuning_task": null,
29
+ "forced_bos_token_id": null,
30
+ "forced_eos_token_id": null,
31
+ "hidden_act": "quick_gelu",
32
+ "hidden_size": 768,
33
+ "id2label": {
34
+ "0": "LABEL_0",
35
+ "1": "LABEL_1"
36
+ },
37
+ "initializer_factor": 1.0,
38
+ "initializer_range": 0.02,
39
+ "intermediate_size": 3072,
40
+ "is_decoder": false,
41
+ "is_encoder_decoder": false,
42
+ "label2id": {
43
+ "LABEL_0": 0,
44
+ "LABEL_1": 1
45
+ },
46
+ "layer_norm_eps": 1e-05,
47
+ "length_penalty": 1.0,
48
+ "max_length": 20,
49
+ "max_position_embeddings": 77,
50
+ "min_length": 0,
51
+ "model_type": "clip_text_model",
52
+ "no_repeat_ngram_size": 0,
53
+ "num_attention_heads": 12,
54
+ "num_beam_groups": 1,
55
+ "num_beams": 1,
56
+ "num_hidden_layers": 12,
57
+ "num_return_sequences": 1,
58
+ "output_attentions": false,
59
+ "output_hidden_states": false,
60
+ "output_scores": false,
61
+ "pad_token_id": 1,
62
+ "prefix": null,
63
+ "problem_type": null,
64
+ "pruned_heads": {},
65
+ "remove_invalid_values": false,
66
+ "repetition_penalty": 1.0,
67
+ "return_dict": true,
68
+ "return_dict_in_generate": false,
69
+ "sep_token_id": null,
70
+ "task_specific_params": null,
71
+ "temperature": 1.0,
72
+ "tf_legacy_loss": false,
73
+ "tie_encoder_decoder": false,
74
+ "tie_word_embeddings": true,
75
+ "tokenizer_class": null,
76
+ "top_k": 50,
77
+ "top_p": 1.0,
78
+ "torch_dtype": null,
79
+ "torchscript": false,
80
+ "transformers_version": "4.22.0.dev0",
81
+ "typical_p": 1.0,
82
+ "use_bfloat16": false,
83
+ "vocab_size": 49408
84
+ },
85
+ "text_config_dict": {
86
+ "hidden_size": 768,
87
+ "intermediate_size": 3072,
88
+ "num_attention_heads": 12,
89
+ "num_hidden_layers": 12
90
+ },
91
+ "torch_dtype": "float32",
92
+ "transformers_version": null,
93
+ "vision_config": {
94
+ "_name_or_path": "",
95
+ "add_cross_attention": false,
96
+ "architectures": null,
97
+ "attention_dropout": 0.0,
98
+ "bad_words_ids": null,
99
+ "bos_token_id": null,
100
+ "chunk_size_feed_forward": 0,
101
+ "cross_attention_hidden_size": null,
102
+ "decoder_start_token_id": null,
103
+ "diversity_penalty": 0.0,
104
+ "do_sample": false,
105
+ "dropout": 0.0,
106
+ "early_stopping": false,
107
+ "encoder_no_repeat_ngram_size": 0,
108
+ "eos_token_id": null,
109
+ "exponential_decay_length_penalty": null,
110
+ "finetuning_task": null,
111
+ "forced_bos_token_id": null,
112
+ "forced_eos_token_id": null,
113
+ "hidden_act": "quick_gelu",
114
+ "hidden_size": 1024,
115
+ "id2label": {
116
+ "0": "LABEL_0",
117
+ "1": "LABEL_1"
118
+ },
119
+ "image_size": 224,
120
+ "initializer_factor": 1.0,
121
+ "initializer_range": 0.02,
122
+ "intermediate_size": 4096,
123
+ "is_decoder": false,
124
+ "is_encoder_decoder": false,
125
+ "label2id": {
126
+ "LABEL_0": 0,
127
+ "LABEL_1": 1
128
+ },
129
+ "layer_norm_eps": 1e-05,
130
+ "length_penalty": 1.0,
131
+ "max_length": 20,
132
+ "min_length": 0,
133
+ "model_type": "clip_vision_model",
134
+ "no_repeat_ngram_size": 0,
135
+ "num_attention_heads": 16,
136
+ "num_beam_groups": 1,
137
+ "num_beams": 1,
138
+ "num_channels": 3,
139
+ "num_hidden_layers": 24,
140
+ "num_return_sequences": 1,
141
+ "output_attentions": false,
142
+ "output_hidden_states": false,
143
+ "output_scores": false,
144
+ "pad_token_id": null,
145
+ "patch_size": 14,
146
+ "prefix": null,
147
+ "problem_type": null,
148
+ "pruned_heads": {},
149
+ "remove_invalid_values": false,
150
+ "repetition_penalty": 1.0,
151
+ "return_dict": true,
152
+ "return_dict_in_generate": false,
153
+ "sep_token_id": null,
154
+ "task_specific_params": null,
155
+ "temperature": 1.0,
156
+ "tf_legacy_loss": false,
157
+ "tie_encoder_decoder": false,
158
+ "tie_word_embeddings": true,
159
+ "tokenizer_class": null,
160
+ "top_k": 50,
161
+ "top_p": 1.0,
162
+ "torch_dtype": null,
163
+ "torchscript": false,
164
+ "transformers_version": "4.22.0.dev0",
165
+ "typical_p": 1.0,
166
+ "use_bfloat16": false
167
+ },
168
+ "vision_config_dict": {
169
+ "hidden_size": 1024,
170
+ "intermediate_size": 4096,
171
+ "num_attention_heads": 16,
172
+ "num_hidden_layers": 24,
173
+ "patch_size": 14
174
+ }
175
+ }
models/StableDiffusion/stable-diffusion-v1-5/scheduler/scheduler_config.json ADDED
@@ -0,0 +1,13 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "_class_name": "PNDMScheduler",
3
+ "_diffusers_version": "0.6.0",
4
+ "beta_end": 0.012,
5
+ "beta_schedule": "scaled_linear",
6
+ "beta_start": 0.00085,
7
+ "num_train_timesteps": 1000,
8
+ "set_alpha_to_one": false,
9
+ "skip_prk_steps": true,
10
+ "steps_offset": 1,
11
+ "trained_betas": null,
12
+ "clip_sample": false
13
+ }
models/StableDiffusion/stable-diffusion-v1-5/text_encoder/config.json ADDED
@@ -0,0 +1,25 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "_name_or_path": "openai/clip-vit-large-patch14",
3
+ "architectures": [
4
+ "CLIPTextModel"
5
+ ],
6
+ "attention_dropout": 0.0,
7
+ "bos_token_id": 0,
8
+ "dropout": 0.0,
9
+ "eos_token_id": 2,
10
+ "hidden_act": "quick_gelu",
11
+ "hidden_size": 768,
12
+ "initializer_factor": 1.0,
13
+ "initializer_range": 0.02,
14
+ "intermediate_size": 3072,
15
+ "layer_norm_eps": 1e-05,
16
+ "max_position_embeddings": 77,
17
+ "model_type": "clip_text_model",
18
+ "num_attention_heads": 12,
19
+ "num_hidden_layers": 12,
20
+ "pad_token_id": 1,
21
+ "projection_dim": 768,
22
+ "torch_dtype": "float32",
23
+ "transformers_version": "4.22.0.dev0",
24
+ "vocab_size": 49408
25
+ }
models/StableDiffusion/stable-diffusion-v1-5/text_encoder/pytorch_model.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:770a47a9ffdcfda0b05506a7888ed714d06131d60267e6cf52765d61cf59fd67
3
+ size 492305335