xichenhku commited on
Commit
81d8e7c
1 Parent(s): 00f5da5

Upload 162 files

Browse files
This view is limited to 50 files because it contains too many changes.   See raw diff
Files changed (50) hide show
  1. .gitattributes +3 -0
  2. app.py +366 -0
  3. configs/inference.yaml +18 -0
  4. data_utils.py +531 -0
  5. demo_example/000_reference.png +3 -0
  6. demo_example/000_source.png +0 -0
  7. demo_example/001_reference.png +0 -0
  8. demo_example/001_source.png +0 -0
  9. demo_example/002_reference.png +0 -0
  10. demo_example/002_source.png +0 -0
  11. demo_example/003_reference.png +0 -0
  12. demo_example/003_source.png +0 -0
  13. demo_example/004_reference.png +0 -0
  14. demo_example/004_source.png +0 -0
  15. demo_example/005_reference.png +0 -0
  16. demo_example/005_source.png +3 -0
  17. demo_example/006_reference.png +0 -0
  18. demo_example/006_source.png +0 -0
  19. demo_example/007_reference.png +3 -0
  20. demo_example/007_source.png +0 -0
  21. depthanything/.DS_Store +0 -0
  22. depthanything/__pycache__/fast_import.cpython-38.pyc +0 -0
  23. depthanything/depth_anything/.DS_Store +0 -0
  24. depthanything/depth_anything/__pycache__/blocks.cpython-38.pyc +0 -0
  25. depthanything/depth_anything/__pycache__/dpt.cpython-38.pyc +0 -0
  26. depthanything/depth_anything/blocks.py +153 -0
  27. depthanything/depth_anything/dpt.py +189 -0
  28. depthanything/depth_anything/util/__pycache__/transform.cpython-38.pyc +0 -0
  29. depthanything/depth_anything/util/transform.py +248 -0
  30. depthanything/fast_import.py +13 -0
  31. depthanything/torchhub/README.md +3 -0
  32. depthanything/torchhub/facebookresearch_dinov2_main/CODE_OF_CONDUCT.md +80 -0
  33. depthanything/torchhub/facebookresearch_dinov2_main/CONTRIBUTING.md +31 -0
  34. depthanything/torchhub/facebookresearch_dinov2_main/LICENSE +400 -0
  35. depthanything/torchhub/facebookresearch_dinov2_main/MODEL_CARD.md +201 -0
  36. depthanything/torchhub/facebookresearch_dinov2_main/README.md +277 -0
  37. depthanything/torchhub/facebookresearch_dinov2_main/__pycache__/hubconf.cpython-38.pyc +0 -0
  38. depthanything/torchhub/facebookresearch_dinov2_main/__pycache__/vision_transformer.cpython-38.pyc +0 -0
  39. depthanything/torchhub/facebookresearch_dinov2_main/conda.yaml +22 -0
  40. depthanything/torchhub/facebookresearch_dinov2_main/dinov2/__init__.py +7 -0
  41. depthanything/torchhub/facebookresearch_dinov2_main/dinov2/__pycache__/__init__.cpython-38.pyc +0 -0
  42. depthanything/torchhub/facebookresearch_dinov2_main/dinov2/configs/__init__.py +23 -0
  43. depthanything/torchhub/facebookresearch_dinov2_main/dinov2/configs/eval/vitb14_pretrain.yaml +6 -0
  44. depthanything/torchhub/facebookresearch_dinov2_main/dinov2/configs/eval/vitg14_pretrain.yaml +7 -0
  45. depthanything/torchhub/facebookresearch_dinov2_main/dinov2/configs/eval/vitl14_pretrain.yaml +6 -0
  46. depthanything/torchhub/facebookresearch_dinov2_main/dinov2/configs/eval/vits14_pretrain.yaml +6 -0
  47. depthanything/torchhub/facebookresearch_dinov2_main/dinov2/configs/ssl_default_config.yaml +115 -0
  48. depthanything/torchhub/facebookresearch_dinov2_main/dinov2/configs/train/vitg14.yaml +26 -0
  49. depthanything/torchhub/facebookresearch_dinov2_main/dinov2/configs/train/vitl14.yaml +26 -0
  50. depthanything/torchhub/facebookresearch_dinov2_main/dinov2/configs/train/vitl16_short.yaml +6 -0
.gitattributes CHANGED
@@ -33,3 +33,6 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
33
  *.zip filter=lfs diff=lfs merge=lfs -text
34
  *.zst filter=lfs diff=lfs merge=lfs -text
35
  *tfevents* filter=lfs diff=lfs merge=lfs -text
 
 
 
 
33
  *.zip filter=lfs diff=lfs merge=lfs -text
34
  *.zst filter=lfs diff=lfs merge=lfs -text
35
  *tfevents* filter=lfs diff=lfs merge=lfs -text
36
+ demo_example/000_reference.png filter=lfs diff=lfs merge=lfs -text
37
+ demo_example/005_source.png filter=lfs diff=lfs merge=lfs -text
38
+ demo_example/007_reference.png filter=lfs diff=lfs merge=lfs -text
app.py ADDED
@@ -0,0 +1,366 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import gradio as gr
2
+ import torch
3
+ import torch.nn.functional as F
4
+ from safetensors.numpy import save_file, load_file
5
+ from omegaconf import OmegaConf
6
+ from transformers import AutoConfig
7
+ import cv2
8
+ from PIL import Image
9
+ import numpy as np
10
+ import json
11
+ import os
12
+ #
13
+ from diffusers import StableDiffusionPipeline, StableDiffusionImg2ImgPipeline, StableDiffusionInpaintPipelineLegacy, StableDiffusionInpaintPipeline, DDIMScheduler, AutoencoderKL
14
+ from diffusers import AutoencoderKL, DDPMScheduler, UNet2DConditionModel, DDIMScheduler
15
+ from diffusers import DDIMScheduler, DDPMScheduler, DPMSolverMultistepScheduler
16
+ from diffusers.image_processor import VaeImageProcessor
17
+ #
18
+ from models.pipeline_mimicbrush import MimicBrushPipeline
19
+ from models.ReferenceNet import ReferenceNet
20
+ from models.depth_guider import DepthGuider
21
+ from mimicbrush import MimicBrush_RefNet
22
+ from data_utils import *
23
+ from modelscope.hub.snapshot_download import snapshot_download as ms_snapshot_download
24
+ from huggingface_hub import snapshot_download
25
+
26
+
27
+
28
+ model_dir = ms_snapshot_download('xichen/MimicBrush', cache_dir='./weights', revision='v1.0.1')
29
+ snapshot_download(repo_id="runwayml/stable-diffusion-v1-5", local_dir="./stable-diffusion-v1-5")
30
+ snapshot_download(repo_id="runwayml/stable-diffusion-inpainting", local_dir="./stable-diffusion-inpainting")
31
+ val_configs = OmegaConf.load('./configs/inference.yaml')
32
+
33
+ # === import Depth Anything ===
34
+ import sys
35
+ sys.path.append("./depthanything")
36
+ from torchvision.transforms import Compose
37
+ from depthanything.fast_import import depth_anything_model
38
+ from depthanything.depth_anything.util.transform import Resize, NormalizeImage, PrepareForNet
39
+ transform = Compose([
40
+ Resize(
41
+ width=518,
42
+ height=518,
43
+ resize_target=False,
44
+ keep_aspect_ratio=True,
45
+ ensure_multiple_of=14,
46
+ resize_method='lower_bound',
47
+ image_interpolation_method=cv2.INTER_CUBIC,
48
+ ),
49
+ NormalizeImage(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]),
50
+ PrepareForNet(),
51
+ ])
52
+ depth_anything_model.load_state_dict(torch.load(val_configs.model_path.depth_model))
53
+
54
+
55
+
56
+ # === load the checkpoint ===
57
+ base_model_path = val_configs.model_path.pretrained_imitativer_path
58
+ vae_model_path = val_configs.model_path.pretrained_vae_name_or_path
59
+ image_encoder_path = val_configs.model_path.image_encoder_path
60
+ ref_model_path = val_configs.model_path.pretrained_reference_path
61
+ mimicbrush_ckpt = val_configs.model_path.mimicbrush_ckpt_path
62
+ device = "cuda"
63
+
64
+
65
+
66
+ def pad_img_to_square(original_image, is_mask=False):
67
+ width, height = original_image.size
68
+
69
+ if height == width:
70
+ return original_image
71
+
72
+ if height > width:
73
+ padding = (height - width) // 2
74
+ new_size = (height, height)
75
+ else:
76
+ padding = (width - height) // 2
77
+ new_size = (width, width)
78
+
79
+ if is_mask:
80
+ new_image = Image.new("RGB", new_size, "black")
81
+ else:
82
+ new_image = Image.new("RGB", new_size, "white")
83
+
84
+ if height > width:
85
+ new_image.paste(original_image, (padding, 0))
86
+ else:
87
+ new_image.paste(original_image, (0, padding))
88
+ return new_image
89
+
90
+
91
+ def collage_region(low, high, mask):
92
+ mask = (np.array(mask) > 128).astype(np.uint8)
93
+ low = np.array(low).astype(np.uint8)
94
+ low = (low * 0).astype(np.uint8)
95
+ high = np.array(high).astype(np.uint8)
96
+ mask_3 = mask
97
+ collage = low * mask_3 + high * (1-mask_3)
98
+ collage = Image.fromarray(collage)
99
+ return collage
100
+
101
+
102
+ def resize_image_keep_aspect_ratio(image, target_size = 512):
103
+ height, width = image.shape[:2]
104
+ if height > width:
105
+ new_height = target_size
106
+ new_width = int(width * (target_size / height))
107
+ else:
108
+ new_width = target_size
109
+ new_height = int(height * (target_size / width))
110
+ resized_image = cv2.resize(image, (new_width, new_height))
111
+ return resized_image
112
+
113
+
114
+ def crop_padding_and_resize(ori_image, square_image):
115
+ ori_height, ori_width, _ = ori_image.shape
116
+ scale = max(ori_height / square_image.shape[0], ori_width / square_image.shape[1])
117
+ resized_square_image = cv2.resize(square_image, (int(square_image.shape[1] * scale), int(square_image.shape[0] * scale)))
118
+ padding_size = max(resized_square_image.shape[0] - ori_height, resized_square_image.shape[1] - ori_width)
119
+ if ori_height < ori_width:
120
+ top = padding_size // 2
121
+ bottom = resized_square_image.shape[0] - (padding_size - top)
122
+ cropped_image = resized_square_image[top:bottom, :,:]
123
+ else:
124
+ left = padding_size // 2
125
+ right = resized_square_image.shape[1] - (padding_size - left)
126
+ cropped_image = resized_square_image[:, left:right,:]
127
+ return cropped_image
128
+
129
+
130
+ def vis_mask(image, mask):
131
+ # mask 3 channle 255
132
+ mask = mask[:,:,0]
133
+ mask_contours, _ = cv2.findContours(mask, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE)
134
+
135
+ # Draw outlines, using random colors
136
+ outline_opacity = 0.5
137
+ outline_thickness = 5
138
+ outline_color = np.concatenate([ [255,255,255], [outline_opacity] ])
139
+
140
+ white_mask = np.ones_like(image) * 255
141
+
142
+ mask_bin_3 = np.stack([mask,mask,mask],-1) > 128
143
+ alpha = 0.5
144
+ image = ( white_mask * alpha + image * (1-alpha) ) * mask_bin_3 + image * (1-mask_bin_3)
145
+ cv2.polylines(image, mask_contours, True, outline_color, outline_thickness, cv2.LINE_AA)
146
+ return image
147
+
148
+
149
+
150
+ noise_scheduler = DDIMScheduler(
151
+ num_train_timesteps=1000,
152
+ beta_start=0.00085,
153
+ beta_end=0.012,
154
+ beta_schedule="scaled_linear",
155
+ clip_sample=False,
156
+ set_alpha_to_one=False,
157
+ steps_offset=1,
158
+ )
159
+
160
+ vae = AutoencoderKL.from_pretrained(vae_model_path).to(dtype=torch.float16)
161
+ unet = UNet2DConditionModel.from_pretrained(base_model_path, subfolder="unet", in_channels=13, low_cpu_mem_usage=False, ignore_mismatched_sizes=True).to(dtype=torch.float16)
162
+
163
+ pipe = MimicBrushPipeline.from_pretrained(
164
+ base_model_path,
165
+ torch_dtype=torch.float16,
166
+ scheduler=noise_scheduler,
167
+ vae=vae,
168
+ unet=unet,
169
+ feature_extractor=None,
170
+ safety_checker=None,
171
+ )
172
+
173
+ depth_guider = DepthGuider()
174
+ referencenet = ReferenceNet.from_pretrained(ref_model_path, subfolder="unet").to(dtype=torch.float16)
175
+ mimicbrush_model = MimicBrush_RefNet(pipe, image_encoder_path, mimicbrush_ckpt, depth_anything_model, depth_guider, referencenet, device)
176
+ mask_processor = VaeImageProcessor(vae_scale_factor=1, do_normalize=False, do_binarize=True, do_convert_grayscale=True)
177
+
178
+
179
+ def infer_single(ref_image, target_image, target_mask, seed = -1, num_inference_steps=50, guidance_scale = 5, enable_shape_control = False):
180
+ #return ref_image
181
+ """
182
+ mask: 0/1 1-channel np.array
183
+ image: rgb np.array
184
+ """
185
+ ref_image = ref_image.astype(np.uint8)
186
+ target_image = target_image.astype(np.uint8)
187
+ target_mask = target_mask .astype(np.uint8)
188
+
189
+ ref_image = Image.fromarray(ref_image.astype(np.uint8))
190
+ ref_image = pad_img_to_square(ref_image)
191
+
192
+ target_image = pad_img_to_square(Image.fromarray(target_image))
193
+ target_image_low = target_image
194
+
195
+
196
+ target_mask = np.stack([target_mask,target_mask,target_mask],-1).astype(np.uint8) * 255
197
+ target_mask_np = target_mask.copy()
198
+ target_mask = Image.fromarray(target_mask)
199
+ target_mask = pad_img_to_square(target_mask, True)
200
+
201
+ target_image_ori = target_image.copy()
202
+ target_image = collage_region(target_image_low, target_image, target_mask)
203
+
204
+
205
+ depth_image = target_image_ori.copy()
206
+ depth_image = np.array(depth_image)
207
+ depth_image = transform({'image': depth_image})['image']
208
+ depth_image = torch.from_numpy(depth_image).unsqueeze(0) / 255
209
+
210
+ if not enable_shape_control:
211
+ depth_image = depth_image * 0
212
+
213
+ mask_pt = mask_processor.preprocess(target_mask, height=512, width=512)
214
+
215
+ pred, depth_pred = mimicbrush_model.generate(pil_image=ref_image, depth_image = depth_image, num_samples=1, num_inference_steps=num_inference_steps,
216
+ seed=seed, image=target_image, mask_image=mask_pt, strength=1.0, guidance_scale=guidance_scale)
217
+
218
+
219
+ depth_pred = F.interpolate(depth_pred, size=(512,512), mode = 'bilinear', align_corners=True)[0][0]
220
+ depth_pred = (depth_pred - depth_pred.min()) / (depth_pred.max() - depth_pred.min()) * 255.0
221
+ depth_pred = depth_pred.detach().cpu().numpy().astype(np.uint8)
222
+ depth_pred = cv2.applyColorMap(depth_pred, cv2.COLORMAP_INFERNO)[:,:,::-1]
223
+
224
+ pred = pred[0]
225
+ pred = np.array(pred).astype(np.uint8)
226
+ return pred, depth_pred.astype(np.uint8)
227
+
228
+
229
+ def inference_single_image(ref_image,
230
+ tar_image,
231
+ tar_mask,
232
+ ddim_steps,
233
+ scale,
234
+ seed,
235
+ enable_shape_control,
236
+ ):
237
+ if seed == -1:
238
+ seed = np.random.randint(10000)
239
+ pred, depth_pred = infer_single(ref_image, tar_image, tar_mask, seed, num_inference_steps=ddim_steps, guidance_scale = scale, enable_shape_control = enable_shape_control)
240
+ return pred, depth_pred
241
+
242
+
243
+
244
+ def run_local(base,
245
+ ref,
246
+ *args):
247
+ image = base["image"].convert("RGB")
248
+ mask = base["mask"].convert("L")
249
+ image = np.asarray(image)
250
+ mask = np.asarray(mask)
251
+ mask = np.where(mask > 128, 1, 0).astype(np.uint8)
252
+
253
+
254
+ ref_image = ref.convert("RGB")
255
+ ref_image = np.asarray(ref_image)
256
+
257
+ if mask.sum() == 0:
258
+ raise gr.Error('No mask for the background image.')
259
+
260
+ mask_3 = np.stack([mask,mask,mask],-1).astype(np.uint8) * 255
261
+
262
+ mask_alpha = mask_3.copy()
263
+ for i in range(10):
264
+ mask_alpha = cv2.GaussianBlur(mask_alpha, (3, 3), 0)
265
+
266
+ synthesis, depth_pred = inference_single_image(ref_image.copy(), image.copy(), mask.copy(), *args)
267
+
268
+
269
+ synthesis = crop_padding_and_resize(image, synthesis)
270
+ depth_pred = crop_padding_and_resize(image, depth_pred)
271
+
272
+
273
+ mask_3_bin = mask_alpha / 255
274
+ synthesis = synthesis * mask_3_bin + image * (1-mask_3_bin)
275
+
276
+ vis_source = vis_mask(image, mask_3).astype(np.uint8)
277
+ return [synthesis.astype(np.uint8), depth_pred.astype(np.uint8), vis_source, mask_3]
278
+
279
+
280
+
281
+ with gr.Blocks() as demo:
282
+ with gr.Column():
283
+ gr.Markdown("# MimicBrush: Zero-shot Image Editing with Reference Imitation ")
284
+ with gr.Row():
285
+ baseline_gallery = gr.Gallery(label='Output', show_label=True, elem_id="gallery", columns=1, height=768)
286
+ with gr.Accordion("Advanced Option", open=True):
287
+ num_samples = 1
288
+ ddim_steps = gr.Slider(label="Steps", minimum=1, maximum=100, value=50, step=1)
289
+ scale = gr.Slider(label="Guidance Scale", minimum=-30.0, maximum=30.0, value=5.0, step=0.1)
290
+ seed = gr.Slider(label="Seed", minimum=-1, maximum=999999999, step=1, value=-1)
291
+ enable_shape_control = gr.Checkbox(label='Keep the original shape', value=False, interactive = True)
292
+
293
+ gr.Markdown("### Tutorial")
294
+ gr.Markdown("1. Upload the source image and the reference image")
295
+ gr.Markdown("2. Mask the to-edit region on the source image ")
296
+ gr.Markdown("3. Click generate ")
297
+ gr.Markdown("#### You shoud click \"keep the original shape\" to conduct texture transfer ")
298
+
299
+
300
+ gr.Markdown("# Upload the source image and reference image")
301
+ gr.Markdown("### Tips: you could adjust the brush size by at the top right")
302
+
303
+ with gr.Row():
304
+ base = gr.Image(label="Background", source="upload", tool="sketch", type="pil", height=512, brush_color='#FFFFFF', mask_opacity=0.5, brush_radius = 100)
305
+ ref = gr.Image(label="Reference", source="upload", type="pil", height=512 )
306
+ run_local_button = gr.Button(label="Generate", value="Run")
307
+
308
+
309
+
310
+ with gr.Row():
311
+ gr.Examples(
312
+ examples=[
313
+ [
314
+ './demo_example/005_source.png',
315
+ './demo_example/005_reference.png',
316
+ ],
317
+ [
318
+ './demo_example/000_source.png',
319
+ './demo_example/000_reference.png',
320
+ ],
321
+ [
322
+ './demo_example/001_source.png',
323
+ './demo_example/001_reference.png',
324
+ ],
325
+ [
326
+ './demo_example/002_source.png',
327
+ './demo_example/002_reference.png',
328
+ ],
329
+ [
330
+ './demo_example/003_source.png',
331
+ './demo_example/003_reference.png',
332
+ ],
333
+ [
334
+ './demo_example/004_source.png',
335
+ './demo_example/004_reference.png',
336
+ ],
337
+ [
338
+ './demo_example/006_source.png',
339
+ './demo_example/006_reference.png',
340
+ ],
341
+ [
342
+ './demo_example/007_source.png',
343
+ './demo_example/007_reference.png',
344
+ ],
345
+ ],
346
+
347
+ inputs=[
348
+ base,
349
+ ref
350
+ ],
351
+ cache_examples=False,
352
+ examples_per_page=100)
353
+
354
+
355
+ run_local_button.click(fn=run_local,
356
+ inputs=[base,
357
+ ref,
358
+ ddim_steps,
359
+ scale,
360
+ seed,
361
+ enable_shape_control
362
+ ],
363
+ outputs=[baseline_gallery]
364
+ )
365
+
366
+ demo.launch(server_name="0.0.0.0")
configs/inference.yaml ADDED
@@ -0,0 +1,18 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ model_path:
2
+ mimicbrush_ckpt_path: ./weights/mimicbrush/mimicbrush.bin
3
+ pretrained_reference_path: ./stable-diffusion-v1-5
4
+ pretrained_imitativer_path: ./stable-diffusion-inpainting
5
+ pretrained_vae_name_or_path: ./weights/sd-vae-ft-mse
6
+ image_encoder_path: ./weights/image_encoder
7
+ depth_model: ./weights/depth_model/depth_anything_vitb14.pth
8
+ data_path:
9
+ test_set_name: Texture_Within # choose from [ PartComp_Cross, PartComp_Within, Texture_Cross, Texture_Within ]
10
+ output_dir: ./test-vis
11
+ bench_dir_partcomp_cross: /mnt/myworkspace/new_project/MimicBrush_bench/PartComp/CrossInstance
12
+ bench_dir_partcomp_within: /mnt/myworkspace/new_project/MimicBrush_bench/PartComp/WithinInstance
13
+ bench_dir_texture_cross: /mnt/myworkspace/new_project/MimicBrush_bench/TextureTrans/CrossInstance
14
+ bench_dir_texture_within: /mnt/myworkspace/new_project/MimicBrush_bench/TextureTrans/WithinInstance
15
+
16
+
17
+
18
+
data_utils.py ADDED
@@ -0,0 +1,531 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import numpy as np
2
+ import torch
3
+ import cv2
4
+ import random
5
+ from PIL import Image
6
+
7
+
8
+ def gaussian_blure(img, intens = 5):
9
+ """
10
+ 高斯模糊
11
+ :param image_path:
12
+ :intens 5,10,15,20
13
+ :return:
14
+ """
15
+ img = np.array(img).astype(np.uint8)
16
+ result = cv2.GaussianBlur(img, (0, 0), intens)
17
+ result = Image.fromarray(result)
18
+ return result
19
+
20
+ def random_mask(mask):
21
+ h,w = mask.shape[0], mask.shape[1]
22
+ mask_black = np.zeros_like(mask)
23
+ box_w = random.uniform(0.4, 0.9) * w
24
+ box_h = random.uniform(0.4, 0.9) * h
25
+ box_w = int(box_w)
26
+ box_h = int(box_h)
27
+ y1 = random.randint(0, h - box_h)
28
+ y2 = y1 + box_h
29
+ x1 = random.randint(0, w - box_w)
30
+ x2 = x1 + box_w
31
+ mask_black[y1:y2,x1:x2] = 1
32
+ mask_black = mask_black.astype(np.uint8)
33
+ return mask_black
34
+
35
+ '''
36
+ def random_mask_grid(mask, p=0.50):
37
+ # 创建一个 h x w 的全零数组,作为初始掩膜
38
+ h,w = mask.shape[0],mask.shape[1]
39
+ mask = np.zeros((h, w), dtype=np.uint8)
40
+ n = random.choice([3,4,5,6,7,8,9,10])
41
+
42
+ # 计算小块的大小
43
+ block_h = h // n
44
+ block_w = w // n
45
+
46
+ # 在每个小块中以概率 p 设置为 1
47
+ for i in range(n):
48
+ for j in range(n):
49
+ if np.random.rand() < p:
50
+ mask[i*block_h:(i+1)*block_h, j*block_w:(j+1)*block_w] = 1
51
+ return mask
52
+ '''
53
+
54
+ def get_SIFT(image):
55
+ orb = cv2.ORB_create(nfeatures=200, edgeThreshold=50)
56
+ keypoint, descriptor = orb.detectAndCompute(image, None)
57
+ coordinates = [(int(kp.pt[1]), int(kp.pt[0])) for kp in keypoint]
58
+ return coordinates
59
+
60
+
61
+ '''
62
+ def random_mask_grid(mask, points_list, p=0.0):
63
+ # 创建一个 h x w 的全零数组,作为初始掩膜
64
+ h, w = mask.shape[:2]
65
+ mask = np.zeros((h, w), dtype=np.uint8)
66
+ n = random.choice([3,4,5,6,7,8,9,10])
67
+
68
+ # 计算小块的大小
69
+ block_h = h // n
70
+ block_w = w // n
71
+
72
+ # 统计每个小块内的点个数
73
+ block_counts = np.zeros((n, n), dtype=np.int32)
74
+ for point in points_list:
75
+ y, x = point
76
+ i = min(y // block_h, n-1)
77
+ j = min(x // block_w, n-1)
78
+ block_counts[i, j] += 1
79
+
80
+ # 找出包含点最多的前5个小块
81
+ top5_blocks = np.argpartition(-block_counts.flatten(), 5)[:5]
82
+
83
+ # 将这些小块对应的像素设为1
84
+ for idx in top5_blocks:
85
+ i, j = divmod(idx, n)
86
+ mask[i*block_h:(i+1)*block_h, j*block_w:(j+1)*block_w] = 1
87
+
88
+ # 在其他小块中按照概率p设置为1
89
+ for i in range(n):
90
+ for j in range(n):
91
+ if (i*n + j) not in top5_blocks and np.random.rand() < p:
92
+ mask[i*block_h:(i+1)*block_h, j*block_w:(j+1)*block_w] = 1
93
+
94
+ return mask
95
+ '''
96
+
97
+ def random_mask_grid(mask, points_list, p=0.50, top5_p=0.70, other_p=0.30):
98
+ # 创建一个 h x w 的全零数组,作为初始掩膜
99
+ h, w = mask.shape[:2]
100
+ mask = np.zeros((h, w), dtype=np.uint8)
101
+ n = random.choice([3,4,5,6,7,8,9,10])
102
+
103
+ # 计算小块的大小
104
+ block_h = h // n
105
+ block_w = w // n
106
+
107
+ # 统计每个小块内的点个数
108
+ block_counts = np.zeros((n, n), dtype=np.int32)
109
+ for point in points_list:
110
+ y, x = point
111
+ i = min(y // block_h, n-1)
112
+ j = min(x // block_w, n-1)
113
+ block_counts[i, j] += 1
114
+
115
+ # 找出包含点最多的前5个小块
116
+ top5_blocks = np.argpartition(-block_counts.flatten(), 5)[:5]
117
+
118
+ # 将这些小块对应的像素设为1
119
+ for idx in top5_blocks:
120
+ i, j = divmod(idx, n)
121
+ if np.random.rand() < top5_p:
122
+ mask[i*block_h:(i+1)*block_h, j*block_w:(j+1)*block_w] = 1
123
+
124
+ # 在其他小块中按照概率p设置为1
125
+ for i in range(n):
126
+ for j in range(n):
127
+ if (i*n + j) not in top5_blocks and np.random.rand() < other_p:
128
+ mask[i*block_h:(i+1)*block_h, j*block_w:(j+1)*block_w] = 1
129
+
130
+ return mask
131
+
132
+ def random_perspective_transform(image, intensity):
133
+ """
134
+ 对图像进行随机透视变换
135
+
136
+ 参数:
137
+ image: 要进行变换的输入图像
138
+ intensity: 变换的强度,范围从0到1,值越大,变换越明显
139
+
140
+ 返回值:
141
+ 变换后的图像
142
+ """
143
+ height, width = image.shape[:2]
144
+
145
+ # 生成随机透视变换的四个目标点
146
+ x_offset = width * 0.4 * intensity
147
+ y_offset = height * 0.4 * intensity
148
+ dst_points = np.float32([[random.uniform(-x_offset, x_offset), random.uniform(-y_offset, y_offset)],
149
+ [width - random.uniform(-x_offset, x_offset), random.uniform(-y_offset, y_offset)],
150
+ [random.uniform(-x_offset, x_offset), height - random.uniform(-y_offset, y_offset)],
151
+ [width - random.uniform(-x_offset, x_offset), height - random.uniform(-y_offset, y_offset)]])
152
+
153
+ # 对应的源点是图像的四个角
154
+ src_points = np.float32([[0, 0], [width, 0], [0, height], [width, height]])
155
+
156
+ # 生成透视变换矩阵
157
+ M = cv2.getPerspectiveTransform(src_points, dst_points)
158
+
159
+ # 进行透视变换
160
+ transformed_image = cv2.warpPerspective(image, M, (width, height))
161
+ mask = np.ones_like(transformed_image)
162
+ transformed_mask = cv2.warpPerspective(mask, M, (width, height))> 0.5
163
+
164
+ kernel_size = 5
165
+ kernel = np.ones((kernel_size, kernel_size), np.uint8)
166
+ transformed_mask = cv2.erode(transformed_mask.astype(np.uint8), kernel, iterations=1).astype(np.uint8)
167
+
168
+ white_back = np.ones_like(transformed_image) * 255
169
+ transformed_image = transformed_image * transformed_mask + white_back * (1-transformed_mask)
170
+ return transformed_image
171
+
172
+
173
+
174
+
175
+ def mask_score(mask):
176
+ '''Scoring the mask according to connectivity.'''
177
+ mask = mask.astype(np.uint8)
178
+ if mask.sum() < 10:
179
+ return 0
180
+ contours, _ = cv2.findContours(mask, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_NONE)
181
+ cnt_area = [cv2.contourArea(cnt) for cnt in contours]
182
+ conc_score = np.max(cnt_area) / sum(cnt_area)
183
+ return conc_score
184
+
185
+
186
+ def sobel(img, mask, thresh = 50):
187
+ '''Calculating the high-frequency map.'''
188
+ H,W = img.shape[0], img.shape[1]
189
+ img = cv2.resize(img,(256,256))
190
+ mask = (cv2.resize(mask,(256,256)) > 0.5).astype(np.uint8)
191
+ kernel = np.ones((5,5),np.uint8)
192
+ mask = cv2.erode(mask, kernel, iterations = 2)
193
+
194
+ Ksize = 3
195
+ sobelx = cv2.Sobel(img, cv2.CV_64F, 1, 0, ksize=Ksize)
196
+ sobely = cv2.Sobel(img, cv2.CV_64F, 0, 1, ksize=Ksize)
197
+ sobel_X = cv2.convertScaleAbs(sobelx)
198
+ sobel_Y = cv2.convertScaleAbs(sobely)
199
+ scharr = cv2.addWeighted(sobel_X, 0.5, sobel_Y, 0.5, 0)
200
+ scharr = np.max(scharr,-1) * mask
201
+
202
+ scharr[scharr < thresh] = 0.0
203
+ scharr = np.stack([scharr,scharr,scharr],-1)
204
+ scharr = (scharr.astype(np.float32)/255 * img.astype(np.float32) ).astype(np.uint8)
205
+ scharr = cv2.resize(scharr,(W,H))
206
+ return scharr
207
+
208
+
209
+ def resize_and_pad(image, box):
210
+ '''Fitting an image to the box region while keeping the aspect ratio.'''
211
+ y1,y2,x1,x2 = box
212
+ H,W = y2-y1, x2-x1
213
+ h,w = image.shape[0], image.shape[1]
214
+ r_box = W / H
215
+ r_image = w / h
216
+ if r_box >= r_image:
217
+ h_target = H
218
+ w_target = int(w * H / h)
219
+ image = cv2.resize(image, (w_target, h_target))
220
+
221
+ w1 = (W - w_target) // 2
222
+ w2 = W - w_target - w1
223
+ pad_param = ((0,0),(w1,w2),(0,0))
224
+ image = np.pad(image, pad_param, 'constant', constant_values=255)
225
+ else:
226
+ w_target = W
227
+ h_target = int(h * W / w)
228
+ image = cv2.resize(image, (w_target, h_target))
229
+
230
+ h1 = (H-h_target) // 2
231
+ h2 = H - h_target - h1
232
+ pad_param =((h1,h2),(0,0),(0,0))
233
+ image = np.pad(image, pad_param, 'constant', constant_values=255)
234
+ return image
235
+
236
+
237
+
238
+ def expand_image_mask(image, mask, ratio=1.4, random = False):
239
+ # expand image and mask
240
+ # pad image with 255
241
+ # pad mask with 0
242
+ h,w = image.shape[0], image.shape[1]
243
+ H,W = int(h * ratio), int(w * ratio)
244
+ if random:
245
+ h1 = np.random.randint(0, int(H - h))
246
+ w1 = np.random.randint(0, int(W - w))
247
+ else:
248
+ h1 = int((H - h) // 2)
249
+ w1 = int((W -w) // 2)
250
+ h2 = H - h - h1
251
+ w2 = W -w - w1
252
+ pad_param_image = ((h1,h2),(w1,w2),(0,0))
253
+ pad_param_mask = ((h1,h2),(w1,w2))
254
+ image = np.pad(image, pad_param_image, 'constant', constant_values=255)
255
+ mask = np.pad(mask, pad_param_mask, 'constant', constant_values=0)
256
+ return image, mask
257
+
258
+
259
+ def resize_box(yyxx, H,W,h,w):
260
+ y1,y2,x1,x2 = yyxx
261
+ y1,y2 = int(y1/H * h), int(y2/H * h)
262
+ x1,x2 = int(x1/W * w), int(x2/W * w)
263
+ y1,y2 = min(y1,h), min(y2,h)
264
+ x1,x2 = min(x1,w), min(x2,w)
265
+ return (y1,y2,x1,x2)
266
+
267
+
268
+ def get_bbox_from_mask(mask):
269
+ h,w = mask.shape[0],mask.shape[1]
270
+
271
+ if mask.sum() < 10:
272
+ return 0,h,0,w
273
+ rows = np.any(mask,axis=1)
274
+ cols = np.any(mask,axis=0)
275
+ y1,y2 = np.where(rows)[0][[0,-1]]
276
+ x1,x2 = np.where(cols)[0][[0,-1]]
277
+ return (y1,y2,x1,x2)
278
+
279
+
280
+ def expand_bbox(mask,yyxx,ratio=[1.2,2.0], min_crop=0):
281
+ y1,y2,x1,x2 = yyxx
282
+ ratio = np.random.randint( ratio[0] * 10, ratio[1] * 10 ) / 10
283
+ H,W = mask.shape[0], mask.shape[1]
284
+ xc, yc = 0.5 * (x1 + x2), 0.5 * (y1 + y2)
285
+ h = ratio * (y2-y1+1)
286
+ w = ratio * (x2-x1+1)
287
+ h = max(h,min_crop)
288
+ w = max(w,min_crop)
289
+
290
+ x1 = int(xc - w * 0.5)
291
+ x2 = int(xc + w * 0.5)
292
+ y1 = int(yc - h * 0.5)
293
+ y2 = int(yc + h * 0.5)
294
+
295
+ x1 = max(0,x1)
296
+ x2 = min(W,x2)
297
+ y1 = max(0,y1)
298
+ y2 = min(H,y2)
299
+ return (y1,y2,x1,x2)
300
+
301
+
302
+ def box2squre(image, box):
303
+ H,W = image.shape[0], image.shape[1]
304
+ y1,y2,x1,x2 = box
305
+ cx = (x1 + x2) // 2
306
+ cy = (y1 + y2) // 2
307
+ h,w = y2-y1, x2-x1
308
+
309
+ if h >= w:
310
+ x1 = cx - h//2
311
+ x2 = cx + h//2
312
+ else:
313
+ y1 = cy - w//2
314
+ y2 = cy + w//2
315
+ x1 = max(0,x1)
316
+ x2 = min(W,x2)
317
+ y1 = max(0,y1)
318
+ y2 = min(H,y2)
319
+ return (y1,y2,x1,x2)
320
+
321
+
322
+ def pad_to_square(image, pad_value = 255, random = False):
323
+ H,W = image.shape[0], image.shape[1]
324
+ if H == W:
325
+ return image
326
+
327
+ padd = abs(H - W)
328
+ if random:
329
+ padd_1 = int(np.random.randint(0,padd))
330
+ else:
331
+ padd_1 = int(padd / 2)
332
+ padd_2 = padd - padd_1
333
+
334
+ if H > W:
335
+ pad_param = ((0,0),(padd_1,padd_2),(0,0))
336
+ else:
337
+ pad_param = ((padd_1,padd_2),(0,0),(0,0))
338
+
339
+ image = np.pad(image, pad_param, 'constant', constant_values=pad_value)
340
+ return image
341
+
342
+
343
+
344
+ def box_in_box(small_box, big_box):
345
+ y1,y2,x1,x2 = small_box
346
+ y1_b, _, x1_b, _ = big_box
347
+ y1,y2,x1,x2 = y1 - y1_b ,y2 - y1_b, x1 - x1_b ,x2 - x1_b
348
+ return (y1,y2,x1,x2 )
349
+
350
+
351
+
352
+ def shuffle_image(image, N):
353
+ height, width = image.shape[:2]
354
+
355
+ block_height = height // N
356
+ block_width = width // N
357
+ blocks = []
358
+
359
+ for i in range(N):
360
+ for j in range(N):
361
+ block = image[i*block_height:(i+1)*block_height, j*block_width:(j+1)*block_width]
362
+ blocks.append(block)
363
+
364
+ np.random.shuffle(blocks)
365
+ shuffled_image = np.zeros((height, width, 3), dtype=np.uint8)
366
+
367
+ for i in range(N):
368
+ for j in range(N):
369
+ shuffled_image[i*block_height:(i+1)*block_height, j*block_width:(j+1)*block_width] = blocks[i*N+j]
370
+ return shuffled_image
371
+
372
+
373
+ def get_mosaic_mask(image, fg_mask, N=16, ratio = 0.5):
374
+ ids = [i for i in range(N * N)]
375
+ masked_number = int(N * N * ratio)
376
+ masked_id = np.random.choice(ids, masked_number, replace=False)
377
+
378
+
379
+
380
+ height, width = image.shape[:2]
381
+ mask = np.ones((height, width))
382
+
383
+ block_height = height // N
384
+ block_width = width // N
385
+
386
+ b_id = 0
387
+ for i in range(N):
388
+ for j in range(N):
389
+ if b_id in masked_id:
390
+ mask[i*block_height:(i+1)*block_height, j*block_width:(j+1)*block_width] = mask[i*block_height:(i+1)*block_height, j*block_width:(j+1)*block_width] * 0
391
+ b_id += 1
392
+ mask = mask * fg_mask
393
+ mask3 = np.stack([mask,mask,mask],-1).copy().astype(np.uint8)
394
+ noise = q_x(image)
395
+ noise_mask = image * mask3 + noise * (1-mask3)
396
+ return noise_mask
397
+
398
+ def extract_canney_noise(image, mask, dilate=True):
399
+ h,w = image.shape[0],image.shape[1]
400
+ mask = cv2.resize(mask.astype(np.uint8),(w,h)) > 0.5
401
+ kernel = np.ones((8, 8), dtype=np.uint8)
402
+ mask = cv2.erode(mask.astype(np.uint8), kernel, 10)
403
+
404
+ canny = cv2.Canny(image, 50,100) * mask
405
+ kernel = np.ones((8, 8), dtype=np.uint8)
406
+ mask = (cv2.dilate(canny, kernel, 5) > 128).astype(np.uint8)
407
+ mask = np.stack([mask,mask,mask],-1)
408
+
409
+ pure_noise = q_x(image, t=1) * 0 + 255
410
+ canny_noise = mask * image + (1-mask) * pure_noise
411
+ return canny_noise
412
+
413
+
414
+ def get_random_structure(size):
415
+ choice = np.random.randint(1, 5)
416
+
417
+ if choice == 1:
418
+ return cv2.getStructuringElement(cv2.MORPH_RECT, (size, size))
419
+ elif choice == 2:
420
+ return cv2.getStructuringElement(cv2.MORPH_ELLIPSE, (size, size))
421
+ elif choice == 3:
422
+ return cv2.getStructuringElement(cv2.MORPH_ELLIPSE, (size, size//2))
423
+ elif choice == 4:
424
+ return cv2.getStructuringElement(cv2.MORPH_ELLIPSE, (size//2, size))
425
+
426
+ def random_dilate(seg, min=3, max=10):
427
+ size = np.random.randint(min, max)
428
+ kernel = get_random_structure(size)
429
+ seg = cv2.dilate(seg,kernel,iterations = 1)
430
+ return seg
431
+
432
+ def random_erode(seg, min=3, max=10):
433
+ size = np.random.randint(min, max)
434
+ kernel = get_random_structure(size)
435
+ seg = cv2.erode(seg,kernel,iterations = 1)
436
+ return seg
437
+
438
+ def compute_iou(seg, gt):
439
+ intersection = seg*gt
440
+ union = seg+gt
441
+ return (np.count_nonzero(intersection) + 1e-6) / (np.count_nonzero(union) + 1e-6)
442
+
443
+
444
+ def select_max_region(mask):
445
+ nums, labels, stats, centroids = cv2.connectedComponentsWithStats(mask, connectivity=8)
446
+ background = 0
447
+ for row in range(stats.shape[0]):
448
+ if stats[row, :][0] == 0 and stats[row, :][1] == 0:
449
+ background = row
450
+ stats_no_bg = np.delete(stats, background, axis=0)
451
+ max_idx = stats_no_bg[:, 4].argmax()
452
+ max_region = np.where(labels==max_idx+1, 1, 0)
453
+
454
+ return max_region.astype(np.uint8)
455
+
456
+
457
+
458
+ def perturb_mask(gt, min_iou = 0.3, max_iou = 0.99):
459
+ iou_target = np.random.uniform(min_iou, max_iou)
460
+ h, w = gt.shape
461
+ gt = gt.astype(np.uint8)
462
+ seg = gt.copy()
463
+
464
+ # Rare case
465
+ if h <= 2 or w <= 2:
466
+ print('GT too small, returning original')
467
+ return seg
468
+
469
+ # Do a bunch of random operations
470
+ for _ in range(250):
471
+ for _ in range(4):
472
+ lx, ly = np.random.randint(w), np.random.randint(h)
473
+ lw, lh = np.random.randint(lx+1,w+1), np.random.randint(ly+1,h+1)
474
+
475
+ # Randomly set one pixel to 1/0. With the following dilate/erode, we can create holes/external regions
476
+ if np.random.rand() < 0.1:
477
+ cx = int((lx + lw) / 2)
478
+ cy = int((ly + lh) / 2)
479
+ seg[cy, cx] = np.random.randint(2) * 255
480
+
481
+ # Dilate/erode
482
+ if np.random.rand() < 0.5:
483
+ seg[ly:lh, lx:lw] = random_dilate(seg[ly:lh, lx:lw])
484
+ else:
485
+ seg[ly:lh, lx:lw] = random_erode(seg[ly:lh, lx:lw])
486
+
487
+ seg = np.logical_or(seg, gt).astype(np.uint8)
488
+ #seg = select_max_region(seg)
489
+
490
+ if compute_iou(seg, gt) < iou_target:
491
+ break
492
+ seg = select_max_region(seg.astype(np.uint8))
493
+ return seg.astype(np.uint8)
494
+
495
+
496
+ def q_x(x_0,t=65):
497
+ '''Adding noise for and given image.'''
498
+ x_0 = torch.from_numpy(x_0).float() / 127.5 - 1
499
+ num_steps = 100
500
+
501
+ betas = torch.linspace(-6,6,num_steps)
502
+ betas = torch.sigmoid(betas)*(0.5e-2 - 1e-5)+1e-5
503
+
504
+ alphas = 1-betas
505
+ alphas_prod = torch.cumprod(alphas,0)
506
+
507
+ alphas_prod_p = torch.cat([torch.tensor([1]).float(),alphas_prod[:-1]],0)
508
+ alphas_bar_sqrt = torch.sqrt(alphas_prod)
509
+ one_minus_alphas_bar_log = torch.log(1 - alphas_prod)
510
+ one_minus_alphas_bar_sqrt = torch.sqrt(1 - alphas_prod)
511
+
512
+ noise = torch.randn_like(x_0)
513
+ alphas_t = alphas_bar_sqrt[t]
514
+ alphas_1_m_t = one_minus_alphas_bar_sqrt[t]
515
+ return (alphas_t * x_0 + alphas_1_m_t * noise).numpy() * 127.5 + 127.5
516
+
517
+
518
+ def extract_target_boundary(img, target_mask):
519
+ Ksize = 3
520
+ sobelx = cv2.Sobel(img, cv2.CV_64F, 1, 0, ksize=Ksize)
521
+ sobely = cv2.Sobel(img, cv2.CV_64F, 0, 1, ksize=Ksize)
522
+
523
+ # sobel-x
524
+ sobel_X = cv2.convertScaleAbs(sobelx)
525
+ # sobel-y
526
+ sobel_Y = cv2.convertScaleAbs(sobely)
527
+ # sobel-xy
528
+ scharr = cv2.addWeighted(sobel_X, 0.5, sobel_Y, 0.5, 0)
529
+ scharr = np.max(scharr,-1).astype(np.float32)/255
530
+ scharr = scharr * target_mask.astype(np.float32)
531
+ return scharr
demo_example/000_reference.png ADDED

Git LFS Details

  • SHA256: 8211b4f36c9fd3e6d66e54421eb62a69cf3e45d351d1cd217e48358c741ed1ae
  • Pointer size: 132 Bytes
  • Size of remote file: 1.35 MB
demo_example/000_source.png ADDED
demo_example/001_reference.png ADDED
demo_example/001_source.png ADDED
demo_example/002_reference.png ADDED
demo_example/002_source.png ADDED
demo_example/003_reference.png ADDED
demo_example/003_source.png ADDED
demo_example/004_reference.png ADDED
demo_example/004_source.png ADDED
demo_example/005_reference.png ADDED
demo_example/005_source.png ADDED

Git LFS Details

  • SHA256: 9fd8ffe3264cf39d29e4cb4e2bd2ecad7e6f7ecb5aa564de2e9538b161eb4295
  • Pointer size: 132 Bytes
  • Size of remote file: 1.02 MB
demo_example/006_reference.png ADDED
demo_example/006_source.png ADDED
demo_example/007_reference.png ADDED

Git LFS Details

  • SHA256: 82e60aa6b4c16c2fbd6fc0f5e7b93376382c9dd91b425f17fb80ff817649d791
  • Pointer size: 132 Bytes
  • Size of remote file: 2.4 MB
demo_example/007_source.png ADDED
depthanything/.DS_Store ADDED
Binary file (6.15 kB). View file
 
depthanything/__pycache__/fast_import.cpython-38.pyc ADDED
Binary file (617 Bytes). View file
 
depthanything/depth_anything/.DS_Store ADDED
Binary file (6.15 kB). View file
 
depthanything/depth_anything/__pycache__/blocks.cpython-38.pyc ADDED
Binary file (3.25 kB). View file
 
depthanything/depth_anything/__pycache__/dpt.cpython-38.pyc ADDED
Binary file (5.1 kB). View file
 
depthanything/depth_anything/blocks.py ADDED
@@ -0,0 +1,153 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import torch.nn as nn
2
+
3
+
4
+ def _make_scratch(in_shape, out_shape, groups=1, expand=False):
5
+ scratch = nn.Module()
6
+
7
+ out_shape1 = out_shape
8
+ out_shape2 = out_shape
9
+ out_shape3 = out_shape
10
+ if len(in_shape) >= 4:
11
+ out_shape4 = out_shape
12
+
13
+ if expand:
14
+ out_shape1 = out_shape
15
+ out_shape2 = out_shape*2
16
+ out_shape3 = out_shape*4
17
+ if len(in_shape) >= 4:
18
+ out_shape4 = out_shape*8
19
+
20
+ scratch.layer1_rn = nn.Conv2d(
21
+ in_shape[0], out_shape1, kernel_size=3, stride=1, padding=1, bias=False, groups=groups
22
+ )
23
+ scratch.layer2_rn = nn.Conv2d(
24
+ in_shape[1], out_shape2, kernel_size=3, stride=1, padding=1, bias=False, groups=groups
25
+ )
26
+ scratch.layer3_rn = nn.Conv2d(
27
+ in_shape[2], out_shape3, kernel_size=3, stride=1, padding=1, bias=False, groups=groups
28
+ )
29
+ if len(in_shape) >= 4:
30
+ scratch.layer4_rn = nn.Conv2d(
31
+ in_shape[3], out_shape4, kernel_size=3, stride=1, padding=1, bias=False, groups=groups
32
+ )
33
+
34
+ return scratch
35
+
36
+
37
+ class ResidualConvUnit(nn.Module):
38
+ """Residual convolution module.
39
+ """
40
+
41
+ def __init__(self, features, activation, bn):
42
+ """Init.
43
+
44
+ Args:
45
+ features (int): number of features
46
+ """
47
+ super().__init__()
48
+
49
+ self.bn = bn
50
+
51
+ self.groups=1
52
+
53
+ self.conv1 = nn.Conv2d(
54
+ features, features, kernel_size=3, stride=1, padding=1, bias=True, groups=self.groups
55
+ )
56
+
57
+ self.conv2 = nn.Conv2d(
58
+ features, features, kernel_size=3, stride=1, padding=1, bias=True, groups=self.groups
59
+ )
60
+
61
+ if self.bn==True:
62
+ self.bn1 = nn.BatchNorm2d(features)
63
+ self.bn2 = nn.BatchNorm2d(features)
64
+
65
+ self.activation = activation
66
+
67
+ self.skip_add = nn.quantized.FloatFunctional()
68
+
69
+ def forward(self, x):
70
+ """Forward pass.
71
+
72
+ Args:
73
+ x (tensor): input
74
+
75
+ Returns:
76
+ tensor: output
77
+ """
78
+
79
+ out = self.activation(x)
80
+ out = self.conv1(out)
81
+ if self.bn==True:
82
+ out = self.bn1(out)
83
+
84
+ out = self.activation(out)
85
+ out = self.conv2(out)
86
+ if self.bn==True:
87
+ out = self.bn2(out)
88
+
89
+ if self.groups > 1:
90
+ out = self.conv_merge(out)
91
+
92
+ return self.skip_add.add(out, x)
93
+
94
+
95
+ class FeatureFusionBlock(nn.Module):
96
+ """Feature fusion block.
97
+ """
98
+
99
+ def __init__(self, features, activation, deconv=False, bn=False, expand=False, align_corners=True, size=None):
100
+ """Init.
101
+
102
+ Args:
103
+ features (int): number of features
104
+ """
105
+ super(FeatureFusionBlock, self).__init__()
106
+
107
+ self.deconv = deconv
108
+ self.align_corners = align_corners
109
+
110
+ self.groups=1
111
+
112
+ self.expand = expand
113
+ out_features = features
114
+ if self.expand==True:
115
+ out_features = features//2
116
+
117
+ self.out_conv = nn.Conv2d(features, out_features, kernel_size=1, stride=1, padding=0, bias=True, groups=1)
118
+
119
+ self.resConfUnit1 = ResidualConvUnit(features, activation, bn)
120
+ self.resConfUnit2 = ResidualConvUnit(features, activation, bn)
121
+
122
+ self.skip_add = nn.quantized.FloatFunctional()
123
+
124
+ self.size=size
125
+
126
+ def forward(self, *xs, size=None):
127
+ """Forward pass.
128
+
129
+ Returns:
130
+ tensor: output
131
+ """
132
+ output = xs[0]
133
+
134
+ if len(xs) == 2:
135
+ res = self.resConfUnit1(xs[1])
136
+ output = self.skip_add.add(output, res)
137
+
138
+ output = self.resConfUnit2(output)
139
+
140
+ if (size is None) and (self.size is None):
141
+ modifier = {"scale_factor": 2}
142
+ elif size is None:
143
+ modifier = {"size": self.size}
144
+ else:
145
+ modifier = {"size": size}
146
+
147
+ output = nn.functional.interpolate(
148
+ output, **modifier, mode="bilinear", align_corners=self.align_corners
149
+ )
150
+
151
+ output = self.out_conv(output)
152
+
153
+ return output
depthanything/depth_anything/dpt.py ADDED
@@ -0,0 +1,189 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import argparse
2
+ import torch
3
+ import torch.nn as nn
4
+ import torch.nn.functional as F
5
+ from huggingface_hub import PyTorchModelHubMixin, hf_hub_download
6
+ import os
7
+ from depth_anything.blocks import FeatureFusionBlock, _make_scratch
8
+
9
+
10
+ def _make_fusion_block(features, use_bn, size = None):
11
+ return FeatureFusionBlock(
12
+ features,
13
+ nn.ReLU(False),
14
+ deconv=False,
15
+ bn=use_bn,
16
+ expand=False,
17
+ align_corners=True,
18
+ size=size,
19
+ )
20
+
21
+
22
+ class DPTHead(nn.Module):
23
+ def __init__(self, nclass, in_channels, features=256, use_bn=False, out_channels=[256, 512, 1024, 1024], use_clstoken=False):
24
+ super(DPTHead, self).__init__()
25
+
26
+ self.nclass = nclass
27
+ self.use_clstoken = use_clstoken
28
+
29
+ self.projects = nn.ModuleList([
30
+ nn.Conv2d(
31
+ in_channels=in_channels,
32
+ out_channels=out_channel,
33
+ kernel_size=1,
34
+ stride=1,
35
+ padding=0,
36
+ ) for out_channel in out_channels
37
+ ])
38
+
39
+ self.resize_layers = nn.ModuleList([
40
+ nn.ConvTranspose2d(
41
+ in_channels=out_channels[0],
42
+ out_channels=out_channels[0],
43
+ kernel_size=4,
44
+ stride=4,
45
+ padding=0),
46
+ nn.ConvTranspose2d(
47
+ in_channels=out_channels[1],
48
+ out_channels=out_channels[1],
49
+ kernel_size=2,
50
+ stride=2,
51
+ padding=0),
52
+ nn.Identity(),
53
+ nn.Conv2d(
54
+ in_channels=out_channels[3],
55
+ out_channels=out_channels[3],
56
+ kernel_size=3,
57
+ stride=2,
58
+ padding=1)
59
+ ])
60
+
61
+ if use_clstoken:
62
+ self.readout_projects = nn.ModuleList()
63
+ for _ in range(len(self.projects)):
64
+ self.readout_projects.append(
65
+ nn.Sequential(
66
+ nn.Linear(2 * in_channels, in_channels),
67
+ nn.GELU()))
68
+
69
+ self.scratch = _make_scratch(
70
+ out_channels,
71
+ features,
72
+ groups=1,
73
+ expand=False,
74
+ )
75
+
76
+ self.scratch.stem_transpose = None
77
+
78
+ self.scratch.refinenet1 = _make_fusion_block(features, use_bn)
79
+ self.scratch.refinenet2 = _make_fusion_block(features, use_bn)
80
+ self.scratch.refinenet3 = _make_fusion_block(features, use_bn)
81
+ self.scratch.refinenet4 = _make_fusion_block(features, use_bn)
82
+
83
+ head_features_1 = features
84
+ head_features_2 = 32
85
+
86
+ if nclass > 1:
87
+ self.scratch.output_conv = nn.Sequential(
88
+ nn.Conv2d(head_features_1, head_features_1, kernel_size=3, stride=1, padding=1),
89
+ nn.ReLU(True),
90
+ nn.Conv2d(head_features_1, nclass, kernel_size=1, stride=1, padding=0),
91
+ )
92
+ else:
93
+ self.scratch.output_conv1 = nn.Conv2d(head_features_1, head_features_1 // 2, kernel_size=3, stride=1, padding=1)
94
+
95
+ self.scratch.output_conv2 = nn.Sequential(
96
+ nn.Conv2d(head_features_1 // 2, head_features_2, kernel_size=3, stride=1, padding=1),
97
+ nn.ReLU(True),
98
+ nn.Conv2d(head_features_2, 1, kernel_size=1, stride=1, padding=0),
99
+ nn.ReLU(True),
100
+ nn.Identity(),
101
+ )
102
+
103
+ def forward(self, out_features, patch_h, patch_w):
104
+ out = []
105
+ for i, x in enumerate(out_features):
106
+ if self.use_clstoken:
107
+ x, cls_token = x[0], x[1]
108
+ readout = cls_token.unsqueeze(1).expand_as(x)
109
+ x = self.readout_projects[i](torch.cat((x, readout), -1))
110
+ else:
111
+ x = x[0]
112
+
113
+ x = x.permute(0, 2, 1).reshape((x.shape[0], x.shape[-1], patch_h, patch_w))
114
+
115
+ x = self.projects[i](x)
116
+ x = self.resize_layers[i](x)
117
+
118
+ out.append(x)
119
+
120
+ layer_1, layer_2, layer_3, layer_4 = out
121
+
122
+ layer_1_rn = self.scratch.layer1_rn(layer_1)
123
+ layer_2_rn = self.scratch.layer2_rn(layer_2)
124
+ layer_3_rn = self.scratch.layer3_rn(layer_3)
125
+ layer_4_rn = self.scratch.layer4_rn(layer_4)
126
+
127
+ path_4 = self.scratch.refinenet4(layer_4_rn, size=layer_3_rn.shape[2:])
128
+ path_3 = self.scratch.refinenet3(path_4, layer_3_rn, size=layer_2_rn.shape[2:])
129
+ path_2 = self.scratch.refinenet2(path_3, layer_2_rn, size=layer_1_rn.shape[2:])
130
+ path_1 = self.scratch.refinenet1(path_2, layer_1_rn)
131
+
132
+ out = self.scratch.output_conv1(path_1)
133
+ out = F.interpolate(out, (int(patch_h * 14), int(patch_w * 14)), mode="bilinear", align_corners=True)
134
+ out = self.scratch.output_conv2(out)
135
+
136
+ return out
137
+
138
+
139
+ class DPT_DINOv2(nn.Module):
140
+ def __init__(self, encoder='vitl', features=256, out_channels=[256, 512, 1024, 1024], use_bn=False, use_clstoken=False, localhub=True):
141
+ super(DPT_DINOv2, self).__init__()
142
+
143
+ assert encoder in ['vits', 'vitb', 'vitl']
144
+
145
+ # in case the Internet connection is not stable, please load the DINOv2 locally
146
+ if localhub:
147
+ #self.pretrained = torch.hub.load('torchhub/facebookresearch_dinov2_main', 'dinov2_{:}14'.format(encoder), source='local', pretrained=False)
148
+ root = os.path.dirname(os.path.dirname(os.path.realpath(__file__)))
149
+ self.pretrained = torch.hub.load(f'{root}/torchhub/facebookresearch_dinov2_main', 'dinov2_{:}14'.format(encoder), source='local', pretrained=False)
150
+ else:
151
+ self.pretrained = torch.hub.load('facebookresearch/dinov2', 'dinov2_{:}14'.format(encoder))
152
+
153
+ dim = self.pretrained.blocks[0].attn.qkv.in_features
154
+
155
+ self.depth_head = DPTHead(1, dim, features, use_bn, out_channels=out_channels, use_clstoken=use_clstoken)
156
+
157
+ def forward(self, x):
158
+ h, w = x.shape[-2:]
159
+
160
+ features = self.pretrained.get_intermediate_layers(x, 4, return_class_token=True)
161
+
162
+ patch_h, patch_w = h // 14, w // 14
163
+
164
+ depth = self.depth_head(features, patch_h, patch_w)
165
+ depth = F.interpolate(depth, size=(h, w), mode="bilinear", align_corners=True)
166
+ depth = F.relu(depth)
167
+
168
+ return depth.squeeze(1)
169
+
170
+
171
+ class DepthAnything(DPT_DINOv2, PyTorchModelHubMixin):
172
+ def __init__(self, config):
173
+ super().__init__(**config)
174
+
175
+
176
+ if __name__ == '__main__':
177
+ parser = argparse.ArgumentParser()
178
+ parser.add_argument(
179
+ "--encoder",
180
+ default="vits",
181
+ type=str,
182
+ choices=["vits", "vitb", "vitl"],
183
+ )
184
+ args = parser.parse_args()
185
+
186
+ model = DepthAnything.from_pretrained("LiheYoung/depth_anything_{:}14".format(args.encoder))
187
+
188
+ print(model)
189
+
depthanything/depth_anything/util/__pycache__/transform.cpython-38.pyc ADDED
Binary file (6.11 kB). View file
 
depthanything/depth_anything/util/transform.py ADDED
@@ -0,0 +1,248 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import random
2
+ from PIL import Image, ImageOps, ImageFilter
3
+ import torch
4
+ from torchvision import transforms
5
+ import torch.nn.functional as F
6
+
7
+ import numpy as np
8
+ import cv2
9
+ import math
10
+
11
+
12
+ def apply_min_size(sample, size, image_interpolation_method=cv2.INTER_AREA):
13
+ """Rezise the sample to ensure the given size. Keeps aspect ratio.
14
+
15
+ Args:
16
+ sample (dict): sample
17
+ size (tuple): image size
18
+
19
+ Returns:
20
+ tuple: new size
21
+ """
22
+ shape = list(sample["disparity"].shape)
23
+
24
+ if shape[0] >= size[0] and shape[1] >= size[1]:
25
+ return sample
26
+
27
+ scale = [0, 0]
28
+ scale[0] = size[0] / shape[0]
29
+ scale[1] = size[1] / shape[1]
30
+
31
+ scale = max(scale)
32
+
33
+ shape[0] = math.ceil(scale * shape[0])
34
+ shape[1] = math.ceil(scale * shape[1])
35
+
36
+ # resize
37
+ sample["image"] = cv2.resize(
38
+ sample["image"], tuple(shape[::-1]), interpolation=image_interpolation_method
39
+ )
40
+
41
+ sample["disparity"] = cv2.resize(
42
+ sample["disparity"], tuple(shape[::-1]), interpolation=cv2.INTER_NEAREST
43
+ )
44
+ sample["mask"] = cv2.resize(
45
+ sample["mask"].astype(np.float32),
46
+ tuple(shape[::-1]),
47
+ interpolation=cv2.INTER_NEAREST,
48
+ )
49
+ sample["mask"] = sample["mask"].astype(bool)
50
+
51
+ return tuple(shape)
52
+
53
+
54
+ class Resize(object):
55
+ """Resize sample to given size (width, height).
56
+ """
57
+
58
+ def __init__(
59
+ self,
60
+ width,
61
+ height,
62
+ resize_target=True,
63
+ keep_aspect_ratio=False,
64
+ ensure_multiple_of=1,
65
+ resize_method="lower_bound",
66
+ image_interpolation_method=cv2.INTER_AREA,
67
+ ):
68
+ """Init.
69
+
70
+ Args:
71
+ width (int): desired output width
72
+ height (int): desired output height
73
+ resize_target (bool, optional):
74
+ True: Resize the full sample (image, mask, target).
75
+ False: Resize image only.
76
+ Defaults to True.
77
+ keep_aspect_ratio (bool, optional):
78
+ True: Keep the aspect ratio of the input sample.
79
+ Output sample might not have the given width and height, and
80
+ resize behaviour depends on the parameter 'resize_method'.
81
+ Defaults to False.
82
+ ensure_multiple_of (int, optional):
83
+ Output width and height is constrained to be multiple of this parameter.
84
+ Defaults to 1.
85
+ resize_method (str, optional):
86
+ "lower_bound": Output will be at least as large as the given size.
87
+ "upper_bound": Output will be at max as large as the given size. (Output size might be smaller than given size.)
88
+ "minimal": Scale as least as possible. (Output size might be smaller than given size.)
89
+ Defaults to "lower_bound".
90
+ """
91
+ self.__width = width
92
+ self.__height = height
93
+
94
+ self.__resize_target = resize_target
95
+ self.__keep_aspect_ratio = keep_aspect_ratio
96
+ self.__multiple_of = ensure_multiple_of
97
+ self.__resize_method = resize_method
98
+ self.__image_interpolation_method = image_interpolation_method
99
+
100
+ def constrain_to_multiple_of(self, x, min_val=0, max_val=None):
101
+ y = (np.round(x / self.__multiple_of) * self.__multiple_of).astype(int)
102
+
103
+ if max_val is not None and y > max_val:
104
+ y = (np.floor(x / self.__multiple_of) * self.__multiple_of).astype(int)
105
+
106
+ if y < min_val:
107
+ y = (np.ceil(x / self.__multiple_of) * self.__multiple_of).astype(int)
108
+
109
+ return y
110
+
111
+ def get_size(self, width, height):
112
+ # determine new height and width
113
+ scale_height = self.__height / height
114
+ scale_width = self.__width / width
115
+
116
+ if self.__keep_aspect_ratio:
117
+ if self.__resize_method == "lower_bound":
118
+ # scale such that output size is lower bound
119
+ if scale_width > scale_height:
120
+ # fit width
121
+ scale_height = scale_width
122
+ else:
123
+ # fit height
124
+ scale_width = scale_height
125
+ elif self.__resize_method == "upper_bound":
126
+ # scale such that output size is upper bound
127
+ if scale_width < scale_height:
128
+ # fit width
129
+ scale_height = scale_width
130
+ else:
131
+ # fit height
132
+ scale_width = scale_height
133
+ elif self.__resize_method == "minimal":
134
+ # scale as least as possbile
135
+ if abs(1 - scale_width) < abs(1 - scale_height):
136
+ # fit width
137
+ scale_height = scale_width
138
+ else:
139
+ # fit height
140
+ scale_width = scale_height
141
+ else:
142
+ raise ValueError(
143
+ f"resize_method {self.__resize_method} not implemented"
144
+ )
145
+
146
+ if self.__resize_method == "lower_bound":
147
+ new_height = self.constrain_to_multiple_of(
148
+ scale_height * height, min_val=self.__height
149
+ )
150
+ new_width = self.constrain_to_multiple_of(
151
+ scale_width * width, min_val=self.__width
152
+ )
153
+ elif self.__resize_method == "upper_bound":
154
+ new_height = self.constrain_to_multiple_of(
155
+ scale_height * height, max_val=self.__height
156
+ )
157
+ new_width = self.constrain_to_multiple_of(
158
+ scale_width * width, max_val=self.__width
159
+ )
160
+ elif self.__resize_method == "minimal":
161
+ new_height = self.constrain_to_multiple_of(scale_height * height)
162
+ new_width = self.constrain_to_multiple_of(scale_width * width)
163
+ else:
164
+ raise ValueError(f"resize_method {self.__resize_method} not implemented")
165
+
166
+ return (new_width, new_height)
167
+
168
+ def __call__(self, sample):
169
+ width, height = self.get_size(
170
+ sample["image"].shape[1], sample["image"].shape[0]
171
+ )
172
+
173
+ # resize sample
174
+ sample["image"] = cv2.resize(
175
+ sample["image"],
176
+ (width, height),
177
+ interpolation=self.__image_interpolation_method,
178
+ )
179
+
180
+ if self.__resize_target:
181
+ if "disparity" in sample:
182
+ sample["disparity"] = cv2.resize(
183
+ sample["disparity"],
184
+ (width, height),
185
+ interpolation=cv2.INTER_NEAREST,
186
+ )
187
+
188
+ if "depth" in sample:
189
+ sample["depth"] = cv2.resize(
190
+ sample["depth"], (width, height), interpolation=cv2.INTER_NEAREST
191
+ )
192
+
193
+ if "semseg_mask" in sample:
194
+ # sample["semseg_mask"] = cv2.resize(
195
+ # sample["semseg_mask"], (width, height), interpolation=cv2.INTER_NEAREST
196
+ # )
197
+ sample["semseg_mask"] = F.interpolate(torch.from_numpy(sample["semseg_mask"]).float()[None, None, ...], (height, width), mode='nearest').numpy()[0, 0]
198
+
199
+ if "mask" in sample:
200
+ sample["mask"] = cv2.resize(
201
+ sample["mask"].astype(np.float32),
202
+ (width, height),
203
+ interpolation=cv2.INTER_NEAREST,
204
+ )
205
+ # sample["mask"] = sample["mask"].astype(bool)
206
+
207
+ # print(sample['image'].shape, sample['depth'].shape)
208
+ return sample
209
+
210
+
211
+ class NormalizeImage(object):
212
+ """Normlize image by given mean and std.
213
+ """
214
+
215
+ def __init__(self, mean, std):
216
+ self.__mean = mean
217
+ self.__std = std
218
+
219
+ def __call__(self, sample):
220
+ sample["image"] = (sample["image"] - self.__mean) / self.__std
221
+
222
+ return sample
223
+
224
+
225
+ class PrepareForNet(object):
226
+ """Prepare sample for usage as network input.
227
+ """
228
+
229
+ def __init__(self):
230
+ pass
231
+
232
+ def __call__(self, sample):
233
+ image = np.transpose(sample["image"], (2, 0, 1))
234
+ sample["image"] = np.ascontiguousarray(image).astype(np.float32)
235
+
236
+ if "mask" in sample:
237
+ sample["mask"] = sample["mask"].astype(np.float32)
238
+ sample["mask"] = np.ascontiguousarray(sample["mask"])
239
+
240
+ if "depth" in sample:
241
+ depth = sample["depth"].astype(np.float32)
242
+ sample["depth"] = np.ascontiguousarray(depth)
243
+
244
+ if "semseg_mask" in sample:
245
+ sample["semseg_mask"] = sample["semseg_mask"].astype(np.float32)
246
+ sample["semseg_mask"] = np.ascontiguousarray(sample["semseg_mask"])
247
+
248
+ return sample
depthanything/fast_import.py ADDED
@@ -0,0 +1,13 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from depth_anything.dpt import DepthAnything
2
+ import numpy as np
3
+ import cv2
4
+ import torch
5
+ from torchvision.transforms import Compose
6
+
7
+ model_configs = {
8
+ 'vitl': {'encoder': 'vitl', 'features': 256, 'out_channels': [256, 512, 1024, 1024]},
9
+ 'vitb': {'encoder': 'vitb', 'features': 128, 'out_channels': [96, 192, 384, 768]},
10
+ 'vits': {'encoder': 'vits', 'features': 64, 'out_channels': [48, 96, 192, 384]}
11
+ }
12
+ encoder = 'vitb' # or 'vitb', 'vits'
13
+ depth_anything_model = DepthAnything(model_configs[encoder]).cuda().eval()
depthanything/torchhub/README.md ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ # Local PyTorch Hub
2
+
3
+ This directory is for loading the DINOv2 encoder locally in case of no Internet connection.
depthanything/torchhub/facebookresearch_dinov2_main/CODE_OF_CONDUCT.md ADDED
@@ -0,0 +1,80 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Code of Conduct
2
+
3
+ ## Our Pledge
4
+
5
+ In the interest of fostering an open and welcoming environment, we as
6
+ contributors and maintainers pledge to make participation in our project and
7
+ our community a harassment-free experience for everyone, regardless of age, body
8
+ size, disability, ethnicity, sex characteristics, gender identity and expression,
9
+ level of experience, education, socio-economic status, nationality, personal
10
+ appearance, race, religion, or sexual identity and orientation.
11
+
12
+ ## Our Standards
13
+
14
+ Examples of behavior that contributes to creating a positive environment
15
+ include:
16
+
17
+ * Using welcoming and inclusive language
18
+ * Being respectful of differing viewpoints and experiences
19
+ * Gracefully accepting constructive criticism
20
+ * Focusing on what is best for the community
21
+ * Showing empathy towards other community members
22
+
23
+ Examples of unacceptable behavior by participants include:
24
+
25
+ * The use of sexualized language or imagery and unwelcome sexual attention or
26
+ advances
27
+ * Trolling, insulting/derogatory comments, and personal or political attacks
28
+ * Public or private harassment
29
+ * Publishing others' private information, such as a physical or electronic
30
+ address, without explicit permission
31
+ * Other conduct which could reasonably be considered inappropriate in a
32
+ professional setting
33
+
34
+ ## Our Responsibilities
35
+
36
+ Project maintainers are responsible for clarifying the standards of acceptable
37
+ behavior and are expected to take appropriate and fair corrective action in
38
+ response to any instances of unacceptable behavior.
39
+
40
+ Project maintainers have the right and responsibility to remove, edit, or
41
+ reject comments, commits, code, wiki edits, issues, and other contributions
42
+ that are not aligned to this Code of Conduct, or to ban temporarily or
43
+ permanently any contributor for other behaviors that they deem inappropriate,
44
+ threatening, offensive, or harmful.
45
+
46
+ ## Scope
47
+
48
+ This Code of Conduct applies within all project spaces, and it also applies when
49
+ an individual is representing the project or its community in public spaces.
50
+ Examples of representing a project or community include using an official
51
+ project e-mail address, posting via an official social media account, or acting
52
+ as an appointed representative at an online or offline event. Representation of
53
+ a project may be further defined and clarified by project maintainers.
54
+
55
+ This Code of Conduct also applies outside the project spaces when there is a
56
+ reasonable belief that an individual's behavior may have a negative impact on
57
+ the project or its community.
58
+
59
+ ## Enforcement
60
+
61
+ Instances of abusive, harassing, or otherwise unacceptable behavior may be
62
+ reported by contacting the project team at <[email protected]>. All
63
+ complaints will be reviewed and investigated and will result in a response that
64
+ is deemed necessary and appropriate to the circumstances. The project team is
65
+ obligated to maintain confidentiality with regard to the reporter of an incident.
66
+ Further details of specific enforcement policies may be posted separately.
67
+
68
+ Project maintainers who do not follow or enforce the Code of Conduct in good
69
+ faith may face temporary or permanent repercussions as determined by other
70
+ members of the project's leadership.
71
+
72
+ ## Attribution
73
+
74
+ This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4,
75
+ available at https://www.contributor-covenant.org/version/1/4/code-of-conduct.html
76
+
77
+ [homepage]: https://www.contributor-covenant.org
78
+
79
+ For answers to common questions about this code of conduct, see
80
+ https://www.contributor-covenant.org/faq
depthanything/torchhub/facebookresearch_dinov2_main/CONTRIBUTING.md ADDED
@@ -0,0 +1,31 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Contributing to DINOv2
2
+ We want to make contributing to this project as easy and transparent as
3
+ possible.
4
+
5
+ ## Pull Requests
6
+ We actively welcome your pull requests.
7
+
8
+ 1. Fork the repo and create your branch from `main`.
9
+ 2. If you've added code that should be tested, add tests.
10
+ 3. If you've changed APIs, update the documentation.
11
+ 4. Ensure the test suite passes.
12
+ 5. Make sure your code lints.
13
+ 6. If you haven't already, complete the Contributor License Agreement ("CLA").
14
+
15
+ ## Contributor License Agreement ("CLA")
16
+ In order to accept your pull request, we need you to submit a CLA. You only need
17
+ to do this once to work on any of Meta's open source projects.
18
+
19
+ Complete your CLA here: <https://code.facebook.com/cla>
20
+
21
+ ## Issues
22
+ We use GitHub issues to track public bugs. Please ensure your description is
23
+ clear and has sufficient instructions to be able to reproduce the issue.
24
+
25
+ Meta has a [bounty program](https://www.facebook.com/whitehat/) for the safe
26
+ disclosure of security bugs. In those cases, please go through the process
27
+ outlined on that page and do not file a public issue.
28
+
29
+ ## License
30
+ By contributing to DINOv2, you agree that your contributions will be licensed
31
+ under the LICENSE file in the root directory of this source tree.
depthanything/torchhub/facebookresearch_dinov2_main/LICENSE ADDED
@@ -0,0 +1,400 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+
2
+ Attribution-NonCommercial 4.0 International
3
+
4
+ =======================================================================
5
+
6
+ Creative Commons Corporation ("Creative Commons") is not a law firm and
7
+ does not provide legal services or legal advice. Distribution of
8
+ Creative Commons public licenses does not create a lawyer-client or
9
+ other relationship. Creative Commons makes its licenses and related
10
+ information available on an "as-is" basis. Creative Commons gives no
11
+ warranties regarding its licenses, any material licensed under their
12
+ terms and conditions, or any related information. Creative Commons
13
+ disclaims all liability for damages resulting from their use to the
14
+ fullest extent possible.
15
+
16
+ Using Creative Commons Public Licenses
17
+
18
+ Creative Commons public licenses provide a standard set of terms and
19
+ conditions that creators and other rights holders may use to share
20
+ original works of authorship and other material subject to copyright
21
+ and certain other rights specified in the public license below. The
22
+ following considerations are for informational purposes only, are not
23
+ exhaustive, and do not form part of our licenses.
24
+
25
+ Considerations for licensors: Our public licenses are
26
+ intended for use by those authorized to give the public
27
+ permission to use material in ways otherwise restricted by
28
+ copyright and certain other rights. Our licenses are
29
+ irrevocable. Licensors should read and understand the terms
30
+ and conditions of the license they choose before applying it.
31
+ Licensors should also secure all rights necessary before
32
+ applying our licenses so that the public can reuse the
33
+ material as expected. Licensors should clearly mark any
34
+ material not subject to the license. This includes other CC-
35
+ licensed material, or material used under an exception or
36
+ limitation to copyright. More considerations for licensors:
37
+ wiki.creativecommons.org/Considerations_for_licensors
38
+
39
+ Considerations for the public: By using one of our public
40
+ licenses, a licensor grants the public permission to use the
41
+ licensed material under specified terms and conditions. If
42
+ the licensor's permission is not necessary for any reason--for
43
+ example, because of any applicable exception or limitation to
44
+ copyright--then that use is not regulated by the license. Our
45
+ licenses grant only permissions under copyright and certain
46
+ other rights that a licensor has authority to grant. Use of
47
+ the licensed material may still be restricted for other
48
+ reasons, including because others have copyright or other
49
+ rights in the material. A licensor may make special requests,
50
+ such as asking that all changes be marked or described.
51
+ Although not required by our licenses, you are encouraged to
52
+ respect those requests where reasonable. More_considerations
53
+ for the public:
54
+ wiki.creativecommons.org/Considerations_for_licensees
55
+
56
+ =======================================================================
57
+
58
+ Creative Commons Attribution-NonCommercial 4.0 International Public
59
+ License
60
+
61
+ By exercising the Licensed Rights (defined below), You accept and agree
62
+ to be bound by the terms and conditions of this Creative Commons
63
+ Attribution-NonCommercial 4.0 International Public License ("Public
64
+ License"). To the extent this Public License may be interpreted as a
65
+ contract, You are granted the Licensed Rights in consideration of Your
66
+ acceptance of these terms and conditions, and the Licensor grants You
67
+ such rights in consideration of benefits the Licensor receives from
68
+ making the Licensed Material available under these terms and
69
+ conditions.
70
+
71
+ Section 1 -- Definitions.
72
+
73
+ a. Adapted Material means material subject to Copyright and Similar
74
+ Rights that is derived from or based upon the Licensed Material
75
+ and in which the Licensed Material is translated, altered,
76
+ arranged, transformed, or otherwise modified in a manner requiring
77
+ permission under the Copyright and Similar Rights held by the
78
+ Licensor. For purposes of this Public License, where the Licensed
79
+ Material is a musical work, performance, or sound recording,
80
+ Adapted Material is always produced where the Licensed Material is
81
+ synched in timed relation with a moving image.
82
+
83
+ b. Adapter's License means the license You apply to Your Copyright
84
+ and Similar Rights in Your contributions to Adapted Material in
85
+ accordance with the terms and conditions of this Public License.
86
+
87
+ c. Copyright and Similar Rights means copyright and/or similar rights
88
+ closely related to copyright including, without limitation,
89
+ performance, broadcast, sound recording, and Sui Generis Database
90
+ Rights, without regard to how the rights are labeled or
91
+ categorized. For purposes of this Public License, the rights
92
+ specified in Section 2(b)(1)-(2) are not Copyright and Similar
93
+ Rights.
94
+ d. Effective Technological Measures means those measures that, in the
95
+ absence of proper authority, may not be circumvented under laws
96
+ fulfilling obligations under Article 11 of the WIPO Copyright
97
+ Treaty adopted on December 20, 1996, and/or similar international
98
+ agreements.
99
+
100
+ e. Exceptions and Limitations means fair use, fair dealing, and/or
101
+ any other exception or limitation to Copyright and Similar Rights
102
+ that applies to Your use of the Licensed Material.
103
+
104
+ f. Licensed Material means the artistic or literary work, database,
105
+ or other material to which the Licensor applied this Public
106
+ License.
107
+
108
+ g. Licensed Rights means the rights granted to You subject to the
109
+ terms and conditions of this Public License, which are limited to
110
+ all Copyright and Similar Rights that apply to Your use of the
111
+ Licensed Material and that the Licensor has authority to license.
112
+
113
+ h. Licensor means the individual(s) or entity(ies) granting rights
114
+ under this Public License.
115
+
116
+ i. NonCommercial means not primarily intended for or directed towards
117
+ commercial advantage or monetary compensation. For purposes of
118
+ this Public License, the exchange of the Licensed Material for
119
+ other material subject to Copyright and Similar Rights by digital
120
+ file-sharing or similar means is NonCommercial provided there is
121
+ no payment of monetary compensation in connection with the
122
+ exchange.
123
+
124
+ j. Share means to provide material to the public by any means or
125
+ process that requires permission under the Licensed Rights, such
126
+ as reproduction, public display, public performance, distribution,
127
+ dissemination, communication, or importation, and to make material
128
+ available to the public including in ways that members of the
129
+ public may access the material from a place and at a time
130
+ individually chosen by them.
131
+
132
+ k. Sui Generis Database Rights means rights other than copyright
133
+ resulting from Directive 96/9/EC of the European Parliament and of
134
+ the Council of 11 March 1996 on the legal protection of databases,
135
+ as amended and/or succeeded, as well as other essentially
136
+ equivalent rights anywhere in the world.
137
+
138
+ l. You means the individual or entity exercising the Licensed Rights
139
+ under this Public License. Your has a corresponding meaning.
140
+
141
+ Section 2 -- Scope.
142
+
143
+ a. License grant.
144
+
145
+ 1. Subject to the terms and conditions of this Public License,
146
+ the Licensor hereby grants You a worldwide, royalty-free,
147
+ non-sublicensable, non-exclusive, irrevocable license to
148
+ exercise the Licensed Rights in the Licensed Material to:
149
+
150
+ a. reproduce and Share the Licensed Material, in whole or
151
+ in part, for NonCommercial purposes only; and
152
+
153
+ b. produce, reproduce, and Share Adapted Material for
154
+ NonCommercial purposes only.
155
+
156
+ 2. Exceptions and Limitations. For the avoidance of doubt, where
157
+ Exceptions and Limitations apply to Your use, this Public
158
+ License does not apply, and You do not need to comply with
159
+ its terms and conditions.
160
+
161
+ 3. Term. The term of this Public License is specified in Section
162
+ 6(a).
163
+
164
+ 4. Media and formats; technical modifications allowed. The
165
+ Licensor authorizes You to exercise the Licensed Rights in
166
+ all media and formats whether now known or hereafter created,
167
+ and to make technical modifications necessary to do so. The
168
+ Licensor waives and/or agrees not to assert any right or
169
+ authority to forbid You from making technical modifications
170
+ necessary to exercise the Licensed Rights, including
171
+ technical modifications necessary to circumvent Effective
172
+ Technological Measures. For purposes of this Public License,
173
+ simply making modifications authorized by this Section 2(a)
174
+ (4) never produces Adapted Material.
175
+
176
+ 5. Downstream recipients.
177
+
178
+ a. Offer from the Licensor -- Licensed Material. Every
179
+ recipient of the Licensed Material automatically
180
+ receives an offer from the Licensor to exercise the
181
+ Licensed Rights under the terms and conditions of this
182
+ Public License.
183
+
184
+ b. No downstream restrictions. You may not offer or impose
185
+ any additional or different terms or conditions on, or
186
+ apply any Effective Technological Measures to, the
187
+ Licensed Material if doing so restricts exercise of the
188
+ Licensed Rights by any recipient of the Licensed
189
+ Material.
190
+
191
+ 6. No endorsement. Nothing in this Public License constitutes or
192
+ may be construed as permission to assert or imply that You
193
+ are, or that Your use of the Licensed Material is, connected
194
+ with, or sponsored, endorsed, or granted official status by,
195
+ the Licensor or others designated to receive attribution as
196
+ provided in Section 3(a)(1)(A)(i).
197
+
198
+ b. Other rights.
199
+
200
+ 1. Moral rights, such as the right of integrity, are not
201
+ licensed under this Public License, nor are publicity,
202
+ privacy, and/or other similar personality rights; however, to
203
+ the extent possible, the Licensor waives and/or agrees not to
204
+ assert any such rights held by the Licensor to the limited
205
+ extent necessary to allow You to exercise the Licensed
206
+ Rights, but not otherwise.
207
+
208
+ 2. Patent and trademark rights are not licensed under this
209
+ Public License.
210
+
211
+ 3. To the extent possible, the Licensor waives any right to
212
+ collect royalties from You for the exercise of the Licensed
213
+ Rights, whether directly or through a collecting society
214
+ under any voluntary or waivable statutory or compulsory
215
+ licensing scheme. In all other cases the Licensor expressly
216
+ reserves any right to collect such royalties, including when
217
+ the Licensed Material is used other than for NonCommercial
218
+ purposes.
219
+
220
+ Section 3 -- License Conditions.
221
+
222
+ Your exercise of the Licensed Rights is expressly made subject to the
223
+ following conditions.
224
+
225
+ a. Attribution.
226
+
227
+ 1. If You Share the Licensed Material (including in modified
228
+ form), You must:
229
+
230
+ a. retain the following if it is supplied by the Licensor
231
+ with the Licensed Material:
232
+
233
+ i. identification of the creator(s) of the Licensed
234
+ Material and any others designated to receive
235
+ attribution, in any reasonable manner requested by
236
+ the Licensor (including by pseudonym if
237
+ designated);
238
+
239
+ ii. a copyright notice;
240
+
241
+ iii. a notice that refers to this Public License;
242
+
243
+ iv. a notice that refers to the disclaimer of
244
+ warranties;
245
+
246
+ v. a URI or hyperlink to the Licensed Material to the
247
+ extent reasonably practicable;
248
+
249
+ b. indicate if You modified the Licensed Material and
250
+ retain an indication of any previous modifications; and
251
+
252
+ c. indicate the Licensed Material is licensed under this
253
+ Public License, and include the text of, or the URI or
254
+ hyperlink to, this Public License.
255
+
256
+ 2. You may satisfy the conditions in Section 3(a)(1) in any
257
+ reasonable manner based on the medium, means, and context in
258
+ which You Share the Licensed Material. For example, it may be
259
+ reasonable to satisfy the conditions by providing a URI or
260
+ hyperlink to a resource that includes the required
261
+ information.
262
+
263
+ 3. If requested by the Licensor, You must remove any of the
264
+ information required by Section 3(a)(1)(A) to the extent
265
+ reasonably practicable.
266
+
267
+ 4. If You Share Adapted Material You produce, the Adapter's
268
+ License You apply must not prevent recipients of the Adapted
269
+ Material from complying with this Public License.
270
+
271
+ Section 4 -- Sui Generis Database Rights.
272
+
273
+ Where the Licensed Rights include Sui Generis Database Rights that
274
+ apply to Your use of the Licensed Material:
275
+
276
+ a. for the avoidance of doubt, Section 2(a)(1) grants You the right
277
+ to extract, reuse, reproduce, and Share all or a substantial
278
+ portion of the contents of the database for NonCommercial purposes
279
+ only;
280
+
281
+ b. if You include all or a substantial portion of the database
282
+ contents in a database in which You have Sui Generis Database
283
+ Rights, then the database in which You have Sui Generis Database
284
+ Rights (but not its individual contents) is Adapted Material; and
285
+
286
+ c. You must comply with the conditions in Section 3(a) if You Share
287
+ all or a substantial portion of the contents of the database.
288
+
289
+ For the avoidance of doubt, this Section 4 supplements and does not
290
+ replace Your obligations under this Public License where the Licensed
291
+ Rights include other Copyright and Similar Rights.
292
+
293
+ Section 5 -- Disclaimer of Warranties and Limitation of Liability.
294
+
295
+ a. UNLESS OTHERWISE SEPARATELY UNDERTAKEN BY THE LICENSOR, TO THE
296
+ EXTENT POSSIBLE, THE LICENSOR OFFERS THE LICENSED MATERIAL AS-IS
297
+ AND AS-AVAILABLE, AND MAKES NO REPRESENTATIONS OR WARRANTIES OF
298
+ ANY KIND CONCERNING THE LICENSED MATERIAL, WHETHER EXPRESS,
299
+ IMPLIED, STATUTORY, OR OTHER. THIS INCLUDES, WITHOUT LIMITATION,
300
+ WARRANTIES OF TITLE, MERCHANTABILITY, FITNESS FOR A PARTICULAR
301
+ PURPOSE, NON-INFRINGEMENT, ABSENCE OF LATENT OR OTHER DEFECTS,
302
+ ACCURACY, OR THE PRESENCE OR ABSENCE OF ERRORS, WHETHER OR NOT
303
+ KNOWN OR DISCOVERABLE. WHERE DISCLAIMERS OF WARRANTIES ARE NOT
304
+ ALLOWED IN FULL OR IN PART, THIS DISCLAIMER MAY NOT APPLY TO YOU.
305
+
306
+ b. TO THE EXTENT POSSIBLE, IN NO EVENT WILL THE LICENSOR BE LIABLE
307
+ TO YOU ON ANY LEGAL THEORY (INCLUDING, WITHOUT LIMITATION,
308
+ NEGLIGENCE) OR OTHERWISE FOR ANY DIRECT, SPECIAL, INDIRECT,
309
+ INCIDENTAL, CONSEQUENTIAL, PUNITIVE, EXEMPLARY, OR OTHER LOSSES,
310
+ COSTS, EXPENSES, OR DAMAGES ARISING OUT OF THIS PUBLIC LICENSE OR
311
+ USE OF THE LICENSED MATERIAL, EVEN IF THE LICENSOR HAS BEEN
312
+ ADVISED OF THE POSSIBILITY OF SUCH LOSSES, COSTS, EXPENSES, OR
313
+ DAMAGES. WHERE A LIMITATION OF LIABILITY IS NOT ALLOWED IN FULL OR
314
+ IN PART, THIS LIMITATION MAY NOT APPLY TO YOU.
315
+
316
+ c. The disclaimer of warranties and limitation of liability provided
317
+ above shall be interpreted in a manner that, to the extent
318
+ possible, most closely approximates an absolute disclaimer and
319
+ waiver of all liability.
320
+
321
+ Section 6 -- Term and Termination.
322
+
323
+ a. This Public License applies for the term of the Copyright and
324
+ Similar Rights licensed here. However, if You fail to comply with
325
+ this Public License, then Your rights under this Public License
326
+ terminate automatically.
327
+
328
+ b. Where Your right to use the Licensed Material has terminated under
329
+ Section 6(a), it reinstates:
330
+
331
+ 1. automatically as of the date the violation is cured, provided
332
+ it is cured within 30 days of Your discovery of the
333
+ violation; or
334
+
335
+ 2. upon express reinstatement by the Licensor.
336
+
337
+ For the avoidance of doubt, this Section 6(b) does not affect any
338
+ right the Licensor may have to seek remedies for Your violations
339
+ of this Public License.
340
+
341
+ c. For the avoidance of doubt, the Licensor may also offer the
342
+ Licensed Material under separate terms or conditions or stop
343
+ distributing the Licensed Material at any time; however, doing so
344
+ will not terminate this Public License.
345
+
346
+ d. Sections 1, 5, 6, 7, and 8 survive termination of this Public
347
+ License.
348
+
349
+ Section 7 -- Other Terms and Conditions.
350
+
351
+ a. The Licensor shall not be bound by any additional or different
352
+ terms or conditions communicated by You unless expressly agreed.
353
+
354
+ b. Any arrangements, understandings, or agreements regarding the
355
+ Licensed Material not stated herein are separate from and
356
+ independent of the terms and conditions of this Public License.
357
+
358
+ Section 8 -- Interpretation.
359
+
360
+ a. For the avoidance of doubt, this Public License does not, and
361
+ shall not be interpreted to, reduce, limit, restrict, or impose
362
+ conditions on any use of the Licensed Material that could lawfully
363
+ be made without permission under this Public License.
364
+
365
+ b. To the extent possible, if any provision of this Public License is
366
+ deemed unenforceable, it shall be automatically reformed to the
367
+ minimum extent necessary to make it enforceable. If the provision
368
+ cannot be reformed, it shall be severed from this Public License
369
+ without affecting the enforceability of the remaining terms and
370
+ conditions.
371
+
372
+ c. No term or condition of this Public License will be waived and no
373
+ failure to comply consented to unless expressly agreed to by the
374
+ Licensor.
375
+
376
+ d. Nothing in this Public License constitutes or may be interpreted
377
+ as a limitation upon, or waiver of, any privileges and immunities
378
+ that apply to the Licensor or You, including from the legal
379
+ processes of any jurisdiction or authority.
380
+
381
+ =======================================================================
382
+
383
+ Creative Commons is not a party to its public
384
+ licenses. Notwithstanding, Creative Commons may elect to apply one of
385
+ its public licenses to material it publishes and in those instances
386
+ will be considered the “Licensor.” The text of the Creative Commons
387
+ public licenses is dedicated to the public domain under the CC0 Public
388
+ Domain Dedication. Except for the limited purpose of indicating that
389
+ material is shared under a Creative Commons public license or as
390
+ otherwise permitted by the Creative Commons policies published at
391
+ creativecommons.org/policies, Creative Commons does not authorize the
392
+ use of the trademark "Creative Commons" or any other trademark or logo
393
+ of Creative Commons without its prior written consent including,
394
+ without limitation, in connection with any unauthorized modifications
395
+ to any of its public licenses or any other arrangements,
396
+ understandings, or agreements concerning use of licensed material. For
397
+ the avoidance of doubt, this paragraph does not form part of the
398
+ public licenses.
399
+
400
+ Creative Commons may be contacted at creativecommons.org.
depthanything/torchhub/facebookresearch_dinov2_main/MODEL_CARD.md ADDED
@@ -0,0 +1,201 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Model Card for DINOv2-S/B/L/g
2
+
3
+ These are Vision Transformer models trained following the method described in the paper:
4
+ "DINOv2: Learning Robust Visual Features without Supervision"
5
+
6
+ We provide 4 models: 1 ViT-g trained from scratch, and 3 ViT-S/B/L models distilled from the ViT-g.
7
+
8
+ ## Model Details
9
+ The model takes an image as input and returns a class token and patch tokens.
10
+
11
+ The embedding dimension is:
12
+ - 384 for ViT-S.
13
+ - 768 for ViT-B.
14
+ - 1024 for ViT-L.
15
+ - 1536 for ViT-g.
16
+
17
+ The models follow a Transformer architecture, with a patch size of 14.
18
+
19
+ For a 224x224 image, this results in 1 class token + 256 patch tokens.
20
+
21
+ The models can accept larger images provided the image shapes are multiples of the patch size (14).
22
+ If this condition is not verified, the model will crop to the closest smaller multiple of the patch size.
23
+
24
+ ### Model Description
25
+
26
+ - **Developed by:** Meta AI
27
+ - **Model type:** Vision Transformer
28
+ - **License:** CC-BY-NC
29
+
30
+ - **Repository:** https://github.com/facebookresearch/dinov2
31
+ - **Paper:** https://arxiv.org/abs/2304.07193
32
+ - **Demo:** https://dinov2.metademolab.com/
33
+
34
+ ## Uses
35
+
36
+ The models are vision backbones providing multi-purpose features for downstream tasks.
37
+
38
+ ### Direct Use
39
+
40
+ The models can be used without fine-tuning, with downstream classifiers as simple as linear layers, to obtain competitive results:
41
+ - on depth estimation, semantic segmentation, using linear layers.
42
+ - on image classification, using k-NN classifiers on the class token.
43
+ - on image classification, with logistic regression classifiers applied on the class token.
44
+ - on image classification, with a linear layer applied on the class token and the average of the patch tokens.
45
+ - on image retrieval using nearest neighbors.
46
+
47
+ ### Downstream Use
48
+
49
+ It is technically possible to perform fine-tuning on the models, for small gains (we measured +2% on ImageNet-1k classification).
50
+ We recommend keeping this as a very last step and only when necessary, as the features already provide good performance out-of-the-box.
51
+
52
+ ## Bias, Risks, and Limitations
53
+
54
+ Despite improvements thanks to the training method not using annotations, we still observe significant biases in our models toward rich households from Western countries.
55
+
56
+ ### Recommendations
57
+
58
+ We expect fine-tuning will increase the biases in the features produced by the model as they will be tuned to the fine-tuning labels.
59
+
60
+ ## How to Get Started with the Model
61
+
62
+ Use the code below to get started with the model.
63
+
64
+ ```python
65
+ import torch
66
+ dinov2_vits14 = torch.hub.load('facebookresearch/dinov2', 'dinov2_vits14')
67
+ dinov2_vitb14 = torch.hub.load('facebookresearch/dinov2', 'dinov2_vitb14')
68
+ dinov2_vitl14 = torch.hub.load('facebookresearch/dinov2', 'dinov2_vitl14')
69
+ dinov2_vitg14 = torch.hub.load('facebookresearch/dinov2', 'dinov2_vitg14')
70
+ ```
71
+
72
+ ## Training Details
73
+
74
+ ### Training Data
75
+
76
+ - **Training data:** LVD-142M (see paper)
77
+ - **Training regime:** fp16 using PyTorch-FSDP mixed-precision.
78
+
79
+ ### Training Procedure
80
+
81
+ - **Training objective:**
82
+ - DINO self-distillation loss with multi-crop
83
+ - iBOT masked-image modeling loss
84
+ - KoLeo regularization on [CLS] tokens
85
+ - **Architectures:**
86
+ - ViT-S (21M params): Patch size 14, embedding dimension 384, 6 heads, MLP FFN
87
+ - ViT-B (86M params): Patch size 14, embedding dimension 768, 12 heads, MLP FFN
88
+ - ViT-L (0.3B params): Patch size 14, embedding dimension 1024, 16 heads, MLP FFN
89
+ - ViT-g (1.1B params): Patch size 14, embedding dimension 1536, 24 heads, SwiGLU FFN
90
+ - **Distillation:**
91
+ - Distillation follows the standard DINOv2 pretraining procedure, except the teacher is a pretrained ViT-g, frozen.
92
+
93
+ ## Evaluation
94
+
95
+ We refer users to the associated paper for the evaluation protocols.
96
+
97
+ <table>
98
+ <tr>
99
+ <th>model</th>
100
+ <th colspan="3">ImageNet-1k</th>
101
+ <th>NYU-Depth v2</th>
102
+ <th>SUN-RGBD</th>
103
+ <th>ADE20k</th>
104
+ <th>iNaturalist 2018</th>
105
+ <th>Oxford-H</th>
106
+ </tr>
107
+ <tr>
108
+ <th rowspan="2">task</th>
109
+ <th>classif. (acc)</th>
110
+ <th>classif. (acc)</th>
111
+ <th>classif. V2 (acc)</th>
112
+ <th>depth (RMSE)</th>
113
+ <th>depth (RMSE)</th>
114
+ <th>segm. (mAP)</th>
115
+ <th>classif. (acc)</th>
116
+ <th>retrieval (mAP)</th>
117
+ </tr>
118
+ <tr>
119
+ <!-- <th>^</th> -->
120
+ <th>k-NN</th>
121
+ <th>linear</th>
122
+ <th>linear</th>
123
+ <th>linear<br />4 layers</th>
124
+ <th>NYU-D transfer</th>
125
+ <th>multiscale</th>
126
+ <th>linear</th>
127
+ <th>nearest neighbor</th>
128
+ </tr>
129
+ <tr>
130
+ <td>ViT-S/14</td>
131
+ <td align="right">79.0%</td>
132
+ <td align="right">81.1%</td>
133
+ <td align="right">70.8%</td>
134
+ <td align="right">0.417</td>
135
+ <td align="right">0.431</td>
136
+ <td align="right">47.2</td>
137
+ <td align="right">69.5%</td>
138
+ <td align="right">43.2</td>
139
+ </tr>
140
+ <tr>
141
+ <td>ViT-B/14</td>
142
+ <td align="right">82.1%</td>
143
+ <td align="right">84.5%</td>
144
+ <td align="right">74.9%</td>
145
+ <td align="right">0.362</td>
146
+ <td align="right">0.400</td>
147
+ <td align="right">51.3</td>
148
+ <td align="right">76.3%</td>
149
+ <td align="right">49.5</td>
150
+ </tr>
151
+ <tr>
152
+ <td>ViT-L/14</td>
153
+ <td align="right">83.5%</td>
154
+ <td align="right">86.3%</td>
155
+ <td align="right">77.6%</td>
156
+ <td align="right">0.333</td>
157
+ <td align="right">0.396</td>
158
+ <td align="right">53.1</td>
159
+ <td align="right">79.8%</td>
160
+ <td align="right">54.0</td>
161
+ </tr>
162
+ <tr>
163
+ <td>ViT-g/14</td>
164
+ <td align="right">83.5%</td>
165
+ <td align="right">86.5%</td>
166
+ <td align="right">78.4%</td>
167
+ <td align="right">0.298</td>
168
+ <td align="right">0.362</td>
169
+ <td align="right">53.0</td>
170
+ <td align="right">81.6%</td>
171
+ <td align="right">52.3</td>
172
+ </tr>
173
+ </table>
174
+
175
+ ## Environmental Impact
176
+
177
+ - **Hardware Type:** Nvidia A100
178
+ - **Hours used:** 22,000 for ViT-g, 4,500 for ViT-S distillation, 5,300 for ViT-B distillation, 8,000 for ViT-L distillation
179
+ - **Cloud Provider:** Private infra
180
+ - **Compute Region:** USA
181
+ - **Carbon Emitted:** 7t CO2eq
182
+
183
+ #### Hardware
184
+
185
+ Nvidia A100 GPUs
186
+
187
+ #### Software
188
+
189
+ PyTorch 2.0,
190
+ xFormers 0.0.18
191
+
192
+ **BibTeX**
193
+
194
+ ```
195
+ @misc{oquab2023dinov2,
196
+ title={DINOv2: Learning Robust Visual Features without Supervision},
197
+ author={Oquab, Maxime and Darcet, Timothée and Moutakanni, Theo and Vo, Huy and Szafraniec, Marc and Khalidov, Vasil and Fernandez, Pierre and Haziza, Daniel and Massa, Francisco and El-Nouby, Alaaeldin and Howes, Russell and Huang, Po-Yao and Xu, Hu and Sharma, Vasu and Li, Shang-Wen and Galuba, Wojciech and Rabbat, Mike and Assran, Mido and Ballas, Nicolas and Synnaeve, Gabriel and Misra, Ishan and Jegou, Herve and Mairal, Julien and Labatut, Patrick and Joulin, Armand and Bojanowski, Piotr},
198
+ journal={arXiv:2304.07193},
199
+ year={2023}
200
+ }
201
+ ```
depthanything/torchhub/facebookresearch_dinov2_main/README.md ADDED
@@ -0,0 +1,277 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # DINOv2: Learning Robust Visual Features without Supervision
2
+
3
+ **[Meta AI Research, FAIR](https://ai.facebook.com/research/)**
4
+
5
+ Maxime Oquab,
6
+ Timothée Darcet,
7
+ Théo Moutakanni,
8
+ Huy V. Vo,
9
+ Marc Szafraniec,
10
+ Vasil Khalidov,
11
+ Patrick Labatut,
12
+ Armand Joulin,
13
+ Piotr Bojanowski
14
+
15
+ [[`Paper`](https://arxiv.org/abs/2304.07193)] [[`Blog`](https://ai.facebook.com/blog/dino-v2-computer-vision-self-supervised-learning/)] [[`Demo`](https://dinov2.metademolab.com)] [[`BibTeX`](#citing-dinov2)]
16
+
17
+ PyTorch implementation and pretrained models for DINOv2. For details, see the paper: **[DINOv2: Learning Robust Visual Features without Supervision](https://arxiv.org/abs/2304.07193)**.
18
+
19
+ DINOv2 models produce high-performance visual features that can be directly employed with classifiers as simple as linear layers on a variety of computer vision tasks; these visual features are robust and perform well across domains without any requirement for fine-tuning. The models were pretrained on a dataset of 142 M images without using any labels or annotations.
20
+
21
+ https://github.com/facebookresearch/dinov2/assets/60359573/f168823e-7922-415a-b429-578badf5c356
22
+
23
+ <div align="center">
24
+ Visualization of the three first principal components of the patch features of all frames, mapped to RGB values.
25
+ </div>
26
+
27
+ ## Pretrained models
28
+
29
+ <table style="margin: auto">
30
+ <tr>
31
+ <th>model</th>
32
+ <th># of<br />params</th>
33
+ <th>ImageNet<br />k-NN</th>
34
+ <th>ImageNet<br />linear</th>
35
+ <th>download</th>
36
+ </tr>
37
+ <tr>
38
+ <td>ViT-S/14 distilled</td>
39
+ <td align="right">21 M</td>
40
+ <td align="right">79.0%</td>
41
+ <td align="right">81.1%</td>
42
+ <td><a href="https://dl.fbaipublicfiles.com/dinov2/dinov2_vits14/dinov2_vits14_pretrain.pth">backbone only</a></td>
43
+ </tr>
44
+ <tr>
45
+ <td>ViT-B/14 distilled</td>
46
+ <td align="right">86 M</td>
47
+ <td align="right">82.1%</td>
48
+ <td align="right">84.5%</td>
49
+ <td><a href="https://dl.fbaipublicfiles.com/dinov2/dinov2_vitb14/dinov2_vitb14_pretrain.pth">backbone only</a></td>
50
+ </tr>
51
+ <tr>
52
+ <td>ViT-L/14 distilled</td>
53
+ <td align="right">300 M</td>
54
+ <td align="right">83.5%</td>
55
+ <td align="right">86.3%</td>
56
+ <td><a href="https://dl.fbaipublicfiles.com/dinov2/dinov2_vitl14/dinov2_vitl14_pretrain.pth">backbone only</a></td>
57
+ </tr>
58
+ <tr>
59
+ <td>ViT-g/14</td>
60
+ <td align="right">1,100 M</td>
61
+ <td align="right">83.5%</td>
62
+ <td align="right">86.5%</td>
63
+ <td><a href="https://dl.fbaipublicfiles.com/dinov2/dinov2_vitg14/dinov2_vitg14_pretrain.pth">backbone only</a></td>
64
+ </tr>
65
+ </table>
66
+
67
+ ### Pretrained models via PyTorch Hub
68
+
69
+ Please follow the instructions [here](https://pytorch.org/get-started/locally/) to install PyTorch (the only required dependency for loading the model). Installing PyTorch with CUDA support is strongly recommended.
70
+
71
+ A corresponding [model card](MODEL_CARD.md) is included in the repository.
72
+
73
+ ```python
74
+ import torch
75
+
76
+ dinov2_vits14 = torch.hub.load('facebookresearch/dinov2', 'dinov2_vits14')
77
+ dinov2_vitb14 = torch.hub.load('facebookresearch/dinov2', 'dinov2_vitb14')
78
+ dinov2_vitl14 = torch.hub.load('facebookresearch/dinov2', 'dinov2_vitl14')
79
+ dinov2_vitg14 = torch.hub.load('facebookresearch/dinov2', 'dinov2_vitg14')
80
+ ```
81
+
82
+ ## Installation
83
+
84
+ The training and evaluation code requires PyTorch 2.0 and [xFormers](https://github.com/facebookresearch/xformers) 0.0.18 as well as a number of other 3rd party packages. Note that the code has only been tested with the specified versions and also expects a Linux environment. To setup all the required dependencies for training and evaluation, please follow the instructions below:
85
+
86
+ *[conda](https://docs.conda.io/projects/conda/en/latest/user-guide/getting-started.html)* **(Recommended)** - Clone the repository and then create and activate a `dinov2` conda environment using the provided environment definition:
87
+
88
+ ```shell
89
+ conda env create -f conda.yaml
90
+ conda activate dinov2
91
+ ```
92
+
93
+ *[pip](https://pip.pypa.io/en/stable/getting-started/)* - Clone the repository and then use the provided `requirements.txt` to install the dependencies:
94
+
95
+ ```shell
96
+ pip install -r requirements.txt
97
+ ```
98
+
99
+ ## Data preparation
100
+
101
+ ### ImageNet-1k
102
+
103
+ The root directory of the dataset should hold the following contents:
104
+
105
+ - `<ROOT>/test/ILSVRC2012_test_00000001.JPEG`
106
+ - `<ROOT>/test/[..]`
107
+ - `<ROOT>/test/ILSVRC2012_test_00100000.JPEG`
108
+ - `<ROOT>/train/n01440764/n01440764_10026.JPEG`
109
+ - `<ROOT>/train/[...]`
110
+ - `<ROOT>/train/n15075141/n15075141_9993.JPEG`
111
+ - `<ROOT>/val/n01440764/ILSVRC2012_val_00000293.JPEG`
112
+ - `<ROOT>/val/[...]`
113
+ - `<ROOT>/val/n15075141/ILSVRC2012_val_00049174.JPEG`
114
+ - `<ROOT>/labels.txt`
115
+
116
+ The provided dataset implementation expects a few additional metadata files to be present under the extra directory:
117
+
118
+ - `<EXTRA>/class-ids-TRAIN.npy`
119
+ - `<EXTRA>/class-ids-VAL.npy`
120
+ - `<EXTRA>/class-names-TRAIN.npy`
121
+ - `<EXTRA>/class-names-VAL.npy`
122
+ - `<EXTRA>/entries-TEST.npy`
123
+ - `<EXTRA>/entries-TRAIN.npy`
124
+ - `<EXTRA>/entries-VAL.npy`
125
+
126
+ These metadata files can be generated (once) with the following lines of Python code:
127
+
128
+ ```python
129
+ from dinov2.data.datasets import ImageNet
130
+
131
+ for split in ImageNet.Split:
132
+ dataset = ImageNet(split=split, root="<ROOT>", extra="<EXTRA>")
133
+ dataset.dump_extra()
134
+ ```
135
+
136
+ Note that the root and extra directories do not have to be distinct directories.
137
+
138
+ ### ImageNet-22k
139
+
140
+ Please adapt the [dataset class](dinov2/data/datasets/image_net_22k.py) to match your local setup.
141
+
142
+ <br />
143
+
144
+ :warning: To execute the commands provided in the next sections for training and evaluation, the `dinov2` package should be included in the Python module search path, i.e. simply prefix the command to run with `PYTHONPATH=.`.
145
+
146
+ ## Training
147
+
148
+ ### Fast setup: training DINOv2 ViT-L/16 on ImageNet-1k
149
+
150
+ Run DINOv2 training on 4 A100-80GB nodes (32 GPUs) in a SLURM cluster environment with submitit:
151
+
152
+ ```shell
153
+ python dinov2/run/train/train.py \
154
+ --nodes 4 \
155
+ --config-file dinov2/configs/train/vitl16_short.yaml \
156
+ --output-dir <PATH/TO/OUTPUT/DIR> \
157
+ train.dataset_path=ImageNet:split=TRAIN:root=<PATH/TO/DATASET>:extra=<PATH/TO/DATASET>
158
+ ```
159
+
160
+ Training time is approximately 1 day and the resulting checkpoint should reach 81.6% on k-NN eval and 82.9% on linear eval.
161
+
162
+ The training code saves the weights of the teacher in the `eval` folder every 12500 iterations for evaluation.
163
+
164
+ ### Long setup: training DINOv2 ViT-L/14 on ImageNet-22k
165
+
166
+ Run DINOv2 training on 12 A100-80GB nodes (96 GPUs) in a SLURM cluster environment with submitit:
167
+
168
+ ```shell
169
+ python dinov2/run/train/train.py \
170
+ --nodes 12 \
171
+ --config-file dinov2/configs/train/vitl14.yaml \
172
+ --output-dir <PATH/TO/OUTPUT/DIR> \
173
+ train.dataset_path=ImageNet22k:root=<PATH/TO/DATASET>:extra=<PATH/TO/DATASET>
174
+ ```
175
+
176
+ Training time is approximately 3.3 days and the resulting checkpoint should reach 82.0% on k-NN eval and 84.5% on linear eval.
177
+
178
+ The training code saves the weights of the teacher in the `eval` folder every 12500 iterations for evaluation.
179
+
180
+
181
+ ## Evaluation
182
+
183
+ The training code regularly saves the teacher weights. In order to evaluate the model, run the following evaluation on a single node:
184
+
185
+ ### k-NN classification on ImageNet-1k
186
+
187
+ ```shell
188
+ python dinov2/run/eval/knn.py \
189
+ --config-file <PATH/TO/OUTPUT/DIR>/config.yaml \
190
+ --pretrained-weights <PATH/TO/OUTPUT/DIR>/eval/training_24999/teacher_checkpoint.pth \
191
+ --output-dir <PATH/TO/OUTPUT/DIR>/eval/training_24999/knn \
192
+ --train-dataset ImageNet:split=TRAIN:root=<PATH/TO/DATASET>:extra=<PATH/TO/DATASET> \
193
+ --val-dataset ImageNet:split=VAL:root=<PATH/TO/DATASET>:extra=<PATH/TO/DATASET>
194
+ ```
195
+
196
+ ### Logistic regression classification on ImageNet-1k
197
+
198
+ ```shell
199
+ python dinov2/run/eval/log_regression.py \
200
+ --config-file <PATH/TO/OUTPUT/DIR>/config.yaml \
201
+ --pretrained-weights <PATH/TO/OUTPUT/DIR>/eval/training_24999/teacher_checkpoint.pth \
202
+ --output-dir <PATH/TO/OUTPUT/DIR>/eval/training_24999/logreg \
203
+ --train-dataset ImageNet:split=TRAIN:root=<PATH/TO/DATASET>:extra=<PATH/TO/DATASET> \
204
+ --val-dataset ImageNet:split=VAL:root=<PATH/TO/DATASET>:extra=<PATH/TO/DATASET>
205
+ ```
206
+
207
+ ### Linear classification with data augmentation on ImageNet-1k
208
+
209
+ ```shell
210
+ python dinov2/run/eval/linear.py \
211
+ --config-file <PATH/TO/OUTPUT/DIR>/config.yaml \
212
+ --pretrained-weights <PATH/TO/OUTPUT/DIR>/eval/training_24999/teacher_checkpoint.pth \
213
+ --output-dir <PATH/TO/OUTPUT/DIR>/eval/training_24999/linear \
214
+ --train-dataset ImageNet:split=TRAIN:root=<PATH/TO/DATASET>:extra=<PATH/TO/DATASET> \
215
+ --val-dataset ImageNet:split=VAL:root=<PATH/TO/DATASET>:extra=<PATH/TO/DATASET>
216
+ ```
217
+
218
+ We release the weights from evaluating the different models:
219
+
220
+ <table style="margin: auto">
221
+ <tr>
222
+ <th>model</th>
223
+ <th>ImageNet<br />top-1</th>
224
+ <th>linear evaluation</th>
225
+ </tr>
226
+ <tr>
227
+ <td>ViT-S/14 distilled</td>
228
+ <td align="right">81.1%</td>
229
+ <td><a href="https://dl.fbaipublicfiles.com/dinov2/dinov2_vits14/dinov2_vits14_linear_head.pth">linear head weights</a></td>
230
+ </tr>
231
+ <tr>
232
+ <td>ViT-B/14 distilled</td>
233
+ <td align="right">84.5%</td>
234
+ <td><a href="https://dl.fbaipublicfiles.com/dinov2/dinov2_vitb14/dinov2_vitb14_linear_head.pth">linear head weights</a></td>
235
+ </tr>
236
+ <tr>
237
+ <td>ViT-L/14 distilled</td>
238
+ <td align="right">86.3%</td>
239
+ <td><a href="https://dl.fbaipublicfiles.com/dinov2/dinov2_vitl14/dinov2_vitl14_linear_head.pth">linear head weights</a></td>
240
+ </tr>
241
+ <tr>
242
+ <td>ViT-g/14</td>
243
+ <td align="right">86.5%</td>
244
+ <td><a href="https://dl.fbaipublicfiles.com/dinov2/dinov2_vitg14/dinov2_vitg14_linear_head.pth">linear head weights</a></td>
245
+ </tr>
246
+ </table>
247
+
248
+ The performance of the provided pretrained model weights can be evaluated as follows on ImageNet-1k:
249
+
250
+ ```shell
251
+ python dinov2/run/eval/linear.py \
252
+ --config-file dinov2/configs/eval/vitg14_pretrain.yaml \
253
+ --pretrained-weights https://dl.fbaipublicfiles.com/dinov2/dinov2_vitg14/dinov2_vitg14_pretrain.pth \
254
+ --train-dataset ImageNet:split=TRAIN:root=<PATH/TO/DATASET>:extra=<PATH/TO/DATASET> \
255
+ --val-dataset ImageNet:split=VAL:root=<PATH/TO/DATASET>:extra=<PATH/TO/DATASET>
256
+ ```
257
+
258
+ ## License
259
+
260
+ DINOv2 code and model weights are released under the CC-BY-NC 4.0 license. See [LICENSE](LICENSE) for additional details.
261
+
262
+ ## Contributing
263
+
264
+ See [contributing](CONTRIBUTING.md) and the [code of conduct](CODE_OF_CONDUCT.md).
265
+
266
+ ## Citing DINOv2
267
+
268
+ If you find this repository useful, please consider giving a star :star: and citation :t-rex::
269
+
270
+ ```
271
+ @misc{oquab2023dinov2,
272
+ title={DINOv2: Learning Robust Visual Features without Supervision},
273
+ author={Oquab, Maxime and Darcet, Timothée and Moutakanni, Theo and Vo, Huy V. and Szafraniec, Marc and Khalidov, Vasil and Fernandez, Pierre and Haziza, Daniel and Massa, Francisco and El-Nouby, Alaaeldin and Howes, Russell and Huang, Po-Yao and Xu, Hu and Sharma, Vasu and Li, Shang-Wen and Galuba, Wojciech and Rabbat, Mike and Assran, Mido and Ballas, Nicolas and Synnaeve, Gabriel and Misra, Ishan and Jegou, Herve and Mairal, Julien and Labatut, Patrick and Joulin, Armand and Bojanowski, Piotr},
274
+ journal={arXiv:2304.07193},
275
+ year={2023}
276
+ }
277
+ ```
depthanything/torchhub/facebookresearch_dinov2_main/__pycache__/hubconf.cpython-38.pyc ADDED
Binary file (4.28 kB). View file
 
depthanything/torchhub/facebookresearch_dinov2_main/__pycache__/vision_transformer.cpython-38.pyc ADDED
Binary file (12 kB). View file
 
depthanything/torchhub/facebookresearch_dinov2_main/conda.yaml ADDED
@@ -0,0 +1,22 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ name: dinov2
2
+ channels:
3
+ - defaults
4
+ - pytorch
5
+ - nvidia
6
+ - xformers
7
+ - conda-forge
8
+ dependencies:
9
+ - python=3.9
10
+ - pytorch::pytorch=2.0.0
11
+ - pytorch::pytorch-cuda=11.7.0
12
+ - pytorch::torchvision=0.15.0
13
+ - omegaconf
14
+ - torchmetrics=0.10.3
15
+ - fvcore
16
+ - iopath
17
+ - xformers::xformers=0.0.18
18
+ - pip
19
+ - pip:
20
+ - git+https://github.com/facebookincubator/submitit
21
+ - --extra-index-url https://pypi.nvidia.com
22
+ - cuml-cu11
depthanything/torchhub/facebookresearch_dinov2_main/dinov2/__init__.py ADDED
@@ -0,0 +1,7 @@
 
 
 
 
 
 
 
 
1
+ # Copyright (c) Meta Platforms, Inc. and affiliates.
2
+ # All rights reserved.
3
+ #
4
+ # This source code is licensed under the license found in the
5
+ # LICENSE file in the root directory of this source tree.
6
+
7
+ __version__ = "0.0.1"
depthanything/torchhub/facebookresearch_dinov2_main/dinov2/__pycache__/__init__.cpython-38.pyc ADDED
Binary file (247 Bytes). View file
 
depthanything/torchhub/facebookresearch_dinov2_main/dinov2/configs/__init__.py ADDED
@@ -0,0 +1,23 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Copyright (c) Meta Platforms, Inc. and affiliates.
2
+ # All rights reserved.
3
+ #
4
+ # This source code is licensed under the license found in the
5
+ # LICENSE file in the root directory of this source tree.
6
+
7
+ import pathlib
8
+
9
+ from omegaconf import OmegaConf
10
+
11
+
12
+ def load_config(config_name: str):
13
+ config_filename = config_name + ".yaml"
14
+ return OmegaConf.load(pathlib.Path(__file__).parent.resolve() / config_filename)
15
+
16
+
17
+ dinov2_default_config = load_config("ssl_default_config")
18
+
19
+
20
+ def load_and_merge_config(config_name: str):
21
+ default_config = OmegaConf.create(dinov2_default_config)
22
+ loaded_config = load_config(config_name)
23
+ return OmegaConf.merge(default_config, loaded_config)
depthanything/torchhub/facebookresearch_dinov2_main/dinov2/configs/eval/vitb14_pretrain.yaml ADDED
@@ -0,0 +1,6 @@
 
 
 
 
 
 
 
1
+ student:
2
+ arch: vit_base
3
+ patch_size: 14
4
+ crops:
5
+ global_crops_size: 518 # this is to set up the position embeddings properly
6
+ local_crops_size: 98
depthanything/torchhub/facebookresearch_dinov2_main/dinov2/configs/eval/vitg14_pretrain.yaml ADDED
@@ -0,0 +1,7 @@
 
 
 
 
 
 
 
 
1
+ student:
2
+ arch: vit_giant2
3
+ patch_size: 14
4
+ ffn_layer: swiglufused
5
+ crops:
6
+ global_crops_size: 518 # this is to set up the position embeddings properly
7
+ local_crops_size: 98
depthanything/torchhub/facebookresearch_dinov2_main/dinov2/configs/eval/vitl14_pretrain.yaml ADDED
@@ -0,0 +1,6 @@
 
 
 
 
 
 
 
1
+ student:
2
+ arch: vit_large
3
+ patch_size: 14
4
+ crops:
5
+ global_crops_size: 518 # this is to set up the position embeddings properly
6
+ local_crops_size: 98
depthanything/torchhub/facebookresearch_dinov2_main/dinov2/configs/eval/vits14_pretrain.yaml ADDED
@@ -0,0 +1,6 @@
 
 
 
 
 
 
 
1
+ student:
2
+ arch: vit_small
3
+ patch_size: 14
4
+ crops:
5
+ global_crops_size: 518 # this is to set up the position embeddings properly
6
+ local_crops_size: 98
depthanything/torchhub/facebookresearch_dinov2_main/dinov2/configs/ssl_default_config.yaml ADDED
@@ -0,0 +1,115 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ MODEL:
2
+ WEIGHTS: ''
3
+ compute_precision:
4
+ grad_scaler: true
5
+ teacher:
6
+ backbone:
7
+ sharding_strategy: SHARD_GRAD_OP
8
+ mixed_precision:
9
+ param_dtype: fp16
10
+ reduce_dtype: fp16
11
+ buffer_dtype: fp32
12
+ dino_head:
13
+ sharding_strategy: SHARD_GRAD_OP
14
+ mixed_precision:
15
+ param_dtype: fp16
16
+ reduce_dtype: fp16
17
+ buffer_dtype: fp32
18
+ ibot_head:
19
+ sharding_strategy: SHARD_GRAD_OP
20
+ mixed_precision:
21
+ param_dtype: fp16
22
+ reduce_dtype: fp16
23
+ buffer_dtype: fp32
24
+ student:
25
+ backbone:
26
+ sharding_strategy: SHARD_GRAD_OP
27
+ mixed_precision:
28
+ param_dtype: fp16
29
+ reduce_dtype: fp16
30
+ buffer_dtype: fp32
31
+ dino_head:
32
+ sharding_strategy: SHARD_GRAD_OP
33
+ mixed_precision:
34
+ param_dtype: fp16
35
+ reduce_dtype: fp32
36
+ buffer_dtype: fp32
37
+ ibot_head:
38
+ sharding_strategy: SHARD_GRAD_OP
39
+ mixed_precision:
40
+ param_dtype: fp16
41
+ reduce_dtype: fp32
42
+ buffer_dtype: fp32
43
+ dino:
44
+ loss_weight: 1.0
45
+ head_n_prototypes: 65536
46
+ head_bottleneck_dim: 256
47
+ head_nlayers: 3
48
+ head_hidden_dim: 2048
49
+ koleo_loss_weight: 0.1
50
+ ibot:
51
+ loss_weight: 1.0
52
+ mask_sample_probability: 0.5
53
+ mask_ratio_min_max:
54
+ - 0.1
55
+ - 0.5
56
+ separate_head: false
57
+ head_n_prototypes: 65536
58
+ head_bottleneck_dim: 256
59
+ head_nlayers: 3
60
+ head_hidden_dim: 2048
61
+ train:
62
+ batch_size_per_gpu: 64
63
+ dataset_path: ImageNet:split=TRAIN
64
+ output_dir: .
65
+ saveckp_freq: 20
66
+ seed: 0
67
+ num_workers: 10
68
+ OFFICIAL_EPOCH_LENGTH: 1250
69
+ cache_dataset: true
70
+ centering: "centering" # or "sinkhorn_knopp"
71
+ student:
72
+ arch: vit_large
73
+ patch_size: 16
74
+ drop_path_rate: 0.3
75
+ layerscale: 1.0e-05
76
+ drop_path_uniform: true
77
+ pretrained_weights: ''
78
+ ffn_layer: "mlp"
79
+ block_chunks: 0
80
+ qkv_bias: true
81
+ proj_bias: true
82
+ ffn_bias: true
83
+ teacher:
84
+ momentum_teacher: 0.992
85
+ final_momentum_teacher: 1
86
+ warmup_teacher_temp: 0.04
87
+ teacher_temp: 0.07
88
+ warmup_teacher_temp_epochs: 30
89
+ optim:
90
+ epochs: 100
91
+ weight_decay: 0.04
92
+ weight_decay_end: 0.4
93
+ base_lr: 0.004 # learning rate for a batch size of 1024
94
+ lr: 0. # will be set after applying scaling rule
95
+ warmup_epochs: 10
96
+ min_lr: 1.0e-06
97
+ clip_grad: 3.0
98
+ freeze_last_layer_epochs: 1
99
+ scaling_rule: sqrt_wrt_1024
100
+ patch_embed_lr_mult: 0.2
101
+ layerwise_decay: 0.9
102
+ adamw_beta1: 0.9
103
+ adamw_beta2: 0.999
104
+ crops:
105
+ global_crops_scale:
106
+ - 0.32
107
+ - 1.0
108
+ local_crops_number: 8
109
+ local_crops_scale:
110
+ - 0.05
111
+ - 0.32
112
+ global_crops_size: 224
113
+ local_crops_size: 96
114
+ evaluation:
115
+ eval_period_iterations: 12500
depthanything/torchhub/facebookresearch_dinov2_main/dinov2/configs/train/vitg14.yaml ADDED
@@ -0,0 +1,26 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ dino:
2
+ head_n_prototypes: 131072
3
+ head_bottleneck_dim: 384
4
+ ibot:
5
+ separate_head: true
6
+ head_n_prototypes: 131072
7
+ train:
8
+ batch_size_per_gpu: 12
9
+ dataset_path: ImageNet22k
10
+ centering: sinkhorn_knopp
11
+ student:
12
+ arch: vit_giant2
13
+ patch_size: 14
14
+ drop_path_rate: 0.4
15
+ ffn_layer: swiglufused
16
+ block_chunks: 4
17
+ teacher:
18
+ momentum_teacher: 0.994
19
+ optim:
20
+ epochs: 500
21
+ weight_decay_end: 0.2
22
+ base_lr: 2.0e-04 # learning rate for a batch size of 1024
23
+ warmup_epochs: 80
24
+ layerwise_decay: 1.0
25
+ crops:
26
+ local_crops_size: 98
depthanything/torchhub/facebookresearch_dinov2_main/dinov2/configs/train/vitl14.yaml ADDED
@@ -0,0 +1,26 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ dino:
2
+ head_n_prototypes: 131072
3
+ head_bottleneck_dim: 384
4
+ ibot:
5
+ separate_head: true
6
+ head_n_prototypes: 131072
7
+ train:
8
+ batch_size_per_gpu: 32
9
+ dataset_path: ImageNet22k
10
+ centering: sinkhorn_knopp
11
+ student:
12
+ arch: vit_large
13
+ patch_size: 14
14
+ drop_path_rate: 0.4
15
+ ffn_layer: swiglufused
16
+ block_chunks: 4
17
+ teacher:
18
+ momentum_teacher: 0.994
19
+ optim:
20
+ epochs: 500
21
+ weight_decay_end: 0.2
22
+ base_lr: 2.0e-04 # learning rate for a batch size of 1024
23
+ warmup_epochs: 80
24
+ layerwise_decay: 1.0
25
+ crops:
26
+ local_crops_size: 98
depthanything/torchhub/facebookresearch_dinov2_main/dinov2/configs/train/vitl16_short.yaml ADDED
@@ -0,0 +1,6 @@
 
 
 
 
 
 
 
1
+ # this corresponds to the default config
2
+ train:
3
+ dataset_path: ImageNet:split=TRAIN
4
+ batch_size_per_gpu: 64
5
+ student:
6
+ block_chunks: 4