Spaces:
Runtime error
Runtime error
| import gradio as gr | |
| import torch | |
| from diffuserslocal.src.diffusers import UNet2DConditionModel | |
| from share_btn import community_icon_html, loading_icon_html, share_js | |
| from diffuserslocal.src.diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion_ldm3d_inpaint import StableDiffusionLDM3DInpaintPipeline | |
| from PIL import Image | |
| import numpy as np | |
| import cv2 | |
| from functools import partial | |
| import tempfile | |
| from mesh import get_mesh | |
| device = "cuda" if torch.cuda.is_available() else "cpu" | |
| model_arch = "zoe" | |
| # Inpainting pipeline | |
| unet = UNet2DConditionModel.from_pretrained("pablodawson/ldm3d-inpainting", cache_dir="cache", subfolder="unet") | |
| pipe = StableDiffusionLDM3DInpaintPipeline.from_pretrained("Intel/ldm3d-4c", cache_dir="cache" ).to(device) | |
| # Depth estimation | |
| model_type = "DPT_Large" # MiDaS v3 - Large (highest accuracy, slowest inference speed) | |
| #model_type = "DPT_Hybrid" # MiDaS v3 - Hybrid (medium accuracy, medium inference speed) | |
| #model_type = "MiDaS_small" # MiDaS v2.1 - Small (lowest accuracy, highest inference speed) | |
| if model_arch == "midas": | |
| midas = torch.hub.load("intel-isl/MiDaS", model_type) | |
| midas.to(device) | |
| midas.eval() | |
| midas_transforms = torch.hub.load("intel-isl/MiDaS", "transforms") | |
| if model_type == "DPT_Large" or model_type == "DPT_Hybrid": | |
| transform = midas_transforms.dpt_transform | |
| else: | |
| transform = midas_transforms.small_transform | |
| def estimate_depth(image): | |
| input_batch = transform(image).to(device) | |
| with torch.no_grad(): | |
| prediction = midas(input_batch) | |
| prediction = torch.nn.functional.interpolate( | |
| prediction.unsqueeze(1), | |
| size=image.shape[:2], | |
| mode="bicubic", | |
| align_corners=False, | |
| ).squeeze() | |
| output = prediction.cpu().numpy() | |
| output= 65535 * (output - np.min(output))/(np.max(output) - np.min(output)) | |
| return Image.fromarray(output.astype("int32")), output.min(), output.max() | |
| elif model_arch == "zoe": | |
| # Zoe_N | |
| repo = "isl-org/ZoeDepth" | |
| model_zoe_n = torch.hub.load(repo, "ZoeD_N", pretrained=True) | |
| zoe = model_zoe_n.to(device) | |
| def estimate_depth(image): | |
| depth_tensor = zoe.infer_pil(image, output_type="tensor") | |
| output = depth_tensor.cpu().numpy() | |
| output_ = 65535 * (1 - (output - np.min(output))/(np.max(output) - np.min(output))) | |
| return Image.fromarray(output_.astype("int32")), output.min(), output.max() | |
| def denormalize(image, min, max): | |
| image = (image / 65535 - 1 ) * (min - max) + min | |
| return image | |
| def read_content(file_path: str) -> str: | |
| """read the content of target file | |
| """ | |
| with open(file_path, 'r', encoding='utf-8') as f: | |
| content = f.read() | |
| return content | |
| def predict_images(dict, depth, prompt="", negative_prompt="", guidance_scale=7.5, steps=20, strength=1.0, scheduler="EulerDiscreteScheduler"): | |
| if negative_prompt == "": | |
| negative_prompt = None | |
| og_size = (dict["image"].shape[1], dict["image"].shape[0]) | |
| init_image = cv2.resize(dict["image"], (512, 512)) | |
| mask = Image.fromarray(cv2.resize(dict["mask"], (512, 512))[:,:,0]) | |
| if (depth is None): | |
| depth_image, _, _ = estimate_depth(init_image) | |
| else: | |
| d_i = depth[:,:,0] | |
| depth_image = 65535 * (d_i - np.min(d_i))/(np.max(d_i) - np.min(d_i)) | |
| depth_image = depth_image.astype("int32") | |
| depth_image = Image.fromarray(depth_image) | |
| init_image = Image.fromarray(init_image.astype("uint8")) | |
| depth_image = depth_image.resize((512, 512)) | |
| output = pipe(prompt = prompt, negative_prompt=negative_prompt, image=init_image, mask_image=mask, depth_image=depth_image, guidance_scale=guidance_scale, num_inference_steps=int(steps), strength=strength) | |
| depth_out = np.array(output.depth[0]) | |
| output_depth_vis = (depth_out - np.min(depth_out)) / (np.max(depth_out) - np.min(depth_out)) * 255 | |
| output_depth_vis = output_depth_vis.astype("uint8") | |
| output_depth = Image.fromarray(output_depth_vis) | |
| return output.rgb[0].resize(og_size), output_depth.resize(og_size), gr.update(visible=True) | |
| css = ''' | |
| .gradio-container{max-width: 1100px !important} | |
| #image_upload{min-height:400px} | |
| #image_upload [data-testid="image"], #image_upload [data-testid="image"] > div{min-height: 400px} | |
| #mask_radio .gr-form{background:transparent; border: none} | |
| #word_mask{margin-top: .75em !important} | |
| #word_mask textarea:disabled{opacity: 0.3} | |
| .footer {margin-bottom: 45px;margin-top: 35px;text-align: center;border-bottom: 1px solid #e5e5e5} | |
| .footer>p {font-size: .8rem; display: inline-block; padding: 0 10px;transform: translateY(10px);background: white} | |
| .dark .footer {border-color: #303030} | |
| .dark .footer>p {background: #0b0f19} | |
| .acknowledgments h4{margin: 1.25em 0 .25em 0;font-weight: bold;font-size: 115%} | |
| #image_upload .touch-none{display: flex} | |
| @keyframes spin { | |
| from { | |
| transform: rotate(0deg); | |
| } | |
| to { | |
| transform: rotate(360deg); | |
| } | |
| } | |
| #share-btn-container {padding-left: 0.5rem !important; padding-right: 0.5rem !important; background-color: #000000; justify-content: center; align-items: center; border-radius: 9999px !important; max-width: 13rem; margin-left: auto;} | |
| div#share-btn-container > div {flex-direction: row;background: black;align-items: center} | |
| #share-btn-container:hover {background-color: #060606} | |
| #share-btn {all: initial; color: #ffffff;font-weight: 600; cursor:pointer; font-family: 'IBM Plex Sans', sans-serif; margin-left: 0.5rem !important; padding-top: 0.5rem !important; padding-bottom: 0.5rem !important;right:0;} | |
| #share-btn * {all: unset} | |
| #share-btn-container div:nth-child(-n+2){width: auto !important;min-height: 0px !important;} | |
| #share-btn-container .wrap {display: none !important} | |
| #share-btn-container.hidden {display: none!important} | |
| #prompt input{width: calc(100% - 160px);border-top-right-radius: 0px;border-bottom-right-radius: 0px;} | |
| #run_button{position:absolute;margin-top: 11px;right: 0;margin-right: 0.8em;border-bottom-left-radius: 0px; | |
| border-top-left-radius: 0px;} | |
| #prompt-container{margin-top:-18px;} | |
| #prompt-container .form{border-top-left-radius: 0;border-top-right-radius: 0} | |
| #image_upload{border-bottom-left-radius: 0px;border-bottom-right-radius: 0px} | |
| ''' | |
| image_blocks = gr.Blocks(css=css, elem_id="total-container") | |
| def create_vis_demo(): | |
| with gr.Row(): | |
| with gr.Column(): | |
| image = gr.Image(source='upload', tool='sketch', elem_id="image_upload", type="numpy", label="Upload",height=400) | |
| depth = gr.Image(source='upload', elem_id="depth_upload", type="numpy", label="Upload",height=400) | |
| with gr.Row(elem_id="prompt-container", mobile_collapse=False, equal_height=True): | |
| with gr.Row(): | |
| prompt = gr.Textbox(placeholder="Your prompt (what you want in place of what is erased)", show_label=False, elem_id="prompt") | |
| btn = gr.Button("Inpaint!", elem_id="run_button") | |
| with gr.Accordion(label="Advanced Settings", open=False): | |
| with gr.Row(mobile_collapse=False, equal_height=True): | |
| guidance_scale = gr.Number(value=7.5, minimum=1.0, maximum=20.0, step=0.1, label="guidance_scale") | |
| steps = gr.Number(value=20, minimum=10, maximum=30, step=1, label="steps") | |
| strength = gr.Number(value=0.99, minimum=0.01, maximum=0.99, step=0.01, label="strength") | |
| negative_prompt = gr.Textbox(label="negative_prompt", placeholder="Your negative prompt", info="what you don't want to see in the image") | |
| with gr.Row(mobile_collapse=False, equal_height=True): | |
| schedulers = ["DEISMultistepScheduler", "HeunDiscreteScheduler", "EulerDiscreteScheduler", "DPMSolverMultistepScheduler", "DPMSolverMultistepScheduler-Karras", "DPMSolverMultistepScheduler-Karras-SDE"] | |
| scheduler = gr.Dropdown(label="Schedulers", choices=schedulers, value="EulerDiscreteScheduler") | |
| with gr.Column(): | |
| image_out = gr.Image(label="Output", elem_id="output-img", height=400) | |
| depth_out = gr.Image(label="Depth", elem_id="depth-img", height=400) | |
| with gr.Group(elem_id="share-btn-container", visible=False) as share_btn_container: | |
| community_icon = gr.HTML(community_icon_html) | |
| loading_icon = gr.HTML(loading_icon_html) | |
| share_button = gr.Button("Share to community", elem_id="share-btn",visible=True) | |
| btn.click(fn=predict_images, inputs=[image, depth, prompt, negative_prompt, guidance_scale, steps, strength, scheduler], outputs=[image_out, depth_out, share_btn_container], api_name='run') | |
| prompt.submit(fn=predict_images, inputs=[image, depth, prompt, negative_prompt, guidance_scale, steps, strength, scheduler], outputs=[image_out, depth_out, share_btn_container]) | |
| share_button.click(None, [], [], _js=share_js) | |
| def predict_images_3d(dict, depth, prompt="", negative_prompt="", guidance_scale=7.5, steps=20, strength=1.0, scheduler="EulerDiscreteScheduler", keep_edges=False): | |
| if negative_prompt == "": | |
| negative_prompt = None | |
| og_size = (dict["image"].shape[1], dict["image"].shape[0]) | |
| init_image = cv2.resize(dict["image"], (512, 512)) | |
| mask = Image.fromarray(cv2.resize(dict["mask"], (512, 512))[:,:,0]) | |
| mask.save("temp_mask.jpg") | |
| if (depth is None): | |
| depth_image, min, max = estimate_depth(init_image) | |
| else: | |
| d_i = depth[:,:,0] | |
| depth_image = 65535 * (d_i - np.min(d_i))/(np.max(d_i) - np.min(d_i)) | |
| depth_image = depth_image.astype("int32") | |
| depth_image = Image.fromarray(depth_image) | |
| init_image = Image.fromarray(init_image.astype("uint8")) | |
| depth_image = depth_image.resize((512, 512)) | |
| output = pipe(prompt = prompt, negative_prompt=negative_prompt, image=init_image, mask_image=mask, depth_image=depth_image, guidance_scale=guidance_scale, num_inference_steps=int(steps), strength=strength) | |
| # resize to original size | |
| #depth_image = depth_image.resize(og_size) | |
| #output_depth = output.depth[0].resize(og_size) | |
| depth_in = denormalize(np.array(depth_image), min, max) | |
| depth_out = denormalize(np.array(output.depth[0]), min, max) | |
| output_image = output.rgb[0] | |
| input_mesh = get_mesh(depth_in,init_image, keep_edges=keep_edges) | |
| output_mesh = get_mesh(depth_out, output_image, keep_edges=keep_edges) | |
| return input_mesh, output_mesh, gr.update(visible=True) | |
| def create_3d_demo(): | |
| gr.Markdown("### Image to 3D mesh") | |
| with gr.Row(): | |
| with gr.Row(): | |
| with gr.Column(): | |
| image = gr.Image(source='upload', tool='sketch', elem_id="image_upload", type="numpy", label="Upload",height=400, shape=(512,512)) | |
| depth = gr.Image(source='upload', elem_id="depth_upload", type="numpy", label="Upload",height=400, shape=(512,512)) | |
| checkbox = gr.Checkbox(label="Keep occlusion edges", value=False) | |
| prompt = gr.Textbox(placeholder="Your prompt (what you want in place of what is erased)", show_label=False, elem_id="prompt") | |
| with gr.Accordion(label="Advanced Settings", open=False): | |
| with gr.Row(mobile_collapse=False, equal_height=True): | |
| guidance_scale = gr.Number(value=7.5, minimum=1.0, maximum=20.0, step=0.1, label="guidance_scale") | |
| steps = gr.Number(value=20, minimum=10, maximum=30, step=1, label="steps") | |
| strength = gr.Number(value=0.99, minimum=0.01, maximum=0.99, step=0.01, label="strength") | |
| negative_prompt = gr.Textbox(label="negative_prompt", placeholder="Your negative prompt", info="what you don't want to see in the image") | |
| with gr.Row(mobile_collapse=False, equal_height=True): | |
| schedulers = ["DEISMultistepScheduler", "HeunDiscreteScheduler", "EulerDiscreteScheduler", "DPMSolverMultistepScheduler", "DPMSolverMultistepScheduler-Karras", "DPMSolverMultistepScheduler-Karras-SDE"] | |
| scheduler = gr.Dropdown(label="Schedulers", choices=schedulers, value="EulerDiscreteScheduler") | |
| with gr.Row() as share_btn_container: | |
| with gr.Column(): | |
| result_og = gr.Model3D(label="original 3d reconstruction", clear_color=[ | |
| 1.0, 1.0, 1.0, 1.0]) | |
| result_new = gr.Model3D(label="inpainted 3d reconstruction", clear_color=[ | |
| 1.0, 1.0, 1.0, 1.0]) | |
| submit = gr.Button("Submit") | |
| submit.click(fn=predict_images_3d, inputs=[image, depth, prompt, negative_prompt, guidance_scale, steps, strength, scheduler, checkbox], outputs=[result_og, result_new, share_btn_container], api_name='run') | |
| with image_blocks as demo: | |
| with gr.Tab("Image", default=True): | |
| create_vis_demo() | |
| with gr.Tab("3D"): | |
| create_3d_demo() | |
| gr.HTML(read_content("header.html")) | |
| image_blocks.queue(max_size=25).launch() |