import os
import gradio as gr
import numpy as np
import torch
from mobile_sam import SamAutomaticMaskGenerator, SamPredictor, sam_model_registry
from PIL import ImageDraw
from utils.tools import box_prompt, format_results, point_prompt
from utils.tools_gradio import fast_process
# Most of our demo code is from [FastSAM Demo](https://huggingface.co/spaces/An-619/FastSAM). Huge thanks for AN-619.
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
# Load the pre-trained model
sam_checkpoint = "./mobile_sam.pt"
model_type = "vit_t"
mobile_sam = sam_model_registry[model_type](checkpoint=sam_checkpoint)
mobile_sam = mobile_sam.to(device=device)
mobile_sam.eval()
mask_generator = SamAutomaticMaskGenerator(mobile_sam)
predictor = SamPredictor(mobile_sam)
# Description
title = "
Faster Segment Anything(MobileSAM)"
description_e = """This is a demo of [Faster Segment Anything(MobileSAM) Model](https://github.com/ChaoningZhang/MobileSAM).
We will provide box mode soon.
Enjoy!
"""
description_p = """##This is a demo of [Faster Segment Anything(MobileSAM) Model](https://github.com/ChaoningZhang/MobileSAM).
# Instructions for point mode
0. Restart by click the Restart button
1. Select a point with Add Mask for the foreground (Must)
2. Select a point with Remove Area for the background (Optional)
3. Click the Start Segmenting.
- Github [link](https://github.com/ChaoningZhang/MobileSAM)
- Model Card [link](https://huggingface.co/dhkim2810/MobileSAM)
We will provide box mode soon.
Enjoy!
"""
examples = [
["assets/picture3.jpg"],
["assets/picture4.jpg"],
["assets/picture5.jpg"],
["assets/picture6.jpg"],
["assets/picture1.jpg"],
["assets/picture2.jpg"],
]
default_example = examples[0]
css = "h1 { text-align: center } .about { text-align: justify; padding-left: 10%; padding-right: 10%; }"
@torch.no_grad()
def segment_everything(
image,
input_size=1024,
better_quality=False,
withContours=True,
use_retina=True,
mask_random_color=True,
):
global mask_generator
input_size = int(input_size)
w, h = image.size
scale = input_size / max(w, h)
new_w = int(w * scale)
new_h = int(h * scale)
image = image.resize((new_w, new_h))
nd_image = np.array(image)
annotations = mask_generator.generate(nd_image)
fig = fast_process(
annotations=annotations,
image=image,
device=device,
scale=(1024 // input_size),
better_quality=better_quality,
mask_random_color=mask_random_color,
bbox=None,
use_retina=use_retina,
withContours=withContours,
)
return fig
def segment_with_points(
image,
input_size=1024,
better_quality=False,
withContours=True,
use_retina=True,
mask_random_color=True,
):
global global_points
global global_point_label
input_size = int(input_size)
w, h = image.size
scale = input_size / max(w, h)
new_w = int(w * scale)
new_h = int(h * scale)
image = image.resize((new_w, new_h))
scaled_points = np.array(
[[int(x * scale) for x in point] for point in global_points]
)
scaled_point_label = np.array(global_point_label)
if scaled_points.size == 0 and scaled_point_label.size == 0:
print("No points selected")
return image, image
print(scaled_points, scaled_points is not None)
print(scaled_point_label, scaled_point_label is not None)
nd_image = np.array(image)
predictor.set_image(nd_image)
masks, scores, logits = predictor.predict(
point_coords=scaled_points,
point_labels=scaled_point_label,
multimask_output=True,
)
results = format_results(masks, scores, logits, 0)
annotations, _ = point_prompt(
results, scaled_points, scaled_point_label, new_h, new_w
)
annotations = np.array([annotations])
fig = fast_process(
annotations=annotations,
image=image,
device=device,
scale=(1024 // input_size),
better_quality=better_quality,
mask_random_color=mask_random_color,
bbox=None,
use_retina=use_retina,
withContours=withContours,
)
global_points = []
global_point_label = []
# return fig, None
return fig, image
def get_points_with_draw(image, label, evt: gr.SelectData):
global global_points
global global_point_label
x, y = evt.index[0], evt.index[1]
point_radius, point_color = 15, (255, 255, 0) if label == "Add Mask" else (
255,
0,
255,
)
global_points.append([x, y])
global_point_label.append(1 if label == "Add Mask" else 0)
print(x, y, label == "Add Mask")
# 创建一个可以在图像上绘图的对象
draw = ImageDraw.Draw(image)
draw.ellipse(
[(x - point_radius, y - point_radius), (x + point_radius, y + point_radius)],
fill=point_color,
)
return image
cond_img_e = gr.Image(label="Input", value=default_example[0], type="pil")
cond_img_p = gr.Image(label="Input with points", value=default_example[0], type="pil")
segm_img_e = gr.Image(label="Segmented Image", interactive=False, type="pil")
segm_img_p = gr.Image(
label="Segmented Image with points", interactive=False, type="pil"
)
global_points = []
global_point_label = []
input_size_slider = gr.components.Slider(
minimum=512,
maximum=1024,
value=1024,
step=64,
label="Input_size",
info="Our model was trained on a size of 1024",
)
with gr.Blocks(css=css, title="Faster Segment Anything(MobileSAM)") as demo:
with gr.Row():
with gr.Column(scale=1):
# Title
gr.Markdown(title)
# with gr.Tab("Everything mode"):
# # Images
# with gr.Row(variant="panel"):
# with gr.Column(scale=1):
# cond_img_e.render()
#
# with gr.Column(scale=1):
# segm_img_e.render()
#
# # Submit & Clear
# with gr.Row():
# with gr.Column():
# input_size_slider.render()
#
# with gr.Row():
# contour_check = gr.Checkbox(
# value=True,
# label="withContours",
# info="draw the edges of the masks",
# )
#
# with gr.Column():
# segment_btn_e = gr.Button(
# "Segment Everything", variant="primary"
# )
# clear_btn_e = gr.Button("Clear", variant="secondary")
#
# gr.Markdown("Try some of the examples below ⬇️")
# gr.Examples(
# examples=examples,
# inputs=[cond_img_e],
# outputs=segm_img_e,
# fn=segment_everything,
# cache_examples=True,
# examples_per_page=4,
# )
#
# with gr.Column():
# with gr.Accordion("Advanced options", open=False):
# # text_box = gr.Textbox(label="text prompt")
# with gr.Row():
# mor_check = gr.Checkbox(
# value=False,
# label="better_visual_quality",
# info="better quality using morphologyEx",
# )
# with gr.Column():
# retina_check = gr.Checkbox(
# value=True,
# label="use_retina",
# info="draw high-resolution segmentation masks",
# )
# # Description
# gr.Markdown(description_e)
#
with gr.Tab("Point mode"):
# Images
with gr.Row(variant="panel"):
with gr.Column(scale=1):
cond_img_p.render()
with gr.Column(scale=1):
segm_img_p.render()
# Submit & Clear
with gr.Row():
with gr.Column():
with gr.Row():
add_or_remove = gr.Radio(
["Add Mask", "Remove Area"],
value="Add Mask",
)
with gr.Column():
segment_btn_p = gr.Button(
"Start segmenting!", variant="primary"
)
clear_btn_p = gr.Button("Restart", variant="secondary")
gr.Markdown("Try some of the examples below ⬇️")
gr.Examples(
examples=examples,
inputs=[cond_img_p],
# outputs=segm_img_p,
# fn=segment_with_points,
# cache_examples=True,
examples_per_page=4,
)
with gr.Column():
# Description
gr.Markdown(description_p)
cond_img_p.select(get_points_with_draw, [cond_img_p, add_or_remove], cond_img_p)
# segment_btn_e.click(
# segment_everything,
# inputs=[
# cond_img_e,
# input_size_slider,
# mor_check,
# contour_check,
# retina_check,
# ],
# outputs=segm_img_e,
# )
segment_btn_p.click(
segment_with_points, inputs=[cond_img_p], outputs=[segm_img_p, cond_img_p]
)
def clear():
return None, None
def clear_text():
return None, None, None
# clear_btn_e.click(clear, outputs=[cond_img_e, segm_img_e])
clear_btn_p.click(clear, outputs=[cond_img_p, segm_img_p])
demo.queue()
demo.launch()