adding app with CLIP image segmentation

app.py

@@ -13,25 +13,16 @@ processor = CLIPSegProcessor.from_pretrained("CIDAS/clipseg-rd64-refined")
 model = CLIPSegForImageSegmentation.from_pretrained("CIDAS/clipseg-rd64-refined")
 classes = list()
 
-def create_mask(image,image_mask,alpha=0.7):
-    mask = np.zeros_like(image)
-    # copy your image_mask to all dimensions (i.e. colors) of your image
-    for i in range(3): 
-        mask[:,:,i] = image_mask.copy()
-    # apply the mask to your image
-    overlay_image = cv2.addWeighted(mask,alpha,image,1-alpha,0)
-    return overlay_image
+def create_rgb_mask(mask):
+    color = tuple(np.random.choice(range(0,256), size=3))
+    gray_3_channel = cv2.merge((mask, mask, mask))
+    gray_3_channel[mask==255] = color
+    return gray_3_channel.astype(np.uint8)
 
-def rescale_bbox(bbox,orig_image_shape=(1024,1024),model_shape=352):
-    bbox = np.asarray(bbox)/model_shape
-    y1,y2 = bbox[::2] *orig_image_shape[0]
-    x1,x2 = bbox[1::2]*orig_image_shape[1]
-    return [int(y1),int(x1),int(y2),int(x2)]
 
 def detect_using_clip(image,prompts=[],threshould=0.4):
     h,w  = image.shape[:2]
-    model_detections = dict()
-    predicted_images = dict()
+    predicted_masks = list()
     inputs = processor(
         text=prompts,
         images=[image] * len(prompts),
@@ -42,31 +33,25 @@ def detect_using_clip(image,prompts=[],threshould=0.4):
         outputs = model(**inputs)
     preds = outputs.logits.unsqueeze(1)
 
-    detection = outputs.logits[0]  # Assuming class index 0
     for i,prompt in enumerate(prompts):
         predicted_image =  torch.sigmoid(preds[i][0]).detach().cpu().numpy()
-        predicted_image = np.where(predicted_image>threshould,np.random.randint(128,255),0)
-        # extract countours from the image
-        lbl_0 = label(predicted_image)
-        props = regionprops(lbl_0)
-        prompt = prompt.lower()
-        
-        model_detections[prompt] = [rescale_bbox(prop.bbox,orig_image_shape=image.shape[:2],model_shape=predicted_image.shape[0]) for prop in props]
-        predicted_images[prompt]= predicted_image
-    return model_detections , predicted_images
+        predicted_image = np.where(predicted_image>threshould,255,0)
 
-def visualize_images(image,detections,predicted_images,prompt):
+        predicted_masks.append(create_rgb_mask(predicted_image))
+    return predicted_masks
+
+def visualize_images(image,predicted_images):
     alpha = 0.7
     # H,W = image.shape[:2]
     prompt = prompt.lower()
     image_resize = cv2.resize(image,(352,352))
-    mask_image = create_mask(image=image_resize,image_mask=predicted_images[prompt])
+    resize_image_copy = image_resize.copy()
+
+    for mask_image in predicted_images:
+        resize_image_copy = cv2.addWeighted(resize_image_copy,alpha,mask_image,1-alpha,10)
+
+    return cv2.convertScaleAbs(resize_image_copy, alpha=1.8, beta=15)     
     
-    if prompt not in detections.keys():
-        print("prompt not in query ..")
-        return image_resize
-    final_image = cv2.addWeighted(image_resize,alpha,mask_image,1-alpha,0)
-    return final_image
 
 def shot(image, labels_text,selected_categoty):
     if "," in labels_text:
@@ -74,20 +59,23 @@ def shot(image, labels_text,selected_categoty):
     else:
         prompts = [labels_text]
     prompts = list(map(lambda x: x.strip(),prompts))
-    model_detections,predicted_images  = detect_using_clip(image,prompts=prompts)
-
-    category_image = visualize_images(image=image,detections=model_detections,predicted_images=predicted_images,prompt=selected_categoty)
     
+    predicted_images  = detect_using_clip(image,prompts=prompts)
+
+    category_image = visualize_images(image=image,predicted_images=predicted_images)
     return category_image
 
 iface = gr.Interface(fn=shot,
-                    inputs = ["image","text","text"],
+                    inputs = ["image","text"],
                     outputs = "image",
                     description ="Add an Image and lists of category to be detected separated by commas(atleast 2 )",
                     title = "Zero-shot Image Segmentation with Prompt ",
                     examples=[
-                        ["images/room.jpg","bed, table, plant, light, window",'plant'],
-                        ["images/image2.png","banner, building,door, sign","sign"]
+                        ["images/room.jpg","bed, table, plant, light, window,light"],
+                        ["images/image2.png","banner, building,door, sign,"],
+                        ["images/seats.jpg","door,table,chairs"],
+                        ["images/vegetables.jpg","carrot,radish,beans,potato,brnjal,basket"]
+                        ["images/room2.jpg","door,platns,dog,coffe table,mug,pillow,table lamp,carpet,pictures,door,clock"]
                         ],
                     # allow_flagging=False, 
                     # analytics_enabled=False,