Upload 170 files

.gitattributes

@@ -32,3 +32,7 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+gligen/ldm/data/imagenet_train_hr_indices.p filter=lfs diff=lfs merge=lfs -text
+gligen/projection_matrix.pth filter=lfs diff=lfs merge=lfs -text
+gligen/ldm/data/imagenet_val_hr_indices.p filter=lfs diff=lfs merge=lfs -text
+gligen/SD_input_conv_weight_bias.pth filter=lfs diff=lfs merge=lfs -text

.gitignore

@@ -110,3 +110,6 @@ create_samples/
 create_samples/*
 
 ckpts/*
+
+**/__pycache__/*
+**/__pycache__

README.md

@@ -1,12 +1,12 @@
 ---
-title: LoCo_Gligen Demo
-emoji: 👁
-colorFrom: blue
-colorTo: purple
+title: Attention Refocusing
+emoji: 🌖
+colorFrom: yellow
+colorTo: indigo
 sdk: gradio
 sdk_version: 3.19.1
 app_file: app.py
 pinned: false
 ---
 
-Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
+Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference

app.py

@@ -1,4 +1,5 @@
 import gradio as gr
+import os
 import torch
 from omegaconf import OmegaConf
 from gligen.task_grounded_generation import grounded_generation_box, load_ckpt, load_common_ckpt
@@ -18,9 +19,11 @@ import warnings
 
 from datetime import datetime
 
+from example_component import create_examples
+
 from huggingface_hub import hf_hub_download
 hf_hub_download = partial(hf_hub_download, library_name="gligen_demo")
-
+import cv2
 import sys
 sys.tracebacklimit = 0
 
@@ -39,8 +42,6 @@ def ckpt_load_helper(modality, is_inpaint, is_style, common_instances=None):
     pretrained_ckpt_gligen, config = load_ckpt_config_from_hf(modality)
     config = OmegaConf.create( config["_content"] ) # config used in training
     config.alpha_scale = 1.0
-    config.model['params']['is_inpaint'] = is_inpaint
-    config.model['params']['is_style'] = is_style
 
     if common_instances is None:
         common_ckpt = load_from_hf('gligen/demo_ckpts_legacy', filename=f'common.pth', subfolder='model')
@@ -138,13 +139,25 @@ class ImageMask(gr.components.Image):
         if x is None:
             return x
         if self.tool == "sketch" and self.source in ["upload", "webcam"] and type(x) != dict:
+           
             decode_image = processing_utils.decode_base64_to_image(x)
+            print('decode to 64')
             width, height = decode_image.size
+            img = np.asarray(decode_image)
+            return {'image':img, 'mask':binarize_2(img)}
+            
             mask = np.zeros((height, width, 4), dtype=np.uint8)
+            
             mask[..., -1] = 255
             mask = self.postprocess(mask)
             x = {'image': x, 'mask': mask}
-        return super().preprocess(x)
+            print('vao preprocess-------------------------')
+        hh = super().preprocess(x)
+        if (hh['image'].min()!=255) and (hh['mask'][:,:,:3].max()==0):
+            
+            hh['mask'] = binarize_2(hh['image'])
+        
+        return hh
 
 
 class Blocks(gr.Blocks):
@@ -180,23 +193,25 @@ class Blocks(gr.Blocks):
 inference model
 '''
 
-@torch.no_grad()
-def inference(task, language_instruction, grounding_instruction, inpainting_boxes_nodrop, image,
+# @torch.no_grad()
+def inference(task, language_instruction, phrase_list, location_list, inpainting_boxes_nodrop, image,
               alpha_sample, guidance_scale, batch_size,
               fix_seed, rand_seed, actual_mask, style_image,
               *args, **kwargs):
-    grounding_instruction = json.loads(grounding_instruction)
-    phrase_list, location_list = [], []
-    for k, v  in grounding_instruction.items():
-        phrase_list.append(k)
-        location_list.append(v)
+    # import pdb; pdb.set_trace()
+    
+    # grounding_instruction = json.loads(grounding_instruction)
+    # phrase_list, location_list = [], []
+    # for k, v  in grounding_instruction.items():
+    #     phrase_list.append(k)
+    #     location_list.append(v)
 
     placeholder_image = Image.open('images/teddy.jpg').convert("RGB")    
     image_list = [placeholder_image] * len(phrase_list) # placeholder input for visual prompt, which is disabled
 
     batch_size = int(batch_size)
     if not 1 <= batch_size <= 4:
-        batch_size = 2
+        batch_size = 1
 
     if style_image == None:
         has_text_mask = 1 
@@ -212,9 +227,6 @@ def inference(task, language_instruction, grounding_instruction, inpainting_boxe
         
         location_list += [ [0.0, 0.0, 1, 0.01]  ] # style image grounding location
 
-    if task == 'Grounded Inpainting':
-        alpha_sample = 1.0
-
     instruction = dict(
         prompt = language_instruction,
         phrases = phrase_list,
@@ -238,21 +250,19 @@ def inference(task, language_instruction, grounding_instruction, inpainting_boxe
                         phrase_list=phrase_list)
 
     with torch.autocast(device_type='cuda', dtype=torch.float16):
-        if task == 'Grounded Generation':
+        if task == 'User provide boxes' or 'Available boxes':
             if style_image == None:
-                return grounded_generation_box(get_model('base'), instruction, *args, **kwargs)
+                result = grounded_generation_box(get_model('base'), instruction, *args, **kwargs)
+                torch.cuda.empty_cache()
+                return result
             else:
                 return grounded_generation_box(get_model('style'), instruction, *args, **kwargs)
-        elif task == 'Grounded Inpainting':
-            assert image is not None
-            instruction['input_image'] = image.convert("RGB")
-            return grounded_generation_box(get_model('inpaint'), instruction, *args, **kwargs)
 
 
 def draw_box(boxes=[], texts=[], img=None):
     if len(boxes) == 0 and img is None:
         return None
-
+    
     if img is None:
         img = Image.new('RGB', (512, 512), (255, 255, 255))
     colors = ["red", "olive", "blue", "green", "orange", "brown", "cyan", "purple"]
@@ -281,7 +291,7 @@ def get_concat(ims):
 
 def auto_append_grounding(language_instruction, grounding_texts):
     for grounding_text in grounding_texts:
-        if grounding_text not in language_instruction and grounding_text != 'auto':
+        if grounding_text.lower() not in language_instruction.lower() and grounding_text != 'auto':
             language_instruction += "; " + grounding_text
     return language_instruction
 
@@ -292,6 +302,7 @@ def generate(task, language_instruction, grounding_texts, sketch_pad,
              alpha_sample, guidance_scale, batch_size,
              fix_seed, rand_seed, use_actual_mask, append_grounding, style_cond_image,
              state):
+    
     if 'boxes' not in state:
         state['boxes'] = []
 
@@ -307,44 +318,18 @@ Please draw boxes accordingly on the sketch pad.""".format(len(boxes), len(groun
 
     boxes = (np.asarray(boxes) / 512).tolist()
     grounding_instruction = json.dumps({obj: box for obj,box in zip(grounding_texts, boxes)})
-
     image = None
     actual_mask = None
-    if task == 'Grounded Inpainting':
-        image = state.get('original_image', sketch_pad['image']).copy()
-        image = center_crop(image)
-        image = Image.fromarray(image)
-
-        if use_actual_mask:
-            actual_mask = sketch_pad['mask'].copy()
-            if actual_mask.ndim == 3:
-                actual_mask = actual_mask[..., 0]
-            actual_mask = center_crop(actual_mask, tgt_size=(64, 64))
-            actual_mask = torch.from_numpy(actual_mask == 0).float()
-
-        if state.get('inpaint_hw', None):
-            boxes = np.asarray(boxes) * 0.9 + 0.05
-            boxes = boxes.tolist()
-            grounding_instruction = json.dumps({obj: box for obj,box in zip(grounding_texts, boxes) if obj != 'auto'})
+  
     
     if append_grounding:
         language_instruction = auto_append_grounding(language_instruction, grounding_texts)
 
     gen_images, gen_overlays = inference(
-        task, language_instruction, grounding_instruction, boxes, image,
+        task, language_instruction, grounding_texts,boxes, boxes, image,
         alpha_sample, guidance_scale, batch_size,
         fix_seed, rand_seed, actual_mask, style_cond_image, clip_model=clip_model,
     )
-
-    for idx, gen_image in enumerate(gen_images):
-
-        if task == 'Grounded Inpainting' and state.get('inpaint_hw', None):
-            hw = min(*state['original_image'].shape[:2])
-            gen_image = sized_center_fill(state['original_image'].copy(), np.array(gen_image.resize((hw, hw))), hw, hw)
-            gen_image = Image.fromarray(gen_image)
-        
-        gen_images[idx] = gen_image
-
     blank_samples = batch_size % 2 if batch_size > 1 else 0
     gen_images = [gr.Image.update(value=x, visible=True) for i,x in enumerate(gen_images)] \
                     + [gr.Image.update(value=None, visible=True) for _ in range(blank_samples)] \
@@ -355,6 +340,9 @@ Please draw boxes accordingly on the sketch pad.""".format(len(boxes), len(groun
 
 def binarize(x):
     return (x != 0).astype('uint8') * 255
+def binarize_2(x):
+    gray_image = cv2.cvtColor(x, cv2.COLOR_BGR2GRAY)
+    return (gray_image!=255).astype('uint8') * 255
 
 def sized_center_crop(img, cropx, cropy):
     y, x = img.shape[:2]
@@ -387,10 +375,20 @@ def center_crop(img, HW=None, tgt_size=(512, 512)):
     img = img.resize(tgt_size)
     return np.array(img)
 
-def draw(task, input, grounding_texts, new_image_trigger, state):
+# 接收 sketchpad 的输入 （左边）
+def draw(task, input, grounding_texts, new_image_trigger, state, generate_parsed, box_image):
+    print('input', generate_parsed)
+   
     if type(input) == dict:
         image = input['image']
         mask = input['mask']
+        if generate_parsed==1:
+            generate_parsed = 0
+            # import pdb; pdb.set_trace()
+            print('do nothing')
+            
+            return [box_image, new_image_trigger, 1., state, generate_parsed]
+           
     else:
         mask = input
 
@@ -398,24 +396,8 @@ def draw(task, input, grounding_texts, new_image_trigger, state):
         mask = mask[..., 0]
 
     image_scale = 1.0
-
-    # resize trigger
-    if task == "Grounded Inpainting":
-        mask_cond = mask.sum() == 0
-        # size_cond = mask.shape != (512, 512)
-        if mask_cond and 'original_image' not in state:
-            image = Image.fromarray(image)
-            width, height = image.size
-            scale = 600 / min(width, height)
-            image = image.resize((int(width * scale), int(height * scale)))
-            state['original_image'] = np.array(image).copy()
-            image_scale = float(height / width)
-            return [None, new_image_trigger + 1, image_scale, state]
-        else:
-            original_image = state['original_image']
-            H, W = original_image.shape[:2]
-            image_scale = float(H / W)
-
+    
+    print('vao draw--------------------')
     mask = binarize(mask)
     if mask.shape != (512, 512):
         # assert False, "should not receive any non- 512x512 masks."
@@ -424,16 +406,16 @@ def draw(task, input, grounding_texts, new_image_trigger, state):
             image = center_crop(state['original_image'], state['inpaint_hw'])
         else:
             mask = np.zeros((512, 512), dtype=np.uint8)
-    # mask = center_crop(mask)
     mask = binarize(mask)
 
     if type(mask) != np.ndarray:
         mask = np.array(mask)
-
-    if mask.sum() == 0 and task != "Grounded Inpainting":
+    # 
+    if mask.sum() == 0:
         state = {}
+        print('delete state')
 
-    if task != 'Grounded Inpainting':
+    if True:
         image = None
     else:
         image = Image.fromarray(image)
@@ -441,20 +423,20 @@ def draw(task, input, grounding_texts, new_image_trigger, state):
     if 'boxes' not in state:
         state['boxes'] = []
 
-    if 'masks' not in state or len(state['masks']) == 0:
+    if 'masks' not in state or len(state['masks']) == 0 :
         state['masks'] = []
         last_mask = np.zeros_like(mask)
     else:
         last_mask = state['masks'][-1]
-
-    if type(mask) == np.ndarray and mask.size > 1:
+  
+    if type(mask) == np.ndarray and mask.size > 1 :
         diff_mask = mask - last_mask
     else:
         diff_mask = np.zeros([])
 
     if diff_mask.sum() > 0:
-        x1x2 = np.where(diff_mask.max(0) != 0)[0]
-        y1y2 = np.where(diff_mask.max(1) != 0)[0]
+        x1x2 = np.where(diff_mask.max(0) > 1)[0]
+        y1y2 = np.where(diff_mask.max(1) > 1)[0]
         y1, y2 = y1y2.min(), y1y2.max()
         x1, x2 = x1x2.min(), x1x2.max()
 
@@ -466,26 +448,73 @@ def draw(task, input, grounding_texts, new_image_trigger, state):
     grounding_texts = [x for x in grounding_texts if len(x) > 0]
     if len(grounding_texts) < len(state['boxes']):
         grounding_texts += [f'Obj. {bid+1}' for bid in range(len(grounding_texts), len(state['boxes']))]
-
+    
     box_image = draw_box(state['boxes'], grounding_texts, image)
+    generate_parsed = 0
+    
+    return [box_image, new_image_trigger, image_scale, state, generate_parsed]
+
+def change_state(bboxes,layout, state, instruction, trigger_stage, boxes):
+    if trigger_stage ==0 :
+        return [boxes, state, 0]
+    # mask = 
+    state['boxes'] = []
+    state['masks'] = []
+    image = None
+    list_boxes = bboxes.split('/')
+    result =[]
+    for b in list_boxes:
+        ints = b[1:-1].split(',')
+        l = []
+        for i in ints:
+            l.append(int(i))
+        result.append(l)
+    print('run change state')
+   
+    for box in result:
+        state['boxes'].append(box)
+    grounding_texts = [x.strip() for x in instruction.split(';')]
+    grounding_texts = [x for x in grounding_texts if len(x) > 0]
+    if len(grounding_texts) < len(result):
+        grounding_texts += [f'Obj. {bid+1}' for bid in range(len(grounding_texts), len(result))]
 
-    if box_image is not None and state.get('inpaint_hw', None):
-        inpaint_hw = state['inpaint_hw']
-        box_image_resize = np.array(box_image.resize((inpaint_hw, inpaint_hw)))
-        original_image = state['original_image'].copy()
-        box_image = sized_center_fill(original_image, box_image_resize, inpaint_hw, inpaint_hw)
-
-    return [box_image, new_image_trigger, image_scale, state]
+    box_image = draw_box(result, grounding_texts)
+    
+    mask = binarize_2(layout['image'])
+    state['masks'].append(mask.copy())
+    # print('done change state', state)
+    print('done change state')
+    # import pdb; pdb.set_trace()
+    return [box_image,state, trigger_stage]
+
+def example_click(name, grounding_instruction, instruction, bboxes,generate_parsed,  trigger_parsed):
+    
+    list_boxes = bboxes.split('/')
+    result =[]
+   
+    for b in list_boxes:
+        ints = b[1:-1].split(',')
+        l = []
+        for i in ints:
+            l.append(int(i))
+        result.append(l)
+    print('run change state')
+    
+    box_image = draw_box(result, instruction)
+    trigger_parsed += 1
+    print('done the example click')
+    return [box_image, trigger_parsed]
 
-def clear(task, sketch_pad_trigger, batch_size, state, switch_task=False):
-    if task != 'Grounded Inpainting':
-        sketch_pad_trigger = sketch_pad_trigger + 1
+def clear(task, sketch_pad_trigger, batch_size, state,trigger_stage, switch_task=False):
+    
+    sketch_pad_trigger = sketch_pad_trigger + 1
+    trigger_stage = 0
     blank_samples = batch_size % 2 if batch_size > 1 else 0
     out_images = [gr.Image.update(value=None, visible=True) for i in range(batch_size)] \
                     + [gr.Image.update(value=None, visible=True) for _ in range(blank_samples)] \
                     + [gr.Image.update(value=None, visible=False) for _ in range(4 - batch_size - blank_samples)]
     state = {}
-    return [None, sketch_pad_trigger, None, 1.0] + out_images + [state]
+    return [None, sketch_pad_trigger, None, 1.0] + out_images + [state] + [trigger_stage]
 
 css = """
 #img2img_image, #img2img_image > .fixed-height, #img2img_image > .fixed-height > div, #img2img_image > .fixed-height > div > img
@@ -502,6 +531,10 @@ css = """
     cursor: pointer;
     text-decoration: none;
 }
+#my_image > div.fixed-height 
+{
+    height: var(--height) !important;
+}
 """
 
 rescale_js = """
@@ -516,42 +549,48 @@ function(x) {
     return x;
 }
 """
-
+# [<a href="https://arxiv.org/abs/2301.07093" target="_blank">Paper</a>]
 with Blocks(
     css=css,
     analytics_enabled=False,
-    title="GLIGen demo",
+    title="Attention-refocusing demo",
 ) as main:
     description = """<p style="text-align: center; font-weight: bold;">
-        <span style="font-size: 28px">GLIGen: Open-Set Grounded Text-to-Image Generation</span>
+        <span style="font-size: 28px">Grounded Text-to-Image Synthesis with Attention Refocusing</span>
         <br>
         <span style="font-size: 18px" id="paper-info">
-            [<a href="https://gligen.github.io" target="_blank">Project Page</a>]
-            [<a href="https://arxiv.org/abs/2301.07093" target="_blank">Paper</a>]
-            [<a href="https://github.com/gligen/GLIGEN" target="_blank">GitHub</a>]
+            [<a href="https://attention-refocusing.github.io/" target="_blank">Project Page</a>]
+            
+            [<a href="https://github.com/Attention-Refocusing/attention-refocusing" target="_blank">GitHub</a>]
         </span>
     </p>
     <p>
-        To ground concepts of interest with desired spatial specification, please (1) &#9000;&#65039; enter the concept names in <em> Grounding Instruction</em>, and (2) &#128433;&#65039; draw their corresponding bounding boxes one by one using <em> Sketch Pad</em> -- the parsed boxes will be displayed automatically.
+        To identify the areas of interest based on specific spatial parameters, you need to (1) &#9000;&#65039; input the names of the concepts you're interested  in <em> Grounding Instruction</em>, and (2) &#128433;&#65039; draw their corresponding bounding boxes using <em> Sketch Pad</em> -- the parsed boxes will automatically be showed up once you've drawn them.
         <br>
         For faster inference without waiting in queue, you may duplicate the space and upgrade to GPU in settings. <a href="https://huggingface.co/spaces/gligen/demo?duplicate=true"><img style="display: inline; margin-top: 0em; margin-bottom: 0em" src="https://bit.ly/3gLdBN6" alt="Duplicate Space" /></a>
     </p>
     """
     gr.HTML(description)
-
+    
     with gr.Row():
         with gr.Column(scale=4):
             sketch_pad_trigger = gr.Number(value=0, visible=False)
             sketch_pad_resize_trigger = gr.Number(value=0, visible=False)
+            trigger_stage = gr.Number(value=0, visible=False)
+            
             init_white_trigger = gr.Number(value=0, visible=False)
-            image_scale = gr.Number(value=0, elem_id="image_scale", visible=False)
+            image_scale = gr.Number(value=1.0, elem_id="image_scale", visible=False)
             new_image_trigger = gr.Number(value=0, visible=False)
-
+            text_box = gr.Textbox(visible=False)
+            generate_parsed = gr.Number(value=0, visible=False)
+            
             task = gr.Radio(
-                choices=["Grounded Generation", 'Grounded Inpainting'],
+                choices=["Available boxes", 'User provide boxes'],
                 type="value",
-                value="Grounded Generation",
+                value="User provide boxes",
                 label="Task",
+                visible=False
+                
             )
             language_instruction = gr.Textbox(
                 label="Language instruction",
@@ -561,33 +600,38 @@ with Blocks(
             )
             with gr.Row():
                 sketch_pad = ImageMask(label="Sketch Pad", elem_id="img2img_image")
-                out_imagebox = gr.Image(type="pil", label="Parsed Sketch Pad")
+                out_imagebox = gr.Image(type="pil",elem_id="my_image" ,label="Parsed Sketch Pad", shape=(512,512))
             with gr.Row():
                 clear_btn = gr.Button(value='Clear')
                 gen_btn = gr.Button(value='Generate')
+            with gr.Row():
+                parsed_btn = gr.Button(value='generate parsed boxes')
+
             with gr.Accordion("Advanced Options", open=False):
                 with gr.Column():
                     alpha_sample = gr.Slider(minimum=0, maximum=1.0, step=0.1, value=0.3, label="Scheduled Sampling (τ)")
                     guidance_scale = gr.Slider(minimum=0, maximum=50, step=0.5, value=7.5, label="Guidance Scale")
-                    batch_size = gr.Slider(minimum=1, maximum=4, step=1, value=2, label="Number of Samples")
+                    batch_size = gr.Slider(minimum=1, maximum=4,visible=False, step=1, value=1, label="Number of Samples")
                     append_grounding = gr.Checkbox(value=True, label="Append grounding instructions to the caption")
                     use_actual_mask = gr.Checkbox(value=False, label="Use actual mask for inpainting", visible=False)
                     with gr.Row():
                         fix_seed = gr.Checkbox(value=True, label="Fixed seed")
                         rand_seed = gr.Slider(minimum=0, maximum=1000, step=1, value=0, label="Seed")
-                    with gr.Row():
-                        use_style_cond = gr.Checkbox(value=False, label="Enable Style Condition")
-                        style_cond_image = gr.Image(type="pil", label="Style Condition", visible=False, interactive=True)
+                   
+                with gr.Row():
+                        use_style_cond = gr.Checkbox(value=False,visible=False, label="Enable Style Condition")
+                        style_cond_image = gr.Image(type="pil",visible=False, label="Style Condition", interactive=True)
         with gr.Column(scale=4):
             gr.HTML('<span style="font-size: 20px; font-weight: bold">Generated Images</span>')
             with gr.Row():
                 out_gen_1 = gr.Image(type="pil", visible=True, show_label=False)
-                out_gen_2 = gr.Image(type="pil", visible=True, show_label=False)
+                out_gen_2 = gr.Image(type="pil", visible=False, show_label=False)
             with gr.Row():
                 out_gen_3 = gr.Image(type="pil", visible=False, show_label=False)
                 out_gen_4 = gr.Image(type="pil", visible=False, show_label=False)
 
         state = gr.State({})
+        
 
         class Controller:
             def __init__(self):
@@ -605,75 +649,43 @@ with Blocks(
                 return [gr.Image.update(visible=True) for _ in range(n_samples + blank_samples)] \
                     + [gr.Image.update(visible=False) for _ in range(4 - n_samples - blank_samples)]
 
-            def resize_centercrop(self, state):
-                self.resizes += 1
-                image = state['original_image'].copy()
-                inpaint_hw = int(0.9 * min(*image.shape[:2]))
-                state['inpaint_hw'] = inpaint_hw
-                image_cc = center_crop(image, inpaint_hw)
-                # print(f'resize triggered {self.resizes}', image.shape, '->', image_cc.shape)
-                return image_cc, state
-
-            def resize_masked(self, state):
-                self.resizes += 1
-                image = state['original_image'].copy()
-                inpaint_hw = int(0.9 * min(*image.shape[:2]))
-                state['inpaint_hw'] = inpaint_hw
-                image_mask = sized_center_mask(image, inpaint_hw, inpaint_hw)
-                state['masked_image'] = image_mask.copy()
-                # print(f'mask triggered {self.resizes}')
-                return image_mask, state
-            
-            def switch_task_hide_cond(self, task):
-                cond = False
-                if task == "Grounded Generation":
-                    cond = True
-
-                return gr.Checkbox.update(visible=cond, value=False), gr.Image.update(value=None, visible=False), gr.Slider.update(visible=cond), gr.Checkbox.update(visible=(not cond), value=False)
-
         controller = Controller()
         main.load(
             lambda x:x+1,
             inputs=sketch_pad_trigger,
             outputs=sketch_pad_trigger,
             queue=False)
+      
         sketch_pad.edit(
             draw,
-            inputs=[task, sketch_pad, grounding_instruction, sketch_pad_resize_trigger, state],
-            outputs=[out_imagebox, sketch_pad_resize_trigger, image_scale, state],
+            inputs=[task, sketch_pad, grounding_instruction, sketch_pad_resize_trigger, state, generate_parsed, out_imagebox],
+            outputs=[out_imagebox, sketch_pad_resize_trigger, image_scale, state, generate_parsed],
             queue=False,
         )
+        trigger_stage.change(
+            change_state,
+            inputs=[text_box,sketch_pad, state, grounding_instruction, trigger_stage,out_imagebox],
+            outputs=[out_imagebox,state,trigger_stage],
+            queue=True
+        )
         grounding_instruction.change(
             draw,
-            inputs=[task, sketch_pad, grounding_instruction, sketch_pad_resize_trigger, state],
-            outputs=[out_imagebox, sketch_pad_resize_trigger, image_scale, state],
+            inputs=[task, sketch_pad, grounding_instruction, sketch_pad_resize_trigger, state, generate_parsed,out_imagebox],
+            outputs=[out_imagebox, sketch_pad_resize_trigger, image_scale, state, generate_parsed],
             queue=False,
         )
         clear_btn.click(
             clear,
-            inputs=[task, sketch_pad_trigger, batch_size, state],
-            outputs=[sketch_pad, sketch_pad_trigger, out_imagebox, image_scale, out_gen_1, out_gen_2, out_gen_3, out_gen_4, state],
-            queue=False)
-        task.change(
-            partial(clear, switch_task=True),
-            inputs=[task, sketch_pad_trigger, batch_size, state],
-            outputs=[sketch_pad, sketch_pad_trigger, out_imagebox, image_scale, out_gen_1, out_gen_2, out_gen_3, out_gen_4, state],
+            inputs=[task, sketch_pad_trigger, batch_size,trigger_stage, state],
+            outputs=[sketch_pad, sketch_pad_trigger, out_imagebox, image_scale, out_gen_1, out_gen_2, out_gen_3, out_gen_4, state, trigger_stage],
             queue=False)
+    
         sketch_pad_trigger.change(
             controller.init_white,
             inputs=[init_white_trigger],
             outputs=[sketch_pad, image_scale, init_white_trigger],
             queue=False)
-        sketch_pad_resize_trigger.change(
-            controller.resize_masked,
-            inputs=[state],
-            outputs=[sketch_pad, state],
-            queue=False)
-        batch_size.change(
-            controller.change_n_samples,
-            inputs=[batch_size],
-            outputs=[out_gen_1, out_gen_2, out_gen_3, out_gen_4],
-            queue=False)
+
         gen_btn.click(
             generate,
             inputs=[
@@ -687,88 +699,98 @@ with Blocks(
             outputs=[out_gen_1, out_gen_2, out_gen_3, out_gen_4, state],
             queue=True
         )
-        sketch_pad_resize_trigger.change(
-            None,
-            None,
-            sketch_pad_resize_trigger,
-            _js=rescale_js,
-            queue=False)
         init_white_trigger.change(
             None,
             None,
             init_white_trigger,
             _js=rescale_js,
             queue=False)
-        use_style_cond.change(
-            lambda cond: gr.Image.update(visible=cond),
-            use_style_cond,
-            style_cond_image,
-            queue=False)
-        task.change(
-            controller.switch_task_hide_cond,
-            inputs=task,
-            outputs=[use_style_cond, style_cond_image, alpha_sample, use_actual_mask],
-            queue=False)
-
-    with gr.Column():
-        gr.Examples(
-            examples=[
-                [
-                    "images/blank.png",
-                    "Grounded Generation",
-                    "a dog and an apple",
-                    "a dog;an apple",
+        examples = [
+        [
+                    'guide_imgs/0_a_cat_on_the_right_of_a_dog.jpg',
+                     "a cat;a dog",
+                    "a cat on the right of a dog",
+                    '(291, 88, 481, 301)/(25, 64, 260, 391)',
+                    1, 1
                 ],
                 [
-                    "images/blank.png",
-                    "Grounded Generation",
-                    "John Lennon is using a pc",
-                    "John Lennon;a pc",
-                [
-                    "images/blank.png",
-                    "Grounded Generation",
-                    "a painting of a fox sitting in a field at sunrise in the style of Claude Mone",
-                    "fox;sunrise",
-                ],
+                    'guide_imgs/0_a_bus_on_the_left_of_a_car.jpg',#'guide_imgs/0_a_bus_on_the_left_of_a_car.jpg',
+                     "a bus;a car",
+                    "a bus and a car",
+                    '(8,128,266,384)/(300,196,502,316)', #'(8,128,266,384)', #/(300,196,502,316)
+                    1, 2
                 ],
                 [
-                    "images/blank.png",
-                    "Grounded Generation",
-                    "a beautiful painting of hot dog by studio ghibli, octane render, brilliantly coloured",
-                    "hot dog",
+                    'guide_imgs/1_Two_cars_on_the_street..jpg',
+                     "a car;a car",
+                    "Two cars on the street.",
+                    '(34, 98, 247, 264)/(271, 122, 481, 293)',
+                    1, 3
                 ],
                 [
-                    "images/blank.png",
-                    "Grounded Generation",
-                    "a sport car, unreal engine, global illumination, ray tracing",
-                    "a sport car",
+                    'guide_imgs/80_two_apples_lay_side_by_side_on_a_wooden_table,_their_glossy_red_and_green_skins_glinting_in_the_sunlight..jpg',
+                     "an apple;an apple",
+                    "two apples lay side by side on a wooden table, their glossy red and green skins glinting in the sunlight.",
+                    '(40, 210, 235, 450)/(275, 210, 470, 450)',
+                    1, 4
                 ],
                 [
-                    "images/flower_beach.jpg",
-                    "Grounded Inpainting",
-                    "a squirrel and the space needle",
-                    "a squirrel;the space needle",
+                    'guide_imgs/10_A_banana_on_the_left_of_an_apple..jpg',
+                     "a banana;an apple",
+                    "A banana on the left of an apple.",
+                    '(62, 193, 225, 354)/(300, 184, 432, 329)',
+                    1, 5
                 ],
                 [
-                    "images/arg_corgis.jpeg",
-                    "Grounded Inpainting",
-                    "a dog and a birthday cake",
-                    "a dog; a birthday cake",
+                    'guide_imgs/15_A_pizza_on_the_right_of_a_suitcase..jpg',
+                     "a pizza ;a suitcase",
+                    "A pizza on the right of a suitcase.",
+                    '(307, 112, 490, 280)/(41, 120, 244, 270)',
+                    1, 6
                 ],
                 [
-                    "images/teddy.jpg",
-                    "Grounded Inpainting",
-                    "a teddy bear wearing a santa claus red shirt; holding a Christmas gift box on hand",
-                    "a santa claus shirt; a Christmas gift box",
-                ],
-            ],
-            inputs=[sketch_pad, task, language_instruction, grounding_instruction],
+                    'guide_imgs/1_A_wine_glass_on_top_of_a_dog..jpg',
+                     "a wine glass;a dog",
+                    "A wine glass on top of a dog.",
+                    '(206, 78, 306, 214)/(137, 222, 367, 432)',
+                    1, 7
+                ]
+                ,
+                [
+                    'guide_imgs/2_A_bicycle_on_top_of_a_boat..jpg',
+                     "a bicycle;a boat",
+                    "A bicycle on top of a boat.",
+                    '(185, 110, 335, 205)/(111, 228, 401, 373)',
+                    1, 8
+                ]
+                ,
+                [
+                    'guide_imgs/4_A_laptop_on_top_of_a_teddy_bear..jpg',
+                     "a laptop;a teddy bear",
+                    "A laptop on top of a teddy bear.",
+                    '(180, 70, 332, 210)/(150, 240, 362, 420)',
+                    1, 9
+                ]
+                ,
+                [
+                    'guide_imgs/0_A_train_on_top_of_a_surfboard..jpg',
+                     "a train;a surfboard",
+                    "A train on top of a surfboard.",
+                    '(130, 80, 385, 240)/(75, 260, 440, 450)',
+                    1, 10
+                ]
+         ]
+     
+    with gr.Column():
+        
+        create_examples(
+            examples=examples,
+            inputs=[sketch_pad, grounding_instruction,language_instruction , text_box, generate_parsed, trigger_stage],
             outputs=None,
             fn=None,
             cache_examples=False,
+            
         )
 
 main.queue(concurrency_count=1, api_open=False)
-main.launch(share=False, show_api=False, show_error=True)
-
-
+main.launch(share=False, show_api=False, show_error=True, debug=False,)

environment.yaml

@@ -1,4 +1,4 @@
-name: loco_gligen_demo
+name: gligen_demo
 channels:
   - xformers/label/dev
   - pytorch

example_component.py

@@ -0,0 +1,805 @@
+"""
+Defines helper methods useful for loading and caching Interface examples.
+"""
+from __future__ import annotations
+
+import ast
+import csv
+import inspect
+import os
+import subprocess
+import tempfile
+import threading
+import warnings
+from pathlib import Path
+from typing import TYPE_CHECKING, Any, Callable, Dict, Iterable, List, Tuple
+
+import matplotlib
+import matplotlib.pyplot as plt
+import numpy as np
+import PIL
+import PIL.Image
+
+from gradio import components, processing_utils, routes, utils
+from gradio.context import Context
+from gradio.documentation import document, set_documentation_group
+from gradio.flagging import CSVLogger
+
+if TYPE_CHECKING:  # Only import for type checking (to avoid circular imports).
+    from gradio.components import IOComponent
+
+CACHED_FOLDER = "gradio_cached_examples"
+LOG_FILE = "log.csv"
+
+set_documentation_group("helpers")
+
+
+def create_examples(
+    examples: List[Any] | List[List[Any]] | str,
+    inputs: IOComponent | List[IOComponent],
+    outputs: IOComponent | List[IOComponent] | None = None,
+    fn: Callable | None = None,
+    cache_examples: bool = False,
+    examples_per_page: int = 10,
+    _api_mode: bool = False,
+    label: str | None = None,
+    elem_id: str | None = None,
+    run_on_click: bool = False,
+    preprocess: bool = True,
+    postprocess: bool = True,
+    batch: bool = False,
+):
+    """Top-level synchronous function that creates Examples. Provided for backwards compatibility, i.e. so that gr.Examples(...) can be used to create the Examples component."""
+    examples_obj = Examples(
+        examples=examples,
+        inputs=inputs,
+        outputs=outputs,
+        fn=fn,
+        cache_examples=cache_examples,
+        examples_per_page=examples_per_page,
+        _api_mode=_api_mode,
+        label=label,
+        elem_id=elem_id,
+        run_on_click=run_on_click,
+        preprocess=preprocess,
+        postprocess=postprocess,
+        batch=batch,
+        _initiated_directly=False,
+    )
+    utils.synchronize_async(examples_obj.create)
+    return examples_obj
+
+
+class Examples:
+    """
+    This class is a wrapper over the Dataset component and can be used to create Examples
+    for Blocks / Interfaces. Populates the Dataset component with examples and
+    assigns event listener so that clicking on an example populates the input/output
+    components. Optionally handles example caching for fast inference.
+
+    Demos: blocks_inputs, fake_gan
+    Guides: more_on_examples_and_flagging, using_hugging_face_integrations, image_classification_in_pytorch, image_classification_in_tensorflow, image_classification_with_vision_transformers, create_your_own_friends_with_a_gan
+    """
+
+    def __init__(
+        self,
+        examples: List[Any] | List[List[Any]] | str,
+        inputs: IOComponent | List[IOComponent],
+        outputs: IOComponent | List[IOComponent] | None = None,
+        fn: Callable | None = None,
+        cache_examples: bool = False,
+        examples_per_page: int = 10,
+        _api_mode: bool = False,
+        label: str | None = "Examples",
+        elem_id: str | None = None,
+        run_on_click: bool = False,
+        preprocess: bool = True,
+        postprocess: bool = True,
+        batch: bool = False,
+        _initiated_directly: bool = True,
+    ):
+        """
+        Parameters:
+            examples: example inputs that can be clicked to populate specific components. Should be nested list, in which the outer list consists of samples and each inner list consists of an input corresponding to each input component. A string path to a directory of examples can also be provided but it should be within the directory with the python file running the gradio app. If there are multiple input components and a directory is provided, a log.csv file must be present in the directory to link corresponding inputs.
+            inputs: the component or list of components corresponding to the examples
+            outputs: optionally, provide the component or list of components corresponding to the output of the examples. Required if `cache` is True.
+            fn: optionally, provide the function to run to generate the outputs corresponding to the examples. Required if `cache` is True.
+            cache_examples: if True, caches examples for fast runtime. If True, then `fn` and `outputs` need to be provided
+            examples_per_page: how many examples to show per page.
+            label: the label to use for the examples component (by default, "Examples")
+            elem_id: an optional string that is assigned as the id of this component in the HTML DOM.
+            run_on_click: if cache_examples is False, clicking on an example does not run the function when an example is clicked. Set this to True to run the function when an example is clicked. Has no effect if cache_examples is True.
+            preprocess: if True, preprocesses the example input before running the prediction function and caching the output. Only applies if cache_examples is True.
+            postprocess: if True, postprocesses the example output after running the prediction function and before caching. Only applies if cache_examples is True.
+            batch: If True, then the function should process a batch of inputs, meaning that it should accept a list of input values for each parameter. Used only if cache_examples is True.
+        """
+        if _initiated_directly:
+            warnings.warn(
+                "Please use gr.Examples(...) instead of gr.examples.Examples(...) to create the Examples.",
+            )
+
+        if cache_examples and (fn is None or outputs is None):
+            raise ValueError("If caching examples, `fn` and `outputs` must be provided")
+
+        if not isinstance(inputs, list):
+            inputs = [inputs]
+        if outputs and not isinstance(outputs, list):
+            outputs = [outputs]
+
+        working_directory = Path().absolute()
+
+        if examples is None:
+            raise ValueError("The parameter `examples` cannot be None")
+        elif isinstance(examples, list) and (
+            len(examples) == 0 or isinstance(examples[0], list)
+        ):
+            pass
+        elif (
+            isinstance(examples, list) and len(inputs) == 1
+        ):  # If there is only one input component, examples can be provided as a regular list instead of a list of lists
+            examples = [[e] for e in examples]
+        elif isinstance(examples, str):
+            if not Path(examples).exists():
+                raise FileNotFoundError(
+                    "Could not find examples directory: " + examples
+                )
+            working_directory = examples
+            if not (Path(examples) / LOG_FILE).exists():
+                if len(inputs) == 1:
+                    examples = [[e] for e in os.listdir(examples)]
+                else:
+                    raise FileNotFoundError(
+                        "Could not find log file (required for multiple inputs): "
+                        + LOG_FILE
+                    )
+            else:
+                with open(Path(examples) / LOG_FILE) as logs:
+                    examples = list(csv.reader(logs))
+                    examples = [
+                        examples[i][: len(inputs)] for i in range(1, len(examples))
+                    ]  # remove header and unnecessary columns
+
+        else:
+            raise ValueError(
+                "The parameter `examples` must either be a string directory or a list"
+                "(if there is only 1 input component) or (more generally), a nested "
+                "list, where each sublist represents a set of inputs."
+            )
+
+        input_has_examples = [False] * len(inputs)
+        for example in examples:
+            for idx, example_for_input in enumerate(example):
+                if not (example_for_input is None):
+                    try:
+                        input_has_examples[idx] = True
+                    except IndexError:
+                        pass  # If there are more example components than inputs, ignore. This can sometimes be intentional (e.g. loading from a log file where outputs and timestamps are also logged)
+
+        inputs_with_examples = [
+            inp for (inp, keep) in zip(inputs, input_has_examples) if keep
+        ]
+        non_none_examples = [
+            [ex for (ex, keep) in zip(example, input_has_examples) if keep]
+            for example in examples
+        ]
+
+        self.examples = examples
+        self.non_none_examples = non_none_examples
+        self.inputs = inputs
+        self.inputs_with_examples = inputs_with_examples
+        self.outputs = outputs
+        self.fn = fn
+        self.cache_examples = cache_examples
+        self._api_mode = _api_mode
+        self.preprocess = preprocess
+        self.postprocess = postprocess
+        self.batch = batch
+
+        with utils.set_directory(working_directory):
+            self.processed_examples = [
+                [
+                    component.postprocess(sample)
+                    for component, sample in zip(inputs, example)
+                ]
+                for example in examples
+            ]
+        self.non_none_processed_examples = [
+            [ex for (ex, keep) in zip(example, input_has_examples) if keep]
+            for example in self.processed_examples
+        ]
+        if cache_examples:
+            for example in self.examples:
+                if len([ex for ex in example if ex is not None]) != len(self.inputs):
+                    warnings.warn(
+                        "Examples are being cached but not all input components have "
+                        "example values. This may result in an exception being thrown by "
+                        "your function. If you do get an error while caching examples, make "
+                        "sure all of your inputs have example values for all of your examples "
+                        "or you provide default values for those particular parameters in your function."
+                    )
+                    break
+
+        with utils.set_directory(working_directory):
+            self.dataset = components.Dataset(
+                components=inputs_with_examples,
+                samples=non_none_examples,
+                type="index",
+                label=label,
+                samples_per_page=examples_per_page,
+                elem_id=elem_id,
+            )
+
+        self.cached_folder = Path(CACHED_FOLDER) / str(self.dataset._id)
+        self.cached_file = Path(self.cached_folder) / "log.csv"
+        self.cache_examples = cache_examples
+        self.run_on_click = run_on_click
+
+    async def create(self) -> None:
+        """Caches the examples if self.cache_examples is True and creates the Dataset
+        component to hold the examples"""
+
+        async def load_example(example_id):
+            # import pdb; pdb.set_trace()
+            if self.cache_examples:
+                processed_example = self.non_none_processed_examples[
+                    example_id
+                ] + await self.load_from_cache(example_id)
+            else:
+                processed_example = self.non_none_processed_examples[example_id]
+            return utils.resolve_singleton(processed_example)
+
+        if Context.root_block:
+            if self.cache_examples and self.outputs:
+                targets = self.inputs_with_examples + self.outputs
+            else:
+                targets = self.inputs_with_examples
+            self.dataset.click(
+                load_example,
+                inputs=[self.dataset],
+                outputs=targets,  # type: ignore
+                postprocess=False,
+                queue=False,
+            )
+            self.dataset.click(
+                self.fn,
+                inputs=[self.dataset],
+                outputs=targets,  # type: ignore
+                postprocess=False,
+                queue=False,
+            )
+            # if self.run_on_click and not self.cache_examples:
+            #     if self.fn is None:
+            #         raise ValueError("Cannot run_on_click if no function is provided")
+            #     self.dataset.click(
+            #         self.fn,
+            #         inputs=self.inputs,  # type: ignore
+            #         outputs=self.outputs,  # type: ignore
+            #     )
+
+        if self.cache_examples:
+            await self.cache()
+
+    async def cache(self) -> None:
+        """
+        Caches all of the examples so that their predictions can be shown immediately.
+        """
+        if Path(self.cached_file).exists():
+            print(
+                f"Using cache from '{utils.abspath(self.cached_folder)}' directory. If method or examples have changed since last caching, delete this folder to clear cache."
+            )
+        else:
+            if Context.root_block is None:
+                raise ValueError("Cannot cache examples if not in a Blocks context")
+
+            print(f"Caching examples at: '{utils.abspath(self.cached_folder)}'")
+            cache_logger = CSVLogger()
+
+            # create a fake dependency to process the examples and get the predictions
+            dependency = Context.root_block.set_event_trigger(
+                event_name="fake_event",
+                fn=self.fn,
+                inputs=self.inputs_with_examples,  # type: ignore
+                outputs=self.outputs,  # type: ignore
+                preprocess=self.preprocess and not self._api_mode,
+                postprocess=self.postprocess and not self._api_mode,
+                batch=self.batch,
+            )
+
+            fn_index = Context.root_block.dependencies.index(dependency)
+            assert self.outputs is not None
+            cache_logger.setup(self.outputs, self.cached_folder)
+            for example_id, _ in enumerate(self.examples):
+                processed_input = self.processed_examples[example_id]
+                if self.batch:
+                    processed_input = [[value] for value in processed_input]
+                prediction = await Context.root_block.process_api(
+                    fn_index=fn_index, inputs=processed_input, request=None, state={}
+                )
+                output = prediction["data"]
+                if self.batch:
+                    output = [value[0] for value in output]
+                cache_logger.flag(output)
+            # Remove the "fake_event" to prevent bugs in loading interfaces from spaces
+            Context.root_block.dependencies.remove(dependency)
+            Context.root_block.fns.pop(fn_index)
+
+    async def load_from_cache(self, example_id: int) -> List[Any]:
+        """Loads a particular cached example for the interface.
+        Parameters:
+            example_id: The id of the example to process (zero-indexed).
+        """
+        # import pdb; pdb.set_trace()
+        with open(self.cached_file, encoding="utf-8") as cache:
+            examples = list(csv.reader(cache))
+        example = examples[example_id + 1]  # +1 to adjust for header
+        output = []
+        assert self.outputs is not None
+        for component, value in zip(self.outputs, example):
+            try:
+                value_as_dict = ast.literal_eval(value)
+                assert utils.is_update(value_as_dict)
+                output.append(value_as_dict)
+            except (ValueError, TypeError, SyntaxError, AssertionError):
+                output.append(component.serialize(value, self.cached_folder))
+        return output
+
+
+class TrackedIterable:
+    def __init__(
+        self,
+        iterable: Iterable | None,
+        index: int | None,
+        length: int | None,
+        desc: str | None,
+        unit: str | None,
+        _tqdm=None,
+        progress: float | None = None,
+    ) -> None:
+        self.iterable = iterable
+        self.index = index
+        self.length = length
+        self.desc = desc
+        self.unit = unit
+        self._tqdm = _tqdm
+        self.progress = progress
+
+
+@document("__call__", "tqdm")
+class Progress(Iterable):
+    """
+    The Progress class provides a custom progress tracker that is used in a function signature.
+    To attach a Progress tracker to a function, simply add a parameter right after the input parameters that has a default value set to a `gradio.Progress()` instance.
+    The Progress tracker can then be updated in the function by calling the Progress object or using the `tqdm` method on an Iterable.
+    The Progress tracker is currently only available with `queue()`.
+    Example:
+        import gradio as gr
+        import time
+        def my_function(x, progress=gr.Progress()):
+            progress(0, desc="Starting...")
+            time.sleep(1)
+            for i in progress.tqdm(range(100)):
+                time.sleep(0.1)
+            return x
+        gr.Interface(my_function, gr.Textbox(), gr.Textbox()).queue().launch()
+    Demos: progress
+    """
+
+    def __init__(
+        self,
+        track_tqdm: bool = False,
+        _callback: Callable | None = None,  # for internal use only
+        _event_id: str | None = None,
+    ):
+        """
+        Parameters:
+            track_tqdm: If True, the Progress object will track any tqdm.tqdm iterations with the tqdm library in the function.
+        """
+        self.track_tqdm = track_tqdm
+        self._callback = _callback
+        self._event_id = _event_id
+        self.iterables: List[TrackedIterable] = []
+
+    def __len__(self):
+        return self.iterables[-1].length
+
+    def __iter__(self):
+        return self
+
+    def __next__(self):
+        """
+        Updates progress tracker with next item in iterable.
+        """
+        if self._callback:
+            current_iterable = self.iterables[-1]
+            while (
+                not hasattr(current_iterable.iterable, "__next__")
+                and len(self.iterables) > 0
+            ):
+                current_iterable = self.iterables.pop()
+            self._callback(
+                event_id=self._event_id,
+                iterables=self.iterables,
+            )
+            assert current_iterable.index is not None, "Index not set."
+            current_iterable.index += 1
+            try:
+                return next(current_iterable.iterable)  # type: ignore
+            except StopIteration:
+                self.iterables.pop()
+                raise StopIteration
+        else:
+            return self
+
+    def __call__(
+        self,
+        progress: float | Tuple[int, int | None] | None,
+        desc: str | None = None,
+        total: int | None = None,
+        unit: str = "steps",
+        _tqdm=None,
+    ):
+        """
+        Updates progress tracker with progress and message text.
+        Parameters:
+            progress: If float, should be between 0 and 1 representing completion. If Tuple, first number represents steps completed, and second value represents total steps or None if unknown. If None, hides progress bar.
+            desc: description to display.
+            total: estimated total number of steps.
+            unit: unit of iterations.
+        """
+        if self._callback:
+            if isinstance(progress, tuple):
+                index, total = progress
+                progress = None
+            else:
+                index = None
+            self._callback(
+                event_id=self._event_id,
+                iterables=self.iterables
+                + [TrackedIterable(None, index, total, desc, unit, _tqdm, progress)],
+            )
+        else:
+            return progress
+
+    def tqdm(
+        self,
+        iterable: Iterable | None,
+        desc: str | None = None,
+        total: int | None = None,
+        unit: str = "steps",
+        _tqdm=None,
+        *args,
+        **kwargs,
+    ):
+        """
+        Attaches progress tracker to iterable, like tqdm.
+        Parameters:
+            iterable: iterable to attach progress tracker to.
+            desc: description to display.
+            total: estimated total number of steps.
+            unit: unit of iterations.
+        """
+        if self._callback:
+            if iterable is None:
+                new_iterable = TrackedIterable(None, 0, total, desc, unit, _tqdm)
+                self.iterables.append(new_iterable)
+                self._callback(event_id=self._event_id, iterables=self.iterables)
+                return self
+            length = len(iterable) if hasattr(iterable, "__len__") else None  # type: ignore
+            self.iterables.append(
+                TrackedIterable(iter(iterable), 0, length, desc, unit, _tqdm)
+            )
+        return self
+
+    def update(self, n=1):
+        """
+        Increases latest iterable with specified number of steps.
+        Parameters:
+            n: number of steps completed.
+        """
+        if self._callback and len(self.iterables) > 0:
+            current_iterable = self.iterables[-1]
+            assert current_iterable.index is not None, "Index not set."
+            current_iterable.index += n
+            self._callback(
+                event_id=self._event_id,
+                iterables=self.iterables,
+            )
+        else:
+            return
+
+    def close(self, _tqdm):
+        """
+        Removes iterable with given _tqdm.
+        """
+        if self._callback:
+            for i in range(len(self.iterables)):
+                if id(self.iterables[i]._tqdm) == id(_tqdm):
+                    self.iterables.pop(i)
+                    break
+            self._callback(
+                event_id=self._event_id,
+                iterables=self.iterables,
+            )
+        else:
+            return
+
+
+def create_tracker(root_blocks, event_id, fn, track_tqdm):
+
+    progress = Progress(_callback=root_blocks._queue.set_progress, _event_id=event_id)
+    if not track_tqdm:
+        return progress, fn
+
+    try:
+        _tqdm = __import__("tqdm")
+    except ModuleNotFoundError:
+        return progress, fn
+    if not hasattr(root_blocks, "_progress_tracker_per_thread"):
+        root_blocks._progress_tracker_per_thread = {}
+
+    def init_tqdm(self, iterable=None, desc=None, *args, **kwargs):
+        self._progress = root_blocks._progress_tracker_per_thread.get(
+            threading.get_ident()
+        )
+        if self._progress is not None:
+            self._progress.event_id = event_id
+            self._progress.tqdm(iterable, desc, _tqdm=self, *args, **kwargs)
+            kwargs["file"] = open(os.devnull, "w")
+        self.__init__orig__(iterable, desc, *args, **kwargs)
+
+    def iter_tqdm(self):
+        if self._progress is not None:
+            return self._progress
+        else:
+            return self.__iter__orig__()
+
+    def update_tqdm(self, n=1):
+        if self._progress is not None:
+            self._progress.update(n)
+        return self.__update__orig__(n)
+
+    def close_tqdm(self):
+        if self._progress is not None:
+            self._progress.close(self)
+        return self.__close__orig__()
+
+    def exit_tqdm(self, exc_type, exc_value, traceback):
+        if self._progress is not None:
+            self._progress.close(self)
+        return self.__exit__orig__(exc_type, exc_value, traceback)
+
+    if not hasattr(_tqdm.tqdm, "__init__orig__"):
+        _tqdm.tqdm.__init__orig__ = _tqdm.tqdm.__init__
+    _tqdm.tqdm.__init__ = init_tqdm
+    if not hasattr(_tqdm.tqdm, "__update__orig__"):
+        _tqdm.tqdm.__update__orig__ = _tqdm.tqdm.update
+    _tqdm.tqdm.update = update_tqdm
+    if not hasattr(_tqdm.tqdm, "__close__orig__"):
+        _tqdm.tqdm.__close__orig__ = _tqdm.tqdm.close
+    _tqdm.tqdm.close = close_tqdm
+    if not hasattr(_tqdm.tqdm, "__exit__orig__"):
+        _tqdm.tqdm.__exit__orig__ = _tqdm.tqdm.__exit__
+    _tqdm.tqdm.__exit__ = exit_tqdm
+    if not hasattr(_tqdm.tqdm, "__iter__orig__"):
+        _tqdm.tqdm.__iter__orig__ = _tqdm.tqdm.__iter__
+    _tqdm.tqdm.__iter__ = iter_tqdm
+    if hasattr(_tqdm, "auto") and hasattr(_tqdm.auto, "tqdm"):
+        _tqdm.auto.tqdm = _tqdm.tqdm
+
+    def tracked_fn(*args):
+        thread_id = threading.get_ident()
+        root_blocks._progress_tracker_per_thread[thread_id] = progress
+        response = fn(*args)
+        del root_blocks._progress_tracker_per_thread[thread_id]
+        return response
+
+    return progress, tracked_fn
+
+
+def special_args(
+    fn: Callable,
+    inputs: List[Any] | None = None,
+    request: routes.Request | None = None,
+):
+    """
+    Checks if function has special arguments Request (via annotation) or Progress (via default value).
+    If inputs is provided, these values will be loaded into the inputs array.
+    Parameters:
+        block_fn: function to check.
+        inputs: array to load special arguments into.
+        request: request to load into inputs.
+    Returns:
+        updated inputs, request index, progress index
+    """
+    signature = inspect.signature(fn)
+    positional_args = []
+    for i, param in enumerate(signature.parameters.values()):
+        if param.kind not in (param.POSITIONAL_ONLY, param.POSITIONAL_OR_KEYWORD):
+            break
+        positional_args.append(param)
+    progress_index = None
+    for i, param in enumerate(positional_args):
+        if isinstance(param.default, Progress):
+            progress_index = i
+            if inputs is not None:
+                inputs.insert(i, param.default)
+        elif param.annotation == routes.Request:
+            if inputs is not None:
+                inputs.insert(i, request)
+    if inputs is not None:
+        while len(inputs) < len(positional_args):
+            i = len(inputs)
+            param = positional_args[i]
+            if param.default == param.empty:
+                warnings.warn("Unexpected argument. Filling with None.")
+                inputs.append(None)
+            else:
+                inputs.append(param.default)
+    return inputs or [], progress_index
+
+
+@document()
+def update(**kwargs) -> dict:
+    """
+    Updates component properties. When a function passed into a Gradio Interface or a Blocks events returns a typical value, it updates the value of the output component. But it is also possible to update the properties of an output component (such as the number of lines of a `Textbox` or the visibility of an `Image`) by returning the component's `update()` function, which takes as parameters any of the constructor parameters for that component.
+    This is a shorthand for using the update method on a component.
+    For example, rather than using gr.Number.update(...) you can just use gr.update(...).
+    Note that your editor's autocompletion will suggest proper parameters
+    if you use the update method on the component.
+    Demos: blocks_essay, blocks_update, blocks_essay_update
+
+    Parameters:
+        kwargs: Key-word arguments used to update the component's properties.
+    Example:
+        # Blocks Example
+        import gradio as gr
+        with gr.Blocks() as demo:
+            radio = gr.Radio([1, 2, 4], label="Set the value of the number")
+            number = gr.Number(value=2, interactive=True)
+            radio.change(fn=lambda value: gr.update(value=value), inputs=radio, outputs=number)
+        demo.launch()
+
+        # Interface example
+        import gradio as gr
+        def change_textbox(choice):
+          if choice == "short":
+              return gr.Textbox.update(lines=2, visible=True)
+          elif choice == "long":
+              return gr.Textbox.update(lines=8, visible=True)
+          else:
+              return gr.Textbox.update(visible=False)
+        gr.Interface(
+          change_textbox,
+          gr.Radio(
+              ["short", "long", "none"], label="What kind of essay would you like to write?"
+          ),
+          gr.Textbox(lines=2),
+          live=True,
+        ).launch()
+    """
+    kwargs["__type__"] = "generic_update"
+    return kwargs
+
+
+def skip() -> dict:
+    return update()
+
+
+@document()
+def make_waveform(
+    audio: str | Tuple[int, np.ndarray],
+    *,
+    bg_color: str = "#f3f4f6",
+    bg_image: str | None = None,
+    fg_alpha: float = 0.75,
+    bars_color: str | Tuple[str, str] = ("#fbbf24", "#ea580c"),
+    bar_count: int = 50,
+    bar_width: float = 0.6,
+):
+    """
+    Generates a waveform video from an audio file. Useful for creating an easy to share audio visualization. The output should be passed into a `gr.Video` component.
+    Parameters:
+        audio: Audio file path or tuple of (sample_rate, audio_data)
+        bg_color: Background color of waveform (ignored if bg_image is provided)
+        bg_image: Background image of waveform
+        fg_alpha: Opacity of foreground waveform
+        bars_color: Color of waveform bars. Can be a single color or a tuple of (start_color, end_color) of gradient
+        bar_count: Number of bars in waveform
+        bar_width: Width of bars in waveform. 1 represents full width, 0.5 represents half width, etc.
+    Returns:
+        A filepath to the output video.
+    """
+    if isinstance(audio, str):
+        audio_file = audio
+        audio = processing_utils.audio_from_file(audio)
+    else:
+        tmp_wav = tempfile.NamedTemporaryFile(suffix=".wav", delete=False)
+        processing_utils.audio_to_file(audio[0], audio[1], tmp_wav.name)
+        audio_file = tmp_wav.name
+    duration = round(len(audio[1]) / audio[0], 4)
+
+    # Helper methods to create waveform
+    def hex_to_RGB(hex_str):
+        return [int(hex_str[i : i + 2], 16) for i in range(1, 6, 2)]
+
+    def get_color_gradient(c1, c2, n):
+        assert n > 1
+        c1_rgb = np.array(hex_to_RGB(c1)) / 255
+        c2_rgb = np.array(hex_to_RGB(c2)) / 255
+        mix_pcts = [x / (n - 1) for x in range(n)]
+        rgb_colors = [((1 - mix) * c1_rgb + (mix * c2_rgb)) for mix in mix_pcts]
+        return [
+            "#" + "".join([format(int(round(val * 255)), "02x") for val in item])
+            for item in rgb_colors
+        ]
+
+    # Reshape audio to have a fixed number of bars
+    samples = audio[1]
+    if len(samples.shape) > 1:
+        samples = np.mean(samples, 1)
+    bins_to_pad = bar_count - (len(samples) % bar_count)
+    samples = np.pad(samples, [(0, bins_to_pad)])
+    samples = np.reshape(samples, (bar_count, -1))
+    samples = np.abs(samples)
+    samples = np.max(samples, 1)
+
+    matplotlib.use("Agg")
+    plt.clf()
+    # Plot waveform
+    color = (
+        bars_color
+        if isinstance(bars_color, str)
+        else get_color_gradient(bars_color[0], bars_color[1], bar_count)
+    )
+    plt.bar(
+        np.arange(0, bar_count),
+        samples * 2,
+        bottom=(-1 * samples),
+        width=bar_width,
+        color=color,
+    )
+    plt.axis("off")
+    plt.margins(x=0)
+    tmp_img = tempfile.NamedTemporaryFile(suffix=".png", delete=False)
+    savefig_kwargs: Dict[str, Any] = {"bbox_inches": "tight"}
+    if bg_image is not None:
+        savefig_kwargs["transparent"] = True
+    else:
+        savefig_kwargs["facecolor"] = bg_color
+    plt.savefig(tmp_img.name, **savefig_kwargs)
+    waveform_img = PIL.Image.open(tmp_img.name)
+    waveform_img = waveform_img.resize((1000, 200))
+
+    # Composite waveform with background image
+    if bg_image is not None:
+        waveform_array = np.array(waveform_img)
+        waveform_array[:, :, 3] = waveform_array[:, :, 3] * fg_alpha
+        waveform_img = PIL.Image.fromarray(waveform_array)
+
+        bg_img = PIL.Image.open(bg_image)
+        waveform_width, waveform_height = waveform_img.size
+        bg_width, bg_height = bg_img.size
+        if waveform_width != bg_width:
+            bg_img = bg_img.resize(
+                (waveform_width, 2 * int(bg_height * waveform_width / bg_width / 2))
+            )
+            bg_width, bg_height = bg_img.size
+        composite_height = max(bg_height, waveform_height)
+        composite = PIL.Image.new("RGBA", (waveform_width, composite_height), "#FFFFFF")
+        composite.paste(bg_img, (0, composite_height - bg_height))
+        composite.paste(
+            waveform_img, (0, composite_height - waveform_height), waveform_img
+        )
+        composite.save(tmp_img.name)
+        img_width, img_height = composite.size
+    else:
+        img_width, img_height = waveform_img.size
+        waveform_img.save(tmp_img.name)
+
+    # Convert waveform to video with ffmpeg
+    output_mp4 = tempfile.NamedTemporaryFile(suffix=".mp4", delete=False)
+
+    ffmpeg_cmd = f"""ffmpeg -loop 1 -i {tmp_img.name} -i {audio_file} -vf "color=c=#FFFFFF77:s={img_width}x{img_height}[bar];[0][bar]overlay=-w+(w/{duration})*t:H-h:shortest=1" -t {duration} -y {output_mp4.name}"""
+
+    subprocess.call(ffmpeg_cmd, shell=True)
+    return output_mp4.name

gligen/SD_input_conv_weight_bias.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b5a0efad69747a766158304f39091c2b6a24cafb5f833d174f32bee8e864a562
+size 130

gligen/evaluator.py

@@ -14,7 +14,7 @@ from trainer import read_official_ckpt, batch_to_device, ImageCaptionSaver, wrap
 from PIL import Image
 import math
 import json
-
+#hello
 
 def draw_masks_from_boxes(boxes,size):
 

gligen/ldm/data/imagenet_train_hr_indices.p

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f86ea1924a1522b20bc0f709a069cc65f09d5fc617a7a31af7aaa3839a5a4d73
+size 132

gligen/ldm/data/imagenet_val_hr_indices.p

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ff1f5eb275a93c0fb53e227679f323ea1d024c87db296453296cebeef86fc0f4
+size 131

gligen/ldm/models/diffusion/ddim.py

@@ -87,7 +87,9 @@ class DDIMSampler(object):
             # set alpha 
             if self.alpha_generator_func != None:
                 self.set_alpha_scale(self.model, alphas[i])
-
+                if  alphas[i] == 0:
+                    self.model.restore_first_conv_from_SD()
+                    
             # run 
             index = total_steps - i - 1
             input["timesteps"] = torch.full((b,), step, device=self.device, dtype=torch.long)
@@ -110,9 +112,7 @@ class DDIMSampler(object):
 
         e_t = self.model(input) 
         if uc is not None and guidance_scale != 1:
-            unconditional_input = dict(x=input["x"], timesteps=input["timesteps"], context=uc)
-            if "inpainting_extra_input" in input:
-                unconditional_input["inpainting_extra_input"] = input["inpainting_extra_input"]
+            unconditional_input = dict(x=input["x"], timesteps=input["timesteps"], context=uc, inpainting_extra_input=input["inpainting_extra_input"], grounding_extra_input=input['grounding_extra_input'])
             e_t_uncond = self.model( unconditional_input ) 
             e_t = e_t_uncond + guidance_scale * (e_t - e_t_uncond)