app.py

import gradio as gr
import os
import torch
from omegaconf import OmegaConf
from gligen.task_grounded_generation import grounded_generation_box, load_ckpt, load_common_ckpt

import json
import numpy as np
from PIL import Image, ImageDraw, ImageFont
from functools import partial
from collections import Counter
import math
import gc

from gradio import processing_utils
from typing import Optional

import warnings

from datetime import datetime

from example_component import create_examples

from huggingface_hub import hf_hub_download
hf_hub_download = partial(hf_hub_download, library_name="gligen_demo")
import cv2
import sys
sys.tracebacklimit = 0


def load_from_hf(repo_id, filename='diffusion_pytorch_model.bin', subfolder=None):
    cache_file = hf_hub_download(repo_id=repo_id, filename=filename, subfolder=subfolder)
    return torch.load(cache_file, map_location='cpu')

def load_ckpt_config_from_hf(modality):
    ckpt = load_from_hf('gligen/demo_ckpts_legacy', filename=f'{modality}.pth', subfolder='model')
    config = load_from_hf('gligen/demo_ckpts_legacy', filename=f'{modality}.pth', subfolder='config')
    return ckpt, config


def ckpt_load_helper(modality, is_inpaint, is_style, common_instances=None):
    pretrained_ckpt_gligen, config = load_ckpt_config_from_hf(modality)
    config = OmegaConf.create( config["_content"] ) # config used in training
    config.alpha_scale = 1.0

    if common_instances is None:
        common_ckpt = load_from_hf('gligen/demo_ckpts_legacy', filename=f'common.pth', subfolder='model')
        common_instances = load_common_ckpt(config, common_ckpt)

    loaded_model_list = load_ckpt(config, pretrained_ckpt_gligen, common_instances)

    return loaded_model_list, common_instances


class Instance:
    def __init__(self, capacity = 2):
        self.model_type = 'base'
        self.loaded_model_list = {}
        self.counter = Counter()
        self.global_counter = Counter()
        self.loaded_model_list['base'], self.common_instances = ckpt_load_helper(
            'gligen-generation-text-box',
            is_inpaint=False, is_style=False, common_instances=None
        )
        self.capacity = capacity

    def _log(self, model_type, batch_size, instruction, phrase_list):
        self.counter[model_type] += 1
        self.global_counter[model_type] += 1
        current_time = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
        print('[{}] Current: {}, All: {}. Samples: {}, prompt: {}, phrases: {}'.format(
            current_time, dict(self.counter), dict(self.global_counter), batch_size, instruction, phrase_list
        ))

    def get_model(self, model_type, batch_size, instruction, phrase_list):
        if model_type in self.loaded_model_list:
            self._log(model_type, batch_size, instruction, phrase_list)
            return self.loaded_model_list[model_type]

        if self.capacity == len(self.loaded_model_list):
            least_used_type = self.counter.most_common()[-1][0]
            del self.loaded_model_list[least_used_type]
            del self.counter[least_used_type]
            gc.collect()
            torch.cuda.empty_cache()

        self.loaded_model_list[model_type] = self._get_model(model_type)
        self._log(model_type, batch_size, instruction, phrase_list)
        return self.loaded_model_list[model_type]

    def _get_model(self, model_type):
        if model_type == 'base':
            return ckpt_load_helper(
                'gligen-generation-text-box',
                is_inpaint=False, is_style=False, common_instances=self.common_instances
            )[0]
        elif model_type == 'inpaint':
            return ckpt_load_helper(
                'gligen-inpainting-text-box',
                is_inpaint=True, is_style=False, common_instances=self.common_instances
            )[0]
        elif model_type == 'style':
            return ckpt_load_helper(
                'gligen-generation-text-image-box',
                is_inpaint=False, is_style=True, common_instances=self.common_instances
            )[0]
        
        assert False

instance = Instance()


def load_clip_model():
    from transformers import CLIPProcessor, CLIPModel
    version = "openai/clip-vit-large-patch14"
    model = CLIPModel.from_pretrained(version).cuda()
    processor = CLIPProcessor.from_pretrained(version)

    return {
        'version': version,
        'model': model,
        'processor': processor,
    }

clip_model = load_clip_model()


class ImageMask(gr.components.Image):
    """
    Sets: source="canvas", tool="sketch"
    """

    is_template = True

    def __init__(self, **kwargs):
        super().__init__(source="upload", tool="sketch", interactive=True, **kwargs)

    def preprocess(self, x):
        if x is None:
            return x
        if self.tool == "sketch" and self.source in ["upload", "webcam"] and type(x) != dict:
           
            decode_image = processing_utils.decode_base64_to_image(x)
            print('decode to 64')
            width, height = decode_image.size
            img = np.asarray(decode_image)
            return {'image':img, 'mask':binarize_2(img)}
            
            mask = np.zeros((height, width, 4), dtype=np.uint8)
            
            mask[..., -1] = 255
            mask = self.postprocess(mask)
            x = {'image': x, 'mask': mask}
            print('vao preprocess-------------------------')
        hh = super().preprocess(x)
        if (hh['image'].min()!=255) and (hh['mask'][:,:,:3].max()==0):
            
            hh['mask'] = binarize_2(hh['image'])
        
        return hh


class Blocks(gr.Blocks):

    def __init__(
        self,
        theme: str = "default",
        analytics_enabled: Optional[bool] = None,
        mode: str = "blocks",
        title: str = "Gradio",
        css: Optional[str] = None,
        **kwargs,
    ):

        self.extra_configs = {
            'thumbnail': kwargs.pop('thumbnail', ''),
            'url': kwargs.pop('url', 'https://gradio.app/'),
            'creator': kwargs.pop('creator', '@teamGradio'),
        }

        super(Blocks, self).__init__(theme, analytics_enabled, mode, title, css, **kwargs)
        warnings.filterwarnings("ignore")

    def get_config_file(self):
        config = super(Blocks, self).get_config_file()

        for k, v in self.extra_configs.items():
            config[k] = v
        
        return config

'''
inference model
'''

# @torch.no_grad()
def inference(task, language_instruction, phrase_list, location_list, inpainting_boxes_nodrop, image,
              alpha_sample, guidance_scale, batch_size,
              fix_seed, rand_seed, actual_mask, style_image,
              *args, **kwargs):
    # import pdb; pdb.set_trace()
    
    # grounding_instruction = json.loads(grounding_instruction)
    # phrase_list, location_list = [], []
    # for k, v  in grounding_instruction.items():
    #     phrase_list.append(k)
    #     location_list.append(v)

    placeholder_image = Image.open('images/teddy.jpg').convert("RGB")    
    image_list = [placeholder_image] * len(phrase_list) # placeholder input for visual prompt, which is disabled

    batch_size = int(batch_size)
    if not 1 <= batch_size <= 4:
        batch_size = 1

    if style_image == None:
        has_text_mask = 1 
        has_image_mask = 0 # then we hack above 'image_list' 
    else:
        valid_phrase_len = len(phrase_list)

        phrase_list += ['placeholder']
        has_text_mask = [1]*valid_phrase_len + [0]

        image_list = [placeholder_image]*valid_phrase_len + [style_image]
        has_image_mask = [0]*valid_phrase_len + [1]
        
        location_list += [ [0.0, 0.0, 1, 0.01]  ] # style image grounding location

    instruction = dict(
        prompt = language_instruction,
        phrases = phrase_list,
        images = image_list,
        locations = location_list,
        alpha_type = [alpha_sample, 0, 1.0 - alpha_sample], 
        has_text_mask = has_text_mask,
        has_image_mask = has_image_mask,
        save_folder_name = language_instruction,
        guidance_scale = guidance_scale,
        batch_size = batch_size,
        fix_seed = bool(fix_seed),
        rand_seed = int(rand_seed),
        actual_mask = actual_mask,
        inpainting_boxes_nodrop = inpainting_boxes_nodrop,
    )

    get_model = partial(instance.get_model,
                        batch_size=batch_size,
                        instruction=language_instruction,
                        phrase_list=phrase_list)

    with torch.autocast(device_type='cuda', dtype=torch.float16):
        if task == 'User provide boxes' or 'Available boxes':
            if style_image == None:
                result = grounded_generation_box(get_model('base'), instruction, *args, **kwargs)
                torch.cuda.empty_cache()
                return result
            else:
                return grounded_generation_box(get_model('style'), instruction, *args, **kwargs)


def draw_box(boxes=[], texts=[], img=None):
    if len(boxes) == 0 and img is None:
        return None
    
    if img is None:
        img = Image.new('RGB', (512, 512), (255, 255, 255))
    colors = ["red", "olive", "blue", "green", "orange", "brown", "cyan", "purple"]
    draw = ImageDraw.Draw(img)
    font = ImageFont.truetype("DejaVuSansMono.ttf", size=18)
    for bid, box in enumerate(boxes):
        draw.rectangle([box[0], box[1], box[2], box[3]], outline=colors[bid % len(colors)], width=4)
        anno_text = texts[bid]
        draw.rectangle([box[0], box[3] - int(font.size * 1.2), box[0] + int((len(anno_text) + 0.8) * font.size * 0.6), box[3]], outline=colors[bid % len(colors)], fill=colors[bid % len(colors)], width=4)
        draw.text([box[0] + int(font.size * 0.2), box[3] - int(font.size*1.2)], anno_text, font=font, fill=(255,255,255))
    return img

def get_concat(ims):
    if len(ims) == 1:
        n_col = 1
    else:
        n_col = 2
    n_row = math.ceil(len(ims) / 2)
    dst = Image.new('RGB', (ims[0].width * n_col, ims[0].height * n_row), color="white")
    for i, im in enumerate(ims):
        row_id = i // n_col
        col_id = i % n_col
        dst.paste(im, (im.width * col_id, im.height * row_id))
    return dst


def auto_append_grounding(language_instruction, grounding_texts):
    for grounding_text in grounding_texts:
        if grounding_text.lower() not in language_instruction.lower() and grounding_text != 'auto':
            language_instruction += "; " + grounding_text
    return language_instruction




def generate(task, language_instruction, grounding_texts, sketch_pad,
             alpha_sample, guidance_scale, batch_size,
             fix_seed, rand_seed, use_actual_mask, append_grounding, style_cond_image,
             state):
    
    if 'boxes' not in state:
        state['boxes'] = []

    boxes = state['boxes']
    grounding_texts = [x.strip() for x in grounding_texts.split(';')]
    # assert len(boxes) == len(grounding_texts)
    if len(boxes) != len(grounding_texts):
        if len(boxes) < len(grounding_texts):
            raise ValueError("""The number of boxes should be equal to the number of grounding objects.
Number of boxes drawn: {}, number of grounding tokens: {}.
Please draw boxes accordingly on the sketch pad.""".format(len(boxes), len(grounding_texts)))
        grounding_texts = grounding_texts + [""] * (len(boxes) - len(grounding_texts))

    boxes = (np.asarray(boxes) / 512).tolist()
    grounding_instruction = json.dumps({obj: box for obj,box in zip(grounding_texts, boxes)})
    image = None
    actual_mask = None
  
    
    if append_grounding:
        language_instruction = auto_append_grounding(language_instruction, grounding_texts)

    gen_images, gen_overlays = inference(
        task, language_instruction, grounding_texts,boxes, boxes, image,
        alpha_sample, guidance_scale, batch_size,
        fix_seed, rand_seed, actual_mask, style_cond_image, clip_model=clip_model,
    )
    blank_samples = batch_size % 2 if batch_size > 1 else 0
    gen_images = [gr.Image.update(value=x, visible=True) for i,x in enumerate(gen_images)] \
                    + [gr.Image.update(value=None, visible=True) for _ in range(blank_samples)] \
                    + [gr.Image.update(value=None, visible=False) for _ in range(4 - batch_size - blank_samples)]

    return gen_images + [state]


def binarize(x):
    return (x != 0).astype('uint8') * 255
def binarize_2(x):
    gray_image = cv2.cvtColor(x, cv2.COLOR_BGR2GRAY)
    return (gray_image!=255).astype('uint8') * 255

def sized_center_crop(img, cropx, cropy):
    y, x = img.shape[:2]
    startx = x // 2 - (cropx // 2)
    starty = y // 2 - (cropy // 2)    
    return img[starty:starty+cropy, startx:startx+cropx]

def sized_center_fill(img, fill, cropx, cropy):
    y, x = img.shape[:2]
    startx = x // 2 - (cropx // 2)
    starty = y // 2 - (cropy // 2)    
    img[starty:starty+cropy, startx:startx+cropx] = fill
    return img

def sized_center_mask(img, cropx, cropy):
    y, x = img.shape[:2]
    startx = x // 2 - (cropx // 2)
    starty = y // 2 - (cropy // 2)    
    center_region = img[starty:starty+cropy, startx:startx+cropx].copy()
    img = (img * 0.2).astype('uint8')
    img[starty:starty+cropy, startx:startx+cropx] = center_region
    return img

def center_crop(img, HW=None, tgt_size=(512, 512)):
    if HW is None:
        H, W = img.shape[:2]
        HW = min(H, W)
    img = sized_center_crop(img, HW, HW)
    img = Image.fromarray(img)
    img = img.resize(tgt_size)
    return np.array(img)

# 接收 sketchpad 的输入 （左边）
def draw(task, input, grounding_texts, new_image_trigger, state, generate_parsed, box_image):
    print('input', generate_parsed)
   
    if type(input) == dict:
        image = input['image']
        mask = input['mask']
        if generate_parsed==1:
            generate_parsed = 0
            # import pdb; pdb.set_trace()
            print('do nothing')
            
            return [box_image, new_image_trigger, 1., state, generate_parsed]
           
    else:
        mask = input

    if mask.ndim == 3:
        mask = mask[..., 0]

    image_scale = 1.0
    
    print('vao draw--------------------')
    mask = binarize(mask)
    if mask.shape != (512, 512):
        # assert False, "should not receive any non- 512x512 masks."
        if 'original_image' in state and state['original_image'].shape[:2] == mask.shape:
            mask = center_crop(mask, state['inpaint_hw'])
            image = center_crop(state['original_image'], state['inpaint_hw'])
        else:
            mask = np.zeros((512, 512), dtype=np.uint8)
    mask = binarize(mask)

    if type(mask) != np.ndarray:
        mask = np.array(mask)
    # 
    if mask.sum() == 0:
        state = {}
        print('delete state')

    if True:
        image = None
    else:
        image = Image.fromarray(image)

    if 'boxes' not in state:
        state['boxes'] = []

    if 'masks' not in state or len(state['masks']) == 0 :
        state['masks'] = []
        last_mask = np.zeros_like(mask)
    else:
        last_mask = state['masks'][-1]
  
    if type(mask) == np.ndarray and mask.size > 1 :
        diff_mask = mask - last_mask
    else:
        diff_mask = np.zeros([])

    if diff_mask.sum() > 0:
        x1x2 = np.where(diff_mask.max(0) > 1)[0]
        y1y2 = np.where(diff_mask.max(1) > 1)[0]
        y1, y2 = y1y2.min(), y1y2.max()
        x1, x2 = x1x2.min(), x1x2.max()

        if (x2 - x1 > 5) and (y2 - y1 > 5):
            state['masks'].append(mask.copy())
            state['boxes'].append((x1, y1, x2, y2))

    grounding_texts = [x.strip() for x in grounding_texts.split(';')]
    grounding_texts = [x for x in grounding_texts if len(x) > 0]
    if len(grounding_texts) < len(state['boxes']):
        grounding_texts += [f'Obj. {bid+1}' for bid in range(len(grounding_texts), len(state['boxes']))]
    
    box_image = draw_box(state['boxes'], grounding_texts, image)
    generate_parsed = 0
    
    return [box_image, new_image_trigger, image_scale, state, generate_parsed]

def change_state(bboxes,layout, state, instruction, trigger_stage, boxes):
    if trigger_stage ==0 :
        return [boxes, state, 0]
    # mask = 
    state['boxes'] = []
    state['masks'] = []
    image = None
    list_boxes = bboxes.split('/')
    result =[]
    for b in list_boxes:
        ints = b[1:-1].split(',')
        l = []
        for i in ints:
            l.append(int(i))
        result.append(l)
    print('run change state')
   
    for box in result:
        state['boxes'].append(box)
    grounding_texts = [x.strip() for x in instruction.split(';')]
    grounding_texts = [x for x in grounding_texts if len(x) > 0]
    if len(grounding_texts) < len(result):
        grounding_texts += [f'Obj. {bid+1}' for bid in range(len(grounding_texts), len(result))]

    box_image = draw_box(result, grounding_texts)
    
    mask = binarize_2(layout['image'])
    state['masks'].append(mask.copy())
    # print('done change state', state)
    print('done change state')
    # import pdb; pdb.set_trace()
    return [box_image,state, trigger_stage]

def example_click(name, grounding_instruction, instruction, bboxes,generate_parsed,  trigger_parsed):
    
    list_boxes = bboxes.split('/')
    result =[]
   
    for b in list_boxes:
        ints = b[1:-1].split(',')
        l = []
        for i in ints:
            l.append(int(i))
        result.append(l)
    print('run change state')
    
    box_image = draw_box(result, instruction)
    trigger_parsed += 1
    print('done the example click')
    return [box_image, trigger_parsed]

def clear(task, sketch_pad_trigger, batch_size, state,trigger_stage, switch_task=False):
    
    sketch_pad_trigger = sketch_pad_trigger + 1
    trigger_stage = 0
    blank_samples = batch_size % 2 if batch_size > 1 else 0
    out_images = [gr.Image.update(value=None, visible=True) for i in range(batch_size)] \
                    + [gr.Image.update(value=None, visible=True) for _ in range(blank_samples)] \
                    + [gr.Image.update(value=None, visible=False) for _ in range(4 - batch_size - blank_samples)]
    state = {}
    return [None, sketch_pad_trigger, None, 1.0] + out_images + [state] + [trigger_stage]

css = """
#img2img_image, #img2img_image > .fixed-height, #img2img_image > .fixed-height > div, #img2img_image > .fixed-height > div > img
{
    height: var(--height) !important;
    max-height: var(--height) !important;
    min-height: var(--height) !important;
}
#paper-info a {
    color:#008AD7;
    text-decoration: none;
}
#paper-info a:hover {
    cursor: pointer;
    text-decoration: none;
}
#my_image > div.fixed-height 
{
    height: var(--height) !important;
}
"""

rescale_js = """
function(x) {
    const root = document.querySelector('gradio-app').shadowRoot || document.querySelector('gradio-app');
    let image_scale = parseFloat(root.querySelector('#image_scale input').value) || 1.0;
    const image_width = root.querySelector('#img2img_image').clientWidth;
    const target_height = parseInt(image_width * image_scale);
    document.body.style.setProperty('--height', `${target_height}px`);
    root.querySelectorAll('button.justify-center.rounded')[0].style.display='none';
    root.querySelectorAll('button.justify-center.rounded')[1].style.display='none';
    return x;
}
"""
# [<a href="https://arxiv.org/abs/2301.07093" target="_blank">Paper</a>]
with Blocks(
    css=css,
    analytics_enabled=False,
    title="Attention-refocusing demo",
) as main:
    description = """<p style="text-align: center; font-weight: bold;">
        <span style="font-size: 28px">Grounded Text-to-Image Synthesis with Attention Refocusing</span>
        <br>
        <span style="font-size: 18px" id="paper-info">
            [<a href="https://attention-refocusing.github.io/" target="_blank">Project Page</a>]
            
            [<a href="https://github.com/Attention-Refocusing/attention-refocusing" target="_blank">GitHub</a>]
        </span>
    </p>
    <p>
        To identify the areas of interest based on specific spatial parameters, you need to (1) &#9000;&#65039; input the names of the concepts you're interested  in <em> Grounding Instruction</em>, and (2) &#128433;&#65039; draw their corresponding bounding boxes using <em> Sketch Pad</em> -- the parsed boxes will automatically be showed up once you've drawn them.
        <br>
        For faster inference without waiting in queue, you may duplicate the space and upgrade to GPU in settings. <a href="https://huggingface.co/spaces/gligen/demo?duplicate=true"><img style="display: inline; margin-top: 0em; margin-bottom: 0em" src="https://bit.ly/3gLdBN6" alt="Duplicate Space" /></a>
    </p>
    """
    gr.HTML(description)
    
    with gr.Row():
        with gr.Column(scale=4):
            sketch_pad_trigger = gr.Number(value=0, visible=False)
            sketch_pad_resize_trigger = gr.Number(value=0, visible=False)
            trigger_stage = gr.Number(value=0, visible=False)
            
            init_white_trigger = gr.Number(value=0, visible=False)
            image_scale = gr.Number(value=1.0, elem_id="image_scale", visible=False)
            new_image_trigger = gr.Number(value=0, visible=False)
            text_box = gr.Textbox(visible=False)
            generate_parsed = gr.Number(value=0, visible=False)
            
            task = gr.Radio(
                choices=["Available boxes", 'User provide boxes'],
                type="value",
                value="User provide boxes",
                label="Task",
                visible=False
                
            )
            language_instruction = gr.Textbox(
                label="Language instruction",
            )
            grounding_instruction = gr.Textbox(
                label="Grounding instruction (Separated by semicolon)",
            )
            with gr.Row():
                sketch_pad = ImageMask(label="Sketch Pad", elem_id="img2img_image")
                out_imagebox = gr.Image(type="pil",elem_id="my_image" ,label="Parsed Sketch Pad", shape=(512,512))
            with gr.Row():
                clear_btn = gr.Button(value='Clear')
                gen_btn = gr.Button(value='Generate')
            with gr.Row():
                parsed_btn = gr.Button(value='generate parsed boxes')

            with gr.Accordion("Advanced Options", open=False):
                with gr.Column():
                    alpha_sample = gr.Slider(minimum=0, maximum=1.0, step=0.1, value=0.3, label="Scheduled Sampling (τ)")
                    guidance_scale = gr.Slider(minimum=0, maximum=50, step=0.5, value=7.5, label="Guidance Scale")
                    batch_size = gr.Slider(minimum=1, maximum=4,visible=False, step=1, value=1, label="Number of Samples")
                    append_grounding = gr.Checkbox(value=True, label="Append grounding instructions to the caption")
                    use_actual_mask = gr.Checkbox(value=False, label="Use actual mask for inpainting", visible=False)
                    with gr.Row():
                        fix_seed = gr.Checkbox(value=True, label="Fixed seed")
                        rand_seed = gr.Slider(minimum=0, maximum=1000, step=1, value=0, label="Seed")
                   
                with gr.Row():
                        use_style_cond = gr.Checkbox(value=False,visible=False, label="Enable Style Condition")
                        style_cond_image = gr.Image(type="pil",visible=False, label="Style Condition", interactive=True)
        with gr.Column(scale=4):
            gr.HTML('<span style="font-size: 20px; font-weight: bold">Generated Images</span>')
            with gr.Row():
                out_gen_1 = gr.Image(type="pil", visible=True, show_label=False)
                out_gen_2 = gr.Image(type="pil", visible=False, show_label=False)
            with gr.Row():
                out_gen_3 = gr.Image(type="pil", visible=False, show_label=False)
                out_gen_4 = gr.Image(type="pil", visible=False, show_label=False)

        state = gr.State({})
        

        class Controller:
            def __init__(self):
                self.calls = 0
                self.tracks = 0
                self.resizes = 0
                self.scales = 0

            def init_white(self, init_white_trigger):
                self.calls += 1
                return np.ones((512, 512), dtype='uint8') * 255, 1.0, init_white_trigger+1

            def change_n_samples(self, n_samples):
                blank_samples = n_samples % 2 if n_samples > 1 else 0
                return [gr.Image.update(visible=True) for _ in range(n_samples + blank_samples)] \
                    + [gr.Image.update(visible=False) for _ in range(4 - n_samples - blank_samples)]

        controller = Controller()
        main.load(
            lambda x:x+1,
            inputs=sketch_pad_trigger,
            outputs=sketch_pad_trigger,
            queue=False)
      
        sketch_pad.edit(
            draw,
            inputs=[task, sketch_pad, grounding_instruction, sketch_pad_resize_trigger, state, generate_parsed, out_imagebox],
            outputs=[out_imagebox, sketch_pad_resize_trigger, image_scale, state, generate_parsed],
            queue=False,
        )
        trigger_stage.change(
            change_state,
            inputs=[text_box,sketch_pad, state, grounding_instruction, trigger_stage,out_imagebox],
            outputs=[out_imagebox,state,trigger_stage],
            queue=True
        )
        grounding_instruction.change(
            draw,
            inputs=[task, sketch_pad, grounding_instruction, sketch_pad_resize_trigger, state, generate_parsed,out_imagebox],
            outputs=[out_imagebox, sketch_pad_resize_trigger, image_scale, state, generate_parsed],
            queue=False,
        )
        clear_btn.click(
            clear,
            inputs=[task, sketch_pad_trigger, batch_size,trigger_stage, state],
            outputs=[sketch_pad, sketch_pad_trigger, out_imagebox, image_scale, out_gen_1, out_gen_2, out_gen_3, out_gen_4, state, trigger_stage],
            queue=False)
    
        sketch_pad_trigger.change(
            controller.init_white,
            inputs=[init_white_trigger],
            outputs=[sketch_pad, image_scale, init_white_trigger],
            queue=False)

        gen_btn.click(
            generate,
            inputs=[
                task, language_instruction, grounding_instruction, sketch_pad,
                alpha_sample, guidance_scale, batch_size,
                fix_seed, rand_seed,
                use_actual_mask,
                append_grounding, style_cond_image,
                state,
            ],
            outputs=[out_gen_1, out_gen_2, out_gen_3, out_gen_4, state],
            queue=True
        )
        init_white_trigger.change(
            None,
            None,
            init_white_trigger,
            _js=rescale_js,
            queue=False)
        examples = [
        [
                    'guide_imgs/0_a_cat_on_the_right_of_a_dog.jpg',
                     "a cat;a dog",
                    "a cat on the right of a dog",
                    '(291, 88, 481, 301)/(25, 64, 260, 391)',
                    1, 1
                ],
                [
                    'guide_imgs/0_a_bus_on_the_left_of_a_car.jpg',#'guide_imgs/0_a_bus_on_the_left_of_a_car.jpg',
                     "a bus;a car",
                    "a bus and a car",
                    '(8,128,266,384)/(300,196,502,316)', #'(8,128,266,384)', #/(300,196,502,316)
                    1, 2
                ],
                [
                    'guide_imgs/1_Two_cars_on_the_street..jpg',
                     "a car;a car",
                    "Two cars on the street.",
                    '(34, 98, 247, 264)/(271, 122, 481, 293)',
                    1, 3
                ],
                [
                    'guide_imgs/80_two_apples_lay_side_by_side_on_a_wooden_table,_their_glossy_red_and_green_skins_glinting_in_the_sunlight..jpg',
                     "an apple;an apple",
                    "two apples lay side by side on a wooden table, their glossy red and green skins glinting in the sunlight.",
                    '(40, 210, 235, 450)/(275, 210, 470, 450)',
                    1, 4
                ],
                [
                    'guide_imgs/10_A_banana_on_the_left_of_an_apple..jpg',
                     "a banana;an apple",
                    "A banana on the left of an apple.",
                    '(62, 193, 225, 354)/(300, 184, 432, 329)',
                    1, 5
                ],
                [
                    'guide_imgs/15_A_pizza_on_the_right_of_a_suitcase..jpg',
                     "a pizza ;a suitcase",
                    "A pizza on the right of a suitcase.",
                    '(307, 112, 490, 280)/(41, 120, 244, 270)',
                    1, 6
                ],
                [
                    'guide_imgs/1_A_wine_glass_on_top_of_a_dog..jpg',
                     "a wine glass;a dog",
                    "A wine glass on top of a dog.",
                    '(206, 78, 306, 214)/(137, 222, 367, 432)',
                    1, 7
                ]
                ,
                [
                    'guide_imgs/2_A_bicycle_on_top_of_a_boat..jpg',
                     "a bicycle;a boat",
                    "A bicycle on top of a boat.",
                    '(185, 110, 335, 205)/(111, 228, 401, 373)',
                    1, 8
                ]
                ,
                [
                    'guide_imgs/4_A_laptop_on_top_of_a_teddy_bear..jpg',
                     "a laptop;a teddy bear",
                    "A laptop on top of a teddy bear.",
                    '(180, 70, 332, 210)/(150, 240, 362, 420)',
                    1, 9
                ]
                ,
                [
                    'guide_imgs/0_A_train_on_top_of_a_surfboard..jpg',
                     "a train;a surfboard",
                    "A train on top of a surfboard.",
                    '(130, 80, 385, 240)/(75, 260, 440, 450)',
                    1, 10
                ]
         ]
     
    with gr.Column():
        
        create_examples(
            examples=examples,
            inputs=[sketch_pad, grounding_instruction,language_instruction , text_box, generate_parsed, trigger_stage],
            outputs=None,
            fn=None,
            cache_examples=False,
            
        )

main.queue(concurrency_count=1, api_open=False)
main.launch(share=False, show_api=False, show_error=True, debug=False,)