Spaces:
Runtime error
Runtime error
| import string | |
| import gradio as gr | |
| import requests | |
| import torch | |
| from models.VLE import VLEForVQA, VLEProcessor, VLEForVQAPipeline | |
| from PIL import Image | |
| device = torch.device('cuda:0' if torch.cuda.is_available() else 'cpu') | |
| print("device:",device) | |
| model_name="hfl/vle-base-for-vqa" | |
| model = VLEForVQA.from_pretrained(model_name) | |
| vle_processor = VLEProcessor.from_pretrained(model_name) | |
| vqa_pipeline = VLEForVQAPipeline(model=model, device=device, vle_processor=vle_processor) | |
| from transformers import BlipProcessor, BlipForConditionalGeneration | |
| cap_processor = BlipProcessor.from_pretrained("Salesforce/blip-image-captioning-base") | |
| cap_model = BlipForConditionalGeneration.from_pretrained("Salesforce/blip-image-captioning-base") | |
| print("cap_model device:",cap_model.device) | |
| cap_model.to(device) | |
| print("cap_model device:",cap_model.device) | |
| def caption(input_image): | |
| inputs = cap_processor(input_image, return_tensors="pt").to(device) | |
| # inputs["num_beams"] = 1 # no num_beams use greedy search | |
| # inputs['num_return_sequences'] =1 | |
| out = cap_model.generate(**inputs) | |
| return "\n".join(cap_processor.batch_decode(out, skip_special_tokens=True)) | |
| import openai | |
| import os | |
| openai.api_key= os.getenv('openai_appkey') | |
| def gpt3_short(question,vqa_answer,caption): | |
| vqa_answer,vqa_score=vqa_answer | |
| prompt="This is the caption of a picture: "+caption+". Question: "+question+" VQA model predicts:"+"A: "+vqa_answer[0]+", socre: "+f"{vqa_score[0]:.2f}"+\ | |
| "; B: "+vqa_answer[1]+", score: "+f"{vqa_score[1]:.2f}"+"; C: "+vqa_answer[2]+", score: "+f"{vqa_score[2]:.2f}"+\ | |
| "; D: "+vqa_answer[3]+", score: "+f"{vqa_score[3]:.2f}"+\ | |
| ". Choose A if A is not in conflict with the description of the picture, otherwise A might be incorrect, and choose the B, C or D based on the description. Answer with A or B or C or D." | |
| # prompt=caption+"\n"+question+"\n"+vqa_answer+"\n Tell me the right answer." | |
| response = openai.Completion.create( | |
| engine="text-davinci-003", | |
| prompt=prompt, | |
| max_tokens=30, | |
| n=1, | |
| stop=None, | |
| temperature=0.7, | |
| ) | |
| answer = response.choices[0].text.strip() | |
| llm_ans=answer | |
| choice=set(["A","B","C","D"]) | |
| llm_ans=llm_ans.replace("\n"," ").replace(":"," ").replace("."," " ).replace(","," ") | |
| sllm_ans=llm_ans.split(" ") | |
| for cho in sllm_ans: | |
| if cho in choice: | |
| llm_ans=cho | |
| break | |
| if llm_ans not in choice: | |
| llm_ans="A" | |
| llm_ans=vqa_answer[ord(llm_ans)-ord("A")] | |
| answer=llm_ans | |
| return answer | |
| def gpt3_long(question,vqa_answer,caption): | |
| vqa_answer,vqa_score=vqa_answer | |
| # prompt="prompt: This is the caption of a picture: "+caption+". Question: "+question+" VQA model predicts:"+"A: "+vqa_answer[0]+"socre:"+str(vqa_score[0])+\ | |
| # " B: "+vqa_answer[1]+" score:"+str(vqa_score[1])+" C: "+vqa_answer[2]+" score:"+str(vqa_score[2])+\ | |
| # " D: "+vqa_answer[3]+'score:'+str(vqa_score[3])+\ | |
| # "Tell me the right answer with a long sentence." | |
| prompt="This is the caption of a picture: "+caption+". Question: "+question+" VQA model predicts:"+" "+vqa_answer[0]+", socre:"+f"{vqa_score[0]:.2f}"+\ | |
| "; "+vqa_answer[1]+", score:"+f"{vqa_score[1]:.2f}"+"; "+vqa_answer[2]+", score:"+f"{vqa_score[2]:.2f}"+\ | |
| "; "+vqa_answer[3]+', score:'+f"{vqa_score[3]:.2f}"+\ | |
| ". Answer the question with a sentence without mentioning the VQA model and the score." | |
| # prompt="prompt: This is the caption of a picture: "+caption+". Question: "+question+" VQA model predicts:"+" "+vqa_answer[0]+" socre:"+str(vqa_score[0])+\ | |
| # " "+vqa_answer[1]+" score:"+str(vqa_score[1])+" "+vqa_answer[2]+" score:"+str(vqa_score[2])+\ | |
| # " "+vqa_answer[3]+'score:'+str(vqa_score[3])+\ | |
| # "Tell me the right answer with a long sentence." | |
| # prompt=caption+"\n"+question+"\n"+vqa_answer+"\n Tell me the right answer." | |
| response = openai.Completion.create( | |
| engine="text-davinci-003", | |
| prompt=prompt, | |
| max_tokens=50, | |
| n=1, | |
| stop=None, | |
| temperature=0.7, | |
| ) | |
| answer = response.choices[0].text.strip() | |
| return answer | |
| def gpt3(question,vqa_answer,caption): | |
| prompt=caption+"\n"+question+"\n"+vqa_answer+"\n Tell me the right answer." | |
| response = openai.Completion.create( | |
| engine="text-davinci-003", | |
| prompt=prompt, | |
| max_tokens=50, | |
| n=1, | |
| stop=None, | |
| temperature=0.7, | |
| ) | |
| answer = response.choices[0].text.strip() | |
| # return "input_text:\n"+prompt+"\n\n output_answer:\n"+answer | |
| return answer | |
| def vle(input_image,input_text): | |
| vqa_answers = vqa_pipeline({"image":input_image, "question":input_text}, top_k=4) | |
| # return [" ".join([str(value) for key,value in vqa.items()] )for vqa in vqa_answers] | |
| return [vqa['answer'] for vqa in vqa_answers],[vqa['score'] for vqa in vqa_answers] | |
| def inference_chat(input_image,input_text): | |
| input_text=input_text[:200] | |
| input_text=" ".join(input_text.split(" ")[:60]) | |
| cap=caption(input_image) | |
| # inputs = processor(images=input_image, text=input_text,return_tensors="pt") | |
| # inputs["max_length"] = 10 | |
| # inputs["num_beams"] = 5 | |
| # inputs['num_return_sequences'] =4 | |
| # out = model_vqa.generate(**inputs) | |
| # out=processor.batch_decode(out, skip_special_tokens=True) | |
| print("Caption:",cap) | |
| out=vle(input_image,input_text) | |
| print("VQA: ",out) | |
| # vqa="\n".join(out[0]) | |
| # gpt3_out=gpt3(input_text,vqa,cap) | |
| gpt3_out=gpt3_long(input_text,out,cap) | |
| # gpt3_out1=gpt3_short(input_text,out,cap) | |
| return out[0][0], gpt3_out #,gpt3_out1 | |
| title = """<h1 align="center">VQA with VLE and LLM</h1>""" | |
| # description = """We demonstrate three visual question answering systems built with VLE and LLM: | |
| # 1. VQA: The image and the question are fed into a VQA model (VLEForVQA) and the model predicts the answer. | |
| # 2. VQA+LLM: The captioning model generates a caption of the image. We feed the caption, the question, and the answer candidates predicted by the VQA model to the LLM, and ask the LLM to generate the most reasonable answer. | |
| # The outptus from VQA+LLM may vary due to the decoding strategy of LLM. For more details about VLE and the VQA pipeline, see [http://vle.hfl-rc.com](http://vle.hfl-rc.com)""" | |
| description_main="""**VLE** (Vision-Language Encoder) is an image-text multimodal understanding model built on the pre-trained text and image encoders. See [https://github.com/iflytek/VLE](https://github.com/iflytek/VLE) for more details. | |
| We demonstrate visual question answering systems built with VLE and LLM.""" | |
| description_detail="""**VQA**: The image and the question are fed to a VQA model (VLEForVQA) and the model predicts the answer. | |
| **VQA+LLM**: We feed the caption, question, and answers predicted by the VQA model to the LLM and ask the LLM to generate the final answer. The outptus from VQA+LLM may vary due to the decoding strategy of the LLM.""" | |
| with gr.Blocks( | |
| css=""" | |
| .message.svelte-w6rprc.svelte-w6rprc.svelte-w6rprc {font-size: 20px; margin-top: 20px} | |
| #component-21 > div.wrap.svelte-w6rprc {height: 600px;} | |
| """ | |
| ) as iface: | |
| state = gr.State([]) | |
| #caption_output = None | |
| gr.Markdown(title) | |
| gr.Markdown(description_main) | |
| #gr.Markdown(article) | |
| with gr.Row(): | |
| with gr.Column(scale=1): | |
| image_input = gr.Image(type="pil",label="VQA Image Input") | |
| with gr.Row(): | |
| with gr.Column(scale=1): | |
| chat_input = gr.Textbox(lines=1, label="VQA Question Input") | |
| with gr.Row(): | |
| # clear_button = gr.Button(value="Clear", interactive=True) | |
| submit_button = gr.Button( | |
| value="Submit", interactive=True, variant="primary" | |
| ) | |
| ''' | |
| cap_submit_button = gr.Button( | |
| value="Submit_CAP", interactive=True, variant="primary" | |
| ) | |
| gpt3_submit_button = gr.Button( | |
| value="Submit_GPT3", interactive=True, variant="primary" | |
| ) | |
| ''' | |
| with gr.Column(): | |
| gr.Markdown(description_detail) | |
| caption_output = gr.Textbox(lines=0, label="VQA ") | |
| gpt3_output_v1 = gr.Textbox(lines=0, label="VQA+LLM") | |
| # image_input.change( | |
| # lambda: ("", [],"","",""), | |
| # [], | |
| # [ caption_output, state,caption_output,gpt3_output_v1,caption_output_v1], | |
| # queue=False, | |
| # ) | |
| chat_input.submit( | |
| inference_chat, | |
| [ | |
| image_input, | |
| chat_input, | |
| ], | |
| [ caption_output,gpt3_output_v1], | |
| ) | |
| # clear_button.click( | |
| # lambda: ("", [],"","",""), | |
| # [], | |
| # [chat_input, state,caption_output,gpt3_output_v1,caption_output_v1], | |
| # queue=False, | |
| # ) | |
| submit_button.click( | |
| inference_chat, | |
| [ | |
| image_input, | |
| chat_input, | |
| ], | |
| [caption_output,gpt3_output_v1], | |
| ) | |
| ''' | |
| cap_submit_button.click( | |
| caption, | |
| [ | |
| image_input, | |
| ], | |
| [caption_output_v1], | |
| ) | |
| gpt3_submit_button.click( | |
| gpt3, | |
| [ | |
| chat_input, | |
| caption_output , | |
| caption_output_v1, | |
| ], | |
| [gpt3_output_v1], | |
| ) | |
| ''' | |
| examples=[['pics/men.jpg',"How many people are there?","3","There are two people in the picture: a man and the driver of the truck."], | |
| ['pics/dogs.png',"Where are the huskies?","on grass","The huskies are sitting on the grass."], | |
| ['pics/horses.jpg',"What are the horses doing?",'walking','The horses are walking and pulling a sleigh through the snow.'], | |
| ['pics/fish.jpg',"What is in the man's hand?","fish","The man in the hat is holding a fishing pole."], | |
| ['pics/tower.jpg',"Where is the photo taken?","paris","The photo appears to have been taken in Paris, near the Eiffel Tower."], | |
| ['pics/traffic.jpg',"What is this man doing?","looking","The man appears to be looking around the street."], | |
| ['pics/chicking.jpg',"What did this animal hatch from?","farm","The animal likely hatched from a farm, ground, tree, or nest."] | |
| ] | |
| examples = gr.Examples( | |
| examples=examples,inputs=[image_input, chat_input,caption_output,gpt3_output_v1], | |
| ) | |
| #iface.queue(concurrency_count=1, api_open=False, max_size=10) | |
| iface.launch(enable_queue=False) |