whyu commited on
Commit
aa34d88
β€’
1 Parent(s): 695d043

initial commit

Browse files
Files changed (4) hide show
  1. README.md +3 -4
  2. app.py +323 -0
  3. mm-vet-v2/mm-vet-v2.json +0 -0
  4. requirements.txt +1 -0
README.md CHANGED
@@ -1,13 +1,12 @@
1
  ---
2
- title: MM-Vet-v2 Evaluator
3
- emoji: ⚑
4
  colorFrom: yellow
5
  colorTo: gray
6
  sdk: gradio
7
- sdk_version: 4.40.0
8
  app_file: app.py
9
  pinned: false
10
- license: apache-2.0
11
  ---
12
 
13
  Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
 
1
  ---
2
+ title: MM-Vet v2 Evaluator
3
+ emoji: πŸ†
4
  colorFrom: yellow
5
  colorTo: gray
6
  sdk: gradio
7
+ sdk_version: 4.24.0
8
  app_file: app.py
9
  pinned: false
 
10
  ---
11
 
12
  Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
app.py ADDED
@@ -0,0 +1,323 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import gradio as gr
2
+ import json
3
+ import os
4
+ from tqdm import tqdm
5
+ import pandas as pd
6
+ import numpy as np
7
+ from collections import Counter
8
+ import time
9
+ from zipfile import ZipFile
10
+ from openai import AzureOpenAI
11
+ from openai._exceptions import RateLimitError, BadRequestError
12
+
13
+ client = AzureOpenAI(
14
+ api_key=os.environ.get("AZURE_OPENAI_API_KEY"),
15
+ api_version=os.environ.get("AZURE_OPENAI_API_VERSION"),
16
+ azure_endpoint=os.getenv("AZURE_OPENAI_API_ENDPOINT"),
17
+ )
18
+ deployment_id = os.environ.get("AZURE_OPENAI_DEP_ID")
19
+ gpt_model = deployment_id
20
+
21
+
22
+ prompt = """Compare the ground truth and prediction from AI models, to give a correctness score for the prediction. <image> in the question indicates where an image is. <AND> in the ground truth means it is totally right only when all elements in the ground truth are present in the prediction, and <OR> means it is totally right when any one element in the ground truth is present in the prediction. The correctness score is 0.0 (totally wrong), 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, or 1.0 (totally right). Just complete the last space of the correctness score.
23
+
24
+ | Question | Ground truth | Prediction | Correctness |
25
+ | --- | --- | --- | --- |
26
+ | What is x in the equation?<image> | -1 <AND> -5 | x = 3 | 0.0 |
27
+ | What is x in the equation?<image> | -1 <AND> -5 | x = -1 | 0.5 |
28
+ | What is x in the equation?<image> | -1 <AND> -5 | x = -5 | 0.5 |
29
+ | What is x in the equation?<image> | -1 <AND> -5 | x = -5 or 5 | 0.5 |
30
+ | What is x in the equation?<image> | -1 <AND> -5 | x = -1 or x = -5 | 1.0 |
31
+ | Can you explain this meme?<image> | This meme is poking fun at the fact that the names of the countries Iceland and Greenland are misleading. Despite its name, Iceland is known for its beautiful green landscapes, while Greenland is mostly covered in ice and snow. The meme is saying that the person has trust issues because the names of these countries do not accurately represent their landscapes. | The meme talks about Iceland and Greenland. It's pointing out that despite their names, Iceland is not very icy and Greenland isn't very green. | 0.4 |
32
+ | Can you explain this meme?<image> | This meme is poking fun at the fact that the names of the countries Iceland and Greenland are misleading. Despite its name, Iceland is known for its beautiful green landscapes, while Greenland is mostly covered in ice and snow. The meme is saying that the person has trust issues because the names of these countries do not accurately represent their landscapes. | The meme is using humor to point out the misleading nature of Iceland's and Greenland's names. Iceland, despite its name, has lush green landscapes while Greenland is mostly covered in ice and snow. The text 'This is why I have trust issues' is a playful way to suggest that these contradictions can lead to distrust or confusion. The humor in this meme is derived from the unexpected contrast between the names of the countries and their actual physical characteristics. | 1.0 |
33
+ """
34
+
35
+
36
+
37
+ def grade(file_obj, progress=gr.Progress()):
38
+ # load metadata
39
+ # Download mm-vet.zip and `unzip mm-vet.zip` and change the path below
40
+ mmvet_path = "mm-vet-v2"
41
+ decimal_places = 1 # number of decimal places to round to
42
+
43
+ mmvet_metadata = os.path.join(mmvet_path, "mm-vet-v2.json")
44
+ with open(mmvet_metadata, 'r') as f:
45
+ data = json.load(f)
46
+
47
+
48
+ counter = Counter()
49
+ cap_set_list = []
50
+ cap_set_counter = []
51
+ len_data = 0
52
+ for id, value in data.items():
53
+ question = value["question"]
54
+ answer = value["answer"]
55
+ cap = value["capability"]
56
+ cap = set(cap)
57
+ counter.update(cap)
58
+ if cap not in cap_set_list:
59
+ cap_set_list.append(cap)
60
+ cap_set_counter.append(1)
61
+ else:
62
+ cap_set_counter[cap_set_list.index(cap)] += 1
63
+
64
+ len_data += 1
65
+
66
+ sorted_list = counter.most_common()
67
+ columns = [k for k, v in sorted_list]
68
+ columns.append("total")
69
+ columns.append("std")
70
+ columns.append('runs')
71
+ df = pd.DataFrame(columns=columns)
72
+
73
+
74
+ cap_set_sorted_indices = np.argsort(-np.array(cap_set_counter))
75
+ new_cap_set_list = []
76
+ new_cap_set_counter = []
77
+ for index in cap_set_sorted_indices:
78
+ new_cap_set_list.append(cap_set_list[index])
79
+ new_cap_set_counter.append(cap_set_counter[index])
80
+
81
+ cap_set_list = new_cap_set_list
82
+ cap_set_counter = new_cap_set_counter
83
+ cap_set_names = ["_".join(list(cap_set)) for cap_set in cap_set_list]
84
+
85
+ columns2 = cap_set_names
86
+ columns2.append("total")
87
+ columns2.append("std")
88
+ columns2.append('runs')
89
+ df2 = pd.DataFrame(columns=columns2)
90
+
91
+
92
+ ###### change your model name ######
93
+ model = file_obj.name.split("/")[-1][:-5]
94
+ # result_path = "results"
95
+ num_run = 1 # we set 5 in the paper
96
+ # model_results_file = os.path.join(result_path, f"{model}.json")
97
+ model_results_file = file_obj.name
98
+
99
+ # grade results for each sample to svae
100
+ grade_file = f'{model}_{gpt_model}-grade-{num_run}runs.json'
101
+ # grade_file = os.path.join(result_path, grade_file)
102
+
103
+ # score results regarding capabilities/capability integration to save
104
+ cap_score_file = f'{model}_{gpt_model}-cap-score-{num_run}runs.csv'
105
+ # cap_score_file = os.path.join(result_path, cap_score_file)
106
+ cap_int_score_file = f'{model}_{gpt_model}-cap-int-score-{num_run}runs.csv'
107
+ # cap_int_score_file = os.path.join(result_path, cap_int_score_file)
108
+
109
+
110
+
111
+ with open(model_results_file) as f:
112
+ results = json.load(f)
113
+ if os.path.exists(grade_file):
114
+ with open(grade_file, 'r') as f:
115
+ grade_results = json.load(f)
116
+ else:
117
+ grade_results = {}
118
+
119
+
120
+ def need_more_runs():
121
+ need_more_runs = False
122
+ if len(grade_results) > 0:
123
+ for k, v in grade_results.items():
124
+ if len(v['score']) < num_run:
125
+ need_more_runs = True
126
+ break
127
+ return need_more_runs or len(grade_results) < len_data
128
+
129
+
130
+ while need_more_runs():
131
+ for j in range(num_run):
132
+ print(f'eval run {j}')
133
+ for id, line in progress.tqdm(data.items(), desc="Grading"):
134
+ if id in grade_results and len(grade_results[id]['score']) >= (j + 1):
135
+ continue
136
+
137
+ model_pred = results[id]
138
+
139
+ queries = line['question'].split('<IMG>')
140
+ query = ""
141
+ for q in queries:
142
+ if q.endswith((".jpg", "jpeg", ".png")):
143
+ query += "<image>"
144
+ else:
145
+ query += q
146
+
147
+ question = prompt + '| ' + ' | '.join([query.replace('\n', '<br>'), line['answer'].replace("<AND>", " <AND> ").replace("<OR>", " <OR> ").replace('\n', '<br>'), model_pred.replace('\n', '<br>'), ""])
148
+ messages = [
149
+ {"role": "user", "content": question},
150
+ ]
151
+
152
+ if id not in grade_results:
153
+ sample_grade = {'model': [], 'content': [], 'score': []}
154
+ else:
155
+ sample_grade = grade_results[id]
156
+
157
+
158
+ grade_sample_run_complete = False
159
+ temperature = 0.0
160
+
161
+ num_sleep = 0
162
+ while not grade_sample_run_complete:
163
+ try:
164
+ response = client.chat.completions.create(
165
+ model=gpt_model,
166
+ max_tokens=3,
167
+ temperature=temperature,
168
+ messages=messages)
169
+ content = response.choices[0].message.content
170
+ flag = True
171
+ try_time = 1
172
+ while flag:
173
+ try:
174
+ content = content.split(' ')[0].strip()
175
+ score = float(content)
176
+ if score > 1.0 or score < 0.0:
177
+ assert False
178
+ flag = False
179
+ except:
180
+ question_try = question + "\n\nPredict the correctness of the answer (digit): "
181
+ # messages = [
182
+ # {"role": "user", "content": question},
183
+ # ]
184
+ messages = [
185
+ {"role": "user", "content": question_try},
186
+ ]
187
+ response = client.chat.completions.create(
188
+ model=gpt_model,
189
+ max_tokens=3,
190
+ temperature=temperature,
191
+ messages=messages)
192
+ content = response.choices[0].message.content
193
+ try_time += 1
194
+ temperature += 0.5
195
+ print(f"{id} try {try_time} times")
196
+ print(content)
197
+ if try_time > 5:
198
+ score = 0.0
199
+ flag = False
200
+ grade_sample_run_complete = True
201
+ response_model = response.model
202
+ except BadRequestError as e:
203
+ content = "BadRequestError"
204
+ score = 0.0
205
+ flag = False
206
+ print(id, "BadRequestError")
207
+ response_model = gpt_model
208
+ break
209
+ # except RateLimitError as e:
210
+ except:
211
+ # gpt4 may have token rate limit
212
+ num_sleep += 1
213
+ if num_sleep > 12:
214
+ score = 0.0
215
+ grade_sample_run_complete = True
216
+ content = "RateLimitError"
217
+ num_sleep = 0
218
+ continue
219
+ print("sleep 5s")
220
+ time.sleep(5)
221
+ response_model = gpt_model
222
+
223
+
224
+ if len(sample_grade['model']) >= j + 1:
225
+ sample_grade['model'][j] = response_model
226
+ sample_grade['content'][j] = content
227
+ sample_grade['score'][j] = score
228
+ else:
229
+ sample_grade['model'].append(response_model)
230
+ sample_grade['content'].append(content)
231
+ sample_grade['score'].append(score)
232
+ grade_results[id] = sample_grade
233
+
234
+ with open(grade_file, 'w') as f:
235
+ json.dump(grade_results, f, indent=4)
236
+
237
+
238
+ assert not need_more_runs()
239
+ cap_socres = {k: [0.0]*num_run for k in columns[:-2]}
240
+ counter['total'] = len_data
241
+
242
+ cap_socres2 = {k: [0.0]*num_run for k in columns2[:-2]}
243
+ counter2 = {columns2[i]:cap_set_counter[i] for i in range(len(cap_set_counter))}
244
+ counter2['total'] = len_data
245
+
246
+ for k, v in grade_results.items():
247
+ for i in range(num_run):
248
+ score = v['score'][i]
249
+ caps = set(data[k]['capability'])
250
+ for c in caps:
251
+ cap_socres[c][i] += score
252
+
253
+ cap_socres['total'][i] += score
254
+
255
+ index = cap_set_list.index(caps)
256
+ cap_socres2[cap_set_names[index]][i] += score
257
+ cap_socres2['total'][i] += score
258
+
259
+ for k, v in cap_socres.items():
260
+ cap_socres[k] = np.array(v) / counter[k] *100
261
+
262
+
263
+ std = round(cap_socres['total'].std(), decimal_places)
264
+ total_copy = cap_socres['total'].copy()
265
+ runs = str(list(np.round(total_copy, decimal_places)))
266
+
267
+ for k, v in cap_socres.items():
268
+ cap_socres[k] = round(v.mean(), decimal_places)
269
+
270
+ cap_socres['std'] = std
271
+ cap_socres['runs'] = runs
272
+ df.loc[model] = cap_socres
273
+
274
+
275
+ for k, v in cap_socres2.items():
276
+ cap_socres2[k] = round(np.mean(np.array(v) / counter2[k] *100), decimal_places)
277
+ cap_socres2['std'] = std
278
+ cap_socres2['runs'] = runs
279
+ df2.loc[model] = cap_socres2
280
+
281
+ df.to_csv(cap_score_file)
282
+ df2.to_csv(cap_int_score_file)
283
+
284
+ files = [cap_score_file, cap_int_score_file, grade_file]
285
+ zip_file = f"results.zip"
286
+ with ZipFile(zip_file, "w") as zipObj:
287
+ for idx, file in enumerate(files):
288
+ zipObj.write(file, file)
289
+ for file in files:
290
+ os.remove(file)
291
+ return zip_file
292
+
293
+
294
+
295
+ # demo = gr.Interface(
296
+ # fn=grade,
297
+ # inputs=gr.File(file_types=[".json"]),
298
+ # outputs="file")
299
+
300
+
301
+ model_result_example = "https://raw.githubusercontent.com/yuweihao/MM-Vet/main/v2/results/gpt-4o-2024-05-13_detail-high.json"
302
+
303
+ markdown = f"""
304
+ # MM-Vet v2: A Challenging Benchmark to Evaluate Large Multimodal Models for Integrated Capabilities
305
+
306
+ We offer MM-Vet v2 LLM-based (GPT-4) evaluator to grade open-ended outputs from your models.
307
+
308
+ Plese upload your json file of your model results containing `{{v1_0: ..., v1_1: ..., }}`like [this json file]({model_result_example}).
309
+
310
+ The grading results will be downloaded as a zip file.
311
+ """
312
+
313
+
314
+ with gr.Blocks() as demo:
315
+ gr.Markdown(markdown)
316
+ with gr.Row():
317
+ inp = gr.File(file_types=[".json"])
318
+ out = gr.File(file_types=[".zip"])
319
+ inp.change(grade, inp, out)
320
+
321
+
322
+ if __name__ == "__main__":
323
+ demo.queue().launch()
mm-vet-v2/mm-vet-v2.json ADDED
The diff for this file is too large to render. See raw diff
 
requirements.txt ADDED
@@ -0,0 +1 @@
 
 
1
+ openai==1.14.2