henryL7 commited on
Commit
969c59e
1 Parent(s): f252438

clean up repo

Browse files
README.md CHANGED
@@ -1,5 +1,5 @@
1
  ---
2
- title: Instrusumeval
3
  emoji: 🥇
4
  colorFrom: green
5
  colorTo: indigo
@@ -12,34 +12,4 @@ license: apache-2.0
12
 
13
  # Start the configuration
14
 
15
- Most of the variables to change for a default leaderboard are in `src/env.py` (replace the path for your leaderboard) and `src/about.py` (for tasks).
16
-
17
- Results files should have the following format and be stored as json files:
18
- ```json
19
- {
20
- "config": {
21
- "model_dtype": "torch.float16", # or torch.bfloat16 or 8bit or 4bit
22
- "model_name": "path of the model on the hub: org/model",
23
- "model_sha": "revision on the hub",
24
- },
25
- "results": {
26
- "task_name": {
27
- "metric_name": score,
28
- },
29
- "task_name2": {
30
- "metric_name": score,
31
- }
32
- }
33
- }
34
- ```
35
-
36
- Request files are created automatically by this tool.
37
-
38
- If you encounter problem on the space, don't hesitate to restart it to remove the create eval-queue, eval-queue-bk, eval-results and eval-results-bk created folder.
39
-
40
- # Code logic for more complex edits
41
-
42
- You'll find
43
- - the main table' columns names and properties in `src/display/utils.py`
44
- - the logic to read all results and request files, then convert them in dataframe lines, in `src/leaderboard/read_evals.py`, and `src/populate.py`
45
- - teh logic to allow or filter submissions in `src/submission/submit.py` and `src/submission/check_validity.py`
 
1
  ---
2
+ title: InstruSumEval
3
  emoji: 🥇
4
  colorFrom: green
5
  colorTo: indigo
 
12
 
13
  # Start the configuration
14
 
15
+ Most of the variables to change for a default leaderboard are in `src/env.py` (replace the path for your leaderboard) and `src/about.py` (for tasks).
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
app.py CHANGED
@@ -1,138 +1,22 @@
1
- import subprocess
2
  import gradio as gr
3
- import pandas as pd
4
  from apscheduler.schedulers.background import BackgroundScheduler
5
- from huggingface_hub import snapshot_download
6
 
7
  from src.about import (
8
  CITATION_BUTTON_LABEL,
9
  CITATION_BUTTON_TEXT,
10
- EVALUATION_QUEUE_TEXT,
11
  INTRODUCTION_TEXT,
12
  LLM_BENCHMARKS_TEXT,
13
  TITLE,
14
  )
15
  from src.display.css_html_js import custom_css
16
- from src.display.utils import (
17
- BENCHMARK_COLS,
18
- COLS,
19
- EVAL_COLS,
20
- EVAL_TYPES,
21
- NUMERIC_INTERVALS,
22
- TYPES,
23
- AutoEvalColumn,
24
- ModelType,
25
- fields,
26
- WeightType,
27
- Precision
28
- )
29
- from src.envs import API, EVAL_REQUESTS_PATH, EVAL_RESULTS_PATH, QUEUE_REPO, REPO_ID, RESULTS_REPO, TOKEN
30
- from src.populate import get_evaluation_queue_df, get_leaderboard_df
31
- from src.submission.submit import add_new_eval
32
- from src import dummy_leaderboard
33
  from src import populate
34
 
35
 
36
  def restart_space():
37
  API.restart_space(repo_id=REPO_ID)
38
 
39
- try:
40
- print(EVAL_REQUESTS_PATH)
41
- snapshot_download(
42
- repo_id=QUEUE_REPO, local_dir=EVAL_REQUESTS_PATH, repo_type="dataset", tqdm_class=None, etag_timeout=30, token=TOKEN
43
- )
44
- except Exception:
45
- restart_space()
46
- try:
47
- print(EVAL_RESULTS_PATH)
48
- snapshot_download(
49
- repo_id=RESULTS_REPO, local_dir=EVAL_RESULTS_PATH, repo_type="dataset", tqdm_class=None, etag_timeout=30, token=TOKEN
50
- )
51
- except Exception:
52
- restart_space()
53
-
54
-
55
- raw_data, original_df = get_leaderboard_df(EVAL_RESULTS_PATH, EVAL_REQUESTS_PATH, COLS, BENCHMARK_COLS)
56
- leaderboard_df = original_df.copy()
57
-
58
- (
59
- finished_eval_queue_df,
60
- running_eval_queue_df,
61
- pending_eval_queue_df,
62
- ) = get_evaluation_queue_df(EVAL_REQUESTS_PATH, EVAL_COLS)
63
-
64
-
65
- # Searching and filtering
66
- def update_table(
67
- hidden_df: pd.DataFrame,
68
- columns: list,
69
- type_query: list,
70
- precision_query: str,
71
- size_query: list,
72
- show_deleted: bool,
73
- query: str,
74
- ):
75
- filtered_df = filter_models(hidden_df, type_query, size_query, precision_query, show_deleted)
76
- filtered_df = filter_queries(query, filtered_df)
77
- df = select_columns(filtered_df, columns)
78
- return df
79
-
80
-
81
- def search_table(df: pd.DataFrame, query: str) -> pd.DataFrame:
82
- return df[(df[AutoEvalColumn.model.name].str.contains(query, case=False))]
83
-
84
-
85
- def select_columns(df: pd.DataFrame, columns: list) -> pd.DataFrame:
86
- always_here_cols = [
87
- AutoEvalColumn.model_type_symbol.name,
88
- AutoEvalColumn.model.name,
89
- ]
90
- # We use COLS to maintain sorting
91
- filtered_df = df[
92
- always_here_cols + [c for c in COLS if c in df.columns and c in columns]
93
- ]
94
- return filtered_df
95
-
96
-
97
- def filter_queries(query: str, filtered_df: pd.DataFrame) -> pd.DataFrame:
98
- final_df = []
99
- if query != "":
100
- queries = [q.strip() for q in query.split(";")]
101
- for _q in queries:
102
- _q = _q.strip()
103
- if _q != "":
104
- temp_filtered_df = search_table(filtered_df, _q)
105
- if len(temp_filtered_df) > 0:
106
- final_df.append(temp_filtered_df)
107
- if len(final_df) > 0:
108
- filtered_df = pd.concat(final_df)
109
- filtered_df = filtered_df.drop_duplicates(
110
- subset=[AutoEvalColumn.model.name, AutoEvalColumn.precision.name, AutoEvalColumn.revision.name]
111
- )
112
-
113
- return filtered_df
114
-
115
-
116
- def filter_models(
117
- df: pd.DataFrame, type_query: list, size_query: list, precision_query: list, show_deleted: bool
118
- ) -> pd.DataFrame:
119
- # Show all models
120
- if show_deleted:
121
- filtered_df = df
122
- else: # Show only still on the hub models
123
- filtered_df = df[df[AutoEvalColumn.still_on_hub.name] == True]
124
-
125
- type_emoji = [t[0] for t in type_query]
126
- filtered_df = filtered_df.loc[df[AutoEvalColumn.model_type_symbol.name].isin(type_emoji)]
127
- filtered_df = filtered_df.loc[df[AutoEvalColumn.precision.name].isin(precision_query + ["None"])]
128
-
129
- numeric_interval = pd.IntervalIndex(sorted([NUMERIC_INTERVALS[s] for s in size_query]))
130
- params_column = pd.to_numeric(df[AutoEvalColumn.params.name], errors="coerce")
131
- mask = params_column.apply(lambda x: any(numeric_interval.contains(x)))
132
- filtered_df = filtered_df.loc[mask]
133
-
134
- return filtered_df
135
-
136
 
137
  demo = gr.Blocks(css=custom_css)
138
  with demo:
@@ -141,85 +25,6 @@ with demo:
141
 
142
  with gr.Tabs(elem_classes="tab-buttons") as tabs:
143
  with gr.TabItem("🏅 LLM Benchmark", elem_id="llm-benchmark-tab-table", id=0):
144
- # with gr.Row():
145
- # search_bar = gr.Textbox(
146
- # placeholder=" 🔍 Search for the model (separate multiple queries with `;`) and press ENTER...",
147
- # show_label=False,
148
- # elem_id="search-bar",
149
- # )
150
- # with gr.Column():
151
- # with gr.Row():
152
- # search_bar = gr.Textbox(
153
- # placeholder=" 🔍 Search for the model (separate multiple queries with `;`) and press ENTER...",
154
- # show_label=False,
155
- # elem_id="search-bar",
156
- # )
157
- # with gr.Row():
158
- # shown_columns = gr.CheckboxGroup(
159
- # choices=[
160
- # c.name
161
- # for c in fields(AutoEvalColumn)
162
- # if not c.hidden and not c.never_hidden
163
- # ],
164
- # value=[
165
- # c.name
166
- # for c in fields(AutoEvalColumn)
167
- # if c.displayed_by_default and not c.hidden and not c.never_hidden
168
- # ],
169
- # label="Select columns to show",
170
- # elem_id="column-select",
171
- # interactive=True,
172
- # )
173
- # with gr.Row():
174
- # deleted_models_visibility = gr.Checkbox(
175
- # value=False, label="Show gated/private/deleted models", interactive=True
176
- # )
177
- # with gr.Column(min_width=320):
178
- # #with gr.Box(elem_id="box-filter"):
179
- # filter_columns_type = gr.CheckboxGroup(
180
- # label="Model types",
181
- # choices=[t.to_str() for t in ModelType],
182
- # value=[t.to_str() for t in ModelType],
183
- # interactive=True,
184
- # elem_id="filter-columns-type",
185
- # )
186
- # filter_columns_precision = gr.CheckboxGroup(
187
- # label="Precision",
188
- # choices=[i.value.name for i in Precision],
189
- # value=[i.value.name for i in Precision],
190
- # interactive=True,
191
- # elem_id="filter-columns-precision",
192
- # )
193
- # filter_columns_size = gr.CheckboxGroup(
194
- # label="Model sizes (in billions of parameters)",
195
- # choices=list(NUMERIC_INTERVALS.keys()),
196
- # value=list(NUMERIC_INTERVALS.keys()),
197
- # interactive=True,
198
- # elem_id="filter-columns-size",
199
- # )
200
-
201
- # leaderboard_table = gr.components.Dataframe(
202
- # value=leaderboard_df[
203
- # [c.name for c in fields(AutoEvalColumn) if c.never_hidden]
204
- # + shown_columns.value
205
- # ],
206
- # headers=[c.name for c in fields(AutoEvalColumn) if c.never_hidden] + shown_columns.value,
207
- # datatype=TYPES,
208
- # elem_id="leaderboard-table",
209
- # interactive=False,
210
- # visible=True,
211
- # )
212
-
213
- # leaderboard_table = gr.components.Dataframe(
214
- # value=leaderboard_df[
215
- # [c.name for c in fields(AutoEvalColumn) if c.never_hidden]
216
- # ],
217
- # headers=[c.name for c in fields(AutoEvalColumn) if c.never_hidden],
218
- # datatype=TYPES,
219
- # elem_id="leaderboard-table",
220
- # interactive=False,
221
- # visible=True,
222
- # )
223
  leaderboard_df = populate.load_leaderboard()
224
  leaderboard_table = gr.components.Dataframe(
225
  value=leaderboard_df,
@@ -228,143 +33,31 @@ with demo:
228
  elem_id="leaderboard-table",
229
  interactive=False,
230
  visible=True,
 
231
  )
232
 
233
- # Dummy leaderboard for handling the case when the user uses backspace key
234
- # hidden_leaderboard_table_for_search = gr.components.Dataframe(
235
- # value=original_df[COLS],
236
- # headers=COLS,
237
- # datatype=TYPES,
238
- # visible=False,
239
- # )
240
- # search_bar.submit(
241
- # update_table,
242
- # [
243
- # hidden_leaderboard_table_for_search,
244
- # shown_columns,
245
- # filter_columns_type,
246
- # filter_columns_precision,
247
- # filter_columns_size,
248
- # deleted_models_visibility,
249
- # search_bar,
250
- # ],
251
- # leaderboard_table,
252
- # )
253
- # for selector in [shown_columns, filter_columns_type, filter_columns_precision, filter_columns_size, deleted_models_visibility]:
254
- # selector.change(
255
- # update_table,
256
- # [
257
- # hidden_leaderboard_table_for_search,
258
- # shown_columns,
259
- # filter_columns_type,
260
- # filter_columns_precision,
261
- # filter_columns_size,
262
- # deleted_models_visibility,
263
- # search_bar,
264
- # ],
265
- # leaderboard_table,
266
- # queue=True,
267
- # )
268
-
269
  with gr.TabItem("📝 About", elem_id="llm-benchmark-tab-table", id=2):
270
  gr.Markdown(LLM_BENCHMARKS_TEXT, elem_classes="markdown-text")
271
 
272
- with gr.TabItem("🚀 Submit here! ", elem_id="llm-benchmark-tab-table", id=3):
273
- with gr.Column():
274
- with gr.Row():
275
- gr.Markdown(EVALUATION_QUEUE_TEXT, elem_classes="markdown-text")
276
-
277
- with gr.Column():
278
- with gr.Accordion(
279
- f"✅ Finished Evaluations ({len(finished_eval_queue_df)})",
280
- open=False,
281
- ):
282
- with gr.Row():
283
- finished_eval_table = gr.components.Dataframe(
284
- value=finished_eval_queue_df,
285
- headers=EVAL_COLS,
286
- datatype=EVAL_TYPES,
287
- row_count=5,
288
- )
289
- with gr.Accordion(
290
- f"🔄 Running Evaluation Queue ({len(running_eval_queue_df)})",
291
- open=False,
292
- ):
293
- with gr.Row():
294
- running_eval_table = gr.components.Dataframe(
295
- value=running_eval_queue_df,
296
- headers=EVAL_COLS,
297
- datatype=EVAL_TYPES,
298
- row_count=5,
299
- )
300
-
301
- with gr.Accordion(
302
- f"⏳ Pending Evaluation Queue ({len(pending_eval_queue_df)})",
303
- open=False,
304
- ):
305
- with gr.Row():
306
- pending_eval_table = gr.components.Dataframe(
307
- value=pending_eval_queue_df,
308
- headers=EVAL_COLS,
309
- datatype=EVAL_TYPES,
310
- row_count=5,
311
- )
312
- with gr.Row():
313
- gr.Markdown("# ✉️✨ Submit your model here!", elem_classes="markdown-text")
314
-
315
  with gr.Row():
316
- with gr.Column():
317
- model_name_textbox = gr.Textbox(label="Model name")
318
- revision_name_textbox = gr.Textbox(label="Revision commit", placeholder="main")
319
- model_type = gr.Dropdown(
320
- choices=[t.to_str(" : ") for t in ModelType if t != ModelType.Unknown],
321
- label="Model type",
322
- multiselect=False,
323
- value=None,
324
- interactive=True,
325
  )
326
 
327
- with gr.Column():
328
- precision = gr.Dropdown(
329
- choices=[i.value.name for i in Precision if i != Precision.Unknown],
330
- label="Precision",
331
- multiselect=False,
332
- value="float16",
333
- interactive=True,
334
- )
335
- weight_type = gr.Dropdown(
336
- choices=[i.value.name for i in WeightType],
337
- label="Weights type",
338
- multiselect=False,
339
- value="Original",
340
- interactive=True,
341
- )
342
- base_model_name_textbox = gr.Textbox(label="Base model (for delta or adapter weights)")
343
 
344
- submit_button = gr.Button("Submit Eval")
345
- submission_result = gr.Markdown()
346
- submit_button.click(
347
- add_new_eval,
348
- [
349
- model_name_textbox,
350
- base_model_name_textbox,
351
- revision_name_textbox,
352
- precision,
353
- weight_type,
354
- model_type,
355
- ],
356
- submission_result,
357
- )
358
 
359
- with gr.Row():
360
- with gr.Accordion("📙 Citation", open=False):
361
- citation_button = gr.Textbox(
362
- value=CITATION_BUTTON_TEXT,
363
- label=CITATION_BUTTON_LABEL,
364
- lines=20,
365
- elem_id="citation-button",
366
- show_copy_button=True,
367
- )
368
 
369
  scheduler = BackgroundScheduler()
370
  scheduler.add_job(restart_space, "interval", seconds=1800)
 
 
1
  import gradio as gr
 
2
  from apscheduler.schedulers.background import BackgroundScheduler
 
3
 
4
  from src.about import (
5
  CITATION_BUTTON_LABEL,
6
  CITATION_BUTTON_TEXT,
 
7
  INTRODUCTION_TEXT,
8
  LLM_BENCHMARKS_TEXT,
9
  TITLE,
10
  )
11
  from src.display.css_html_js import custom_css
12
+ from src.envs import API, REPO_ID
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
13
  from src import populate
14
 
15
 
16
  def restart_space():
17
  API.restart_space(repo_id=REPO_ID)
18
 
19
+ restart_space()
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
20
 
21
  demo = gr.Blocks(css=custom_css)
22
  with demo:
 
25
 
26
  with gr.Tabs(elem_classes="tab-buttons") as tabs:
27
  with gr.TabItem("🏅 LLM Benchmark", elem_id="llm-benchmark-tab-table", id=0):
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
28
  leaderboard_df = populate.load_leaderboard()
29
  leaderboard_table = gr.components.Dataframe(
30
  value=leaderboard_df,
 
33
  elem_id="leaderboard-table",
34
  interactive=False,
35
  visible=True,
36
+ height=600,
37
  )
38
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
39
  with gr.TabItem("📝 About", elem_id="llm-benchmark-tab-table", id=2):
40
  gr.Markdown(LLM_BENCHMARKS_TEXT, elem_classes="markdown-text")
41
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
42
  with gr.Row():
43
+ with gr.Accordion("📙 Citation", open=False):
44
+ citation_button = gr.Textbox(
45
+ value=CITATION_BUTTON_TEXT,
46
+ label=CITATION_BUTTON_LABEL,
47
+ lines=6,
48
+ elem_id="citation-button",
49
+ show_copy_button=True,
 
 
50
  )
51
 
52
+ # with gr.TabItem("🚀 Submit here! ", elem_id="llm-benchmark-tab-table", id=3):
53
+ # with gr.Column():
54
+ # with gr.Row():
55
+ # gr.Markdown(EVALUATION_QUEUE_TEXT, elem_classes="markdown-text")
 
 
 
 
 
 
 
 
 
 
 
 
56
 
57
+ # with gr.Row():
58
+ # gr.Markdown("# ✉️✨ Submit your model here!", elem_classes="markdown-text")
 
 
 
 
 
 
 
 
 
 
 
 
59
 
60
+
 
 
 
 
 
 
 
 
61
 
62
  scheduler = BackgroundScheduler()
63
  scheduler.add_job(restart_space, "interval", seconds=1800)
data/models.yaml CHANGED
@@ -34,6 +34,8 @@
34
  fdir: 'gpt-4-0125-preview'
35
  - name: 'gpt-4-turbo-2024-04-09'
36
  fdir: 'gpt-4-turbo-2024-04-09'
 
 
37
  - name: 'claude-3-opus'
38
  fdir: 'claude-3-opus-20240229'
39
  - name: 'claude-3-haiku'
 
34
  fdir: 'gpt-4-0125-preview'
35
  - name: 'gpt-4-turbo-2024-04-09'
36
  fdir: 'gpt-4-turbo-2024-04-09'
37
+ - name: 'gpt-4o'
38
+ fdir: 'gpt-4o'
39
  - name: 'claude-3-opus'
40
  fdir: 'claude-3-opus-20240229'
41
  - name: 'claude-3-haiku'
predictions/gpt-4o.jsonl ADDED
The diff for this file is too large to render. See raw diff
 
predictions/gpt-4o_swap.jsonl ADDED
The diff for this file is too large to render. See raw diff
 
requirements.txt CHANGED
@@ -11,8 +11,4 @@ pandas==2.0.0
11
  python-dateutil==2.8.2
12
  requests==2.28.2
13
  tqdm==4.65.0
14
- transformers==4.35.2
15
- tokenizers>=0.15.0
16
- git+https://github.com/EleutherAI/lm-evaluation-harness.git@b281b0921b636bc36ad05c0b0b0763bd6dd43463#egg=lm-eval
17
- accelerate==0.24.1
18
  sentencepiece
 
11
  python-dateutil==2.8.2
12
  requests==2.28.2
13
  tqdm==4.65.0
 
 
 
 
14
  sentencepiece
src/about.py CHANGED
@@ -1,72 +1,55 @@
1
- from dataclasses import dataclass
2
- from enum import Enum
3
-
4
- @dataclass
5
- class Task:
6
- benchmark: str
7
- metric: str
8
- col_name: str
9
-
10
-
11
- # Select your tasks here
12
  # ---------------------------------------------------
13
- class Tasks(Enum):
14
- # task_key in the json file, metric_key in the json file, name to display in the leaderboard
15
- task0 = Task("anli_r1", "acc", "ANLI")
16
- task1 = Task("logiqa", "acc_norm", "LogiQA")
17
-
18
- NUM_FEWSHOT = 0 # Change with your few shot
19
- # ---------------------------------------------------
20
-
21
-
22
 
23
  # Your leaderboard name
24
- TITLE = """<h1 align="center" id="space-title">InstruSumEval leaderboard</h1>"""
25
 
26
  # What does your leaderboard evaluate?
27
  INTRODUCTION_TEXT = """
28
- ## This leaderboard evaluates the *evaluation* capabilities of language models on the InstruSum benchmark.
 
 
 
 
 
 
 
 
29
  """
30
 
31
  # Which evaluations are you running? how can people reproduce what you have?
32
  LLM_BENCHMARKS_TEXT = f"""
33
  ## How it works
34
 
35
- ## Reproducibility
36
- To reproduce our results, here is the commands you can run:
37
-
38
- """
39
 
40
- EVALUATION_QUEUE_TEXT = """
41
- ## Some good practices before submitting a model
 
42
 
43
- ### 1) Make sure you can load your model and tokenizer using AutoClasses:
44
- ```python
45
- from transformers import AutoConfig, AutoModel, AutoTokenizer
46
- config = AutoConfig.from_pretrained("your model name", revision=revision)
47
- model = AutoModel.from_pretrained("your model name", revision=revision)
48
- tokenizer = AutoTokenizer.from_pretrained("your model name", revision=revision)
49
- ```
50
- If this step fails, follow the error messages to debug your model before submitting it. It's likely your model has been improperly uploaded.
51
 
52
- Note: make sure your model is public!
53
- Note: if your model needs `use_remote_code=True`, we do not support this option yet but we are working on adding it, stay posted!
 
 
54
 
55
- ### 2) Convert your model weights to [safetensors](https://huggingface.co/docs/safetensors/index)
56
- It's a new format for storing weights which is safer and faster to load and use. It will also allow us to add the number of parameters of your model to the `Extended Viewer`!
 
57
 
58
- ### 3) Make sure your model has an open license!
59
- This is a leaderboard for Open LLMs, and we'd love for as many people as possible to know they can use your model 🤗
60
 
61
- ### 4) Fill up your model card
62
- When we add extra information about models to the leaderboard, it will be automatically taken from the model card
63
-
64
- ## In case of model failure
65
- If your model is displayed in the `FAILED` category, its execution stopped.
66
- Make sure you have followed the above steps first.
67
- If everything is done, check you can launch the EleutherAIHarness on your model locally, using the above command without modifications (you can add `--limit` to limit the number of examples per task).
68
  """
69
 
70
- CITATION_BUTTON_LABEL = "Copy the following snippet to cite these results"
71
- CITATION_BUTTON_TEXT = r"""
72
- """
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
  # ---------------------------------------------------
 
 
 
 
 
 
 
 
 
2
 
3
  # Your leaderboard name
4
+ TITLE = """<h1 align="center" id="space-title">InstruSumEval Leaderboard</h1>"""
5
 
6
  # What does your leaderboard evaluate?
7
  INTRODUCTION_TEXT = """
8
+ - This leaderboard evaluates the *evaluation* capabilities of language models on the [InstruSum](https://huggingface.co/datasets/Salesforce/InstruSum) benchmark from our paper ["Benchmarking Generation and Evaluation Capabilities of Large Language Models for Instruction Controllable Summarization"](https://arxiv.org/abs/2311.09184).
9
+ - InstruSum is a benchmark for instruction-controllable summarization, where the goal is to generate summaries that satisfy user-provided instructions.
10
+ - The benchmark contains human evaluations for the generated summaries, on which the models are evaluated as judges for *long-context* instruction-following.
11
+
12
+ ### Metrics
13
+ - **Accuracy**: The percentage of times the model agrees with the human evaluator.
14
+ - **Agreement**: The Cohen's Kappa score between the model and human evaluator.
15
+ - **Self-Accuracy**: The percentage of times the model agrees with itself when the inputs are swapped.
16
+ - **Self-Agreement**: The Cohen's Kappa score between the model and itself when the inputs are swapped.
17
  """
18
 
19
  # Which evaluations are you running? how can people reproduce what you have?
20
  LLM_BENCHMARKS_TEXT = f"""
21
  ## How it works
22
 
23
+ ### Task
24
+ The LLMs are evaluated as judges in a pairwise comparison task.
25
+ Each judge is presented with two **instruction-controllable** summaries and asked to select the better one.
26
+ The model's accuracy and agreement with the human evaluator are then calculated.
27
 
28
+ ### Dataset
29
+ The human annotations are from the [InstruSum](https://huggingface.co/datasets/Salesforce/InstruSum) dataset.
30
+ Its pairwise annotation [subset](https://huggingface.co/datasets/Salesforce/InstruSum/viewer/human_eval_pairwise) is used for evaluation.
31
 
32
+ This subset contains converted pairwise human evaluation results based on the human evaluation results in the [`human_eval`](https://huggingface.co/datasets/Salesforce/InstruSum/viewer/human_eval) subset.
 
 
 
 
 
 
 
33
 
34
+ The conversion process is as follows:
35
+ - The ranking-based human evaluation results are convered into pairwise comparisons for the *overall quality* aspect.
36
+ - Only comparisons where the annotators reached a consensus are included.
37
+ - Comparisons that resulted in a tie are excluded.
38
 
39
+ ### Evaluation Details
40
+ - The instruction-controllable summarization is treated as a *long-context* instruction-following task.
41
+ Therefore, the source article and the instruction is combined to form a single instruction for the model to follow.
42
 
43
+ - The LLMs are evaluated on the pairwise comparison task.
44
+ The [prompt](https://github.com/princeton-nlp/LLMBar/blob/main/LLMEvaluator/evaluators/prompts/comparison/Vanilla.txt) from [LLMBar](https://github.com/princeton-nlp/LLMBar) is adopted for the evaluation.
45
 
46
+ - The pairwise comparison is conducted bidirectionally. The model's responses are swapped to evaluate the self-agreement.
 
 
 
 
 
 
47
  """
48
 
49
+ CITATION_BUTTON_LABEL = "Please cite our paper if you use InstruSum in your work."
50
+ CITATION_BUTTON_TEXT = r"""@article{liu2023benchmarking,
51
+ title={Benchmarking generation and evaluation capabilities of large language models for instruction controllable summarization},
52
+ author={Liu, Yixin and Fabbri, Alexander R and Chen, Jiawen and Zhao, Yilun and Han, Simeng and Joty, Shafiq and Liu, Pengfei and Radev, Dragomir and Wu, Chien-Sheng and Cohan, Arman},
53
+ journal={arXiv preprint arXiv:2311.09184},
54
+ year={2023}
55
+ }"""
src/display/utils.py DELETED
@@ -1,135 +0,0 @@
1
- from dataclasses import dataclass, make_dataclass
2
- from enum import Enum
3
-
4
- import pandas as pd
5
-
6
- from src.about import Tasks
7
-
8
- def fields(raw_class):
9
- return [v for k, v in raw_class.__dict__.items() if k[:2] != "__" and k[-2:] != "__"]
10
-
11
-
12
- # These classes are for user facing column names,
13
- # to avoid having to change them all around the code
14
- # when a modif is needed
15
- @dataclass
16
- class ColumnContent:
17
- name: str
18
- type: str
19
- displayed_by_default: bool
20
- hidden: bool = False
21
- never_hidden: bool = False
22
-
23
- ## Leaderboard columns
24
- auto_eval_column_dict = []
25
- # Init
26
- auto_eval_column_dict.append(["model_type_symbol", ColumnContent, ColumnContent("T", "str", True, never_hidden=True)])
27
- auto_eval_column_dict.append(["model", ColumnContent, ColumnContent("Model", "markdown", True, never_hidden=True)])
28
- #Scores
29
- auto_eval_column_dict.append(["average", ColumnContent, ColumnContent("Average ⬆️", "number", True)])
30
- for task in Tasks:
31
- auto_eval_column_dict.append([task.name, ColumnContent, ColumnContent(task.value.col_name, "number", True)])
32
- # Model information
33
- auto_eval_column_dict.append(["model_type", ColumnContent, ColumnContent("Type", "str", False)])
34
- auto_eval_column_dict.append(["architecture", ColumnContent, ColumnContent("Architecture", "str", False)])
35
- auto_eval_column_dict.append(["weight_type", ColumnContent, ColumnContent("Weight type", "str", False, True)])
36
- auto_eval_column_dict.append(["precision", ColumnContent, ColumnContent("Precision", "str", False)])
37
- auto_eval_column_dict.append(["license", ColumnContent, ColumnContent("Hub License", "str", False)])
38
- auto_eval_column_dict.append(["params", ColumnContent, ColumnContent("#Params (B)", "number", False)])
39
- auto_eval_column_dict.append(["likes", ColumnContent, ColumnContent("Hub ❤️", "number", False)])
40
- auto_eval_column_dict.append(["still_on_hub", ColumnContent, ColumnContent("Available on the hub", "bool", False)])
41
- auto_eval_column_dict.append(["revision", ColumnContent, ColumnContent("Model sha", "str", False, False)])
42
-
43
- # We use make dataclass to dynamically fill the scores from Tasks
44
- AutoEvalColumn = make_dataclass("AutoEvalColumn", auto_eval_column_dict, frozen=True)
45
-
46
- ## For the queue columns in the submission tab
47
- @dataclass(frozen=True)
48
- class EvalQueueColumn: # Queue column
49
- model = ColumnContent("model", "markdown", True)
50
- revision = ColumnContent("revision", "str", True)
51
- private = ColumnContent("private", "bool", True)
52
- precision = ColumnContent("precision", "str", True)
53
- weight_type = ColumnContent("weight_type", "str", "Original")
54
- status = ColumnContent("status", "str", True)
55
-
56
- ## All the model information that we might need
57
- @dataclass
58
- class ModelDetails:
59
- name: str
60
- display_name: str = ""
61
- symbol: str = "" # emoji
62
-
63
-
64
- class ModelType(Enum):
65
- PT = ModelDetails(name="pretrained", symbol="🟢")
66
- FT = ModelDetails(name="fine-tuned", symbol="🔶")
67
- IFT = ModelDetails(name="instruction-tuned", symbol="⭕")
68
- RL = ModelDetails(name="RL-tuned", symbol="🟦")
69
- Unknown = ModelDetails(name="", symbol="?")
70
-
71
- def to_str(self, separator=" "):
72
- return f"{self.value.symbol}{separator}{self.value.name}"
73
-
74
- @staticmethod
75
- def from_str(type):
76
- if "fine-tuned" in type or "🔶" in type:
77
- return ModelType.FT
78
- if "pretrained" in type or "🟢" in type:
79
- return ModelType.PT
80
- if "RL-tuned" in type or "🟦" in type:
81
- return ModelType.RL
82
- if "instruction-tuned" in type or "⭕" in type:
83
- return ModelType.IFT
84
- return ModelType.Unknown
85
-
86
- class WeightType(Enum):
87
- Adapter = ModelDetails("Adapter")
88
- Original = ModelDetails("Original")
89
- Delta = ModelDetails("Delta")
90
-
91
- class Precision(Enum):
92
- float16 = ModelDetails("float16")
93
- bfloat16 = ModelDetails("bfloat16")
94
- float32 = ModelDetails("float32")
95
- #qt_8bit = ModelDetails("8bit")
96
- #qt_4bit = ModelDetails("4bit")
97
- #qt_GPTQ = ModelDetails("GPTQ")
98
- Unknown = ModelDetails("?")
99
-
100
- def from_str(precision):
101
- if precision in ["torch.float16", "float16"]:
102
- return Precision.float16
103
- if precision in ["torch.bfloat16", "bfloat16"]:
104
- return Precision.bfloat16
105
- if precision in ["float32"]:
106
- return Precision.float32
107
- #if precision in ["8bit"]:
108
- # return Precision.qt_8bit
109
- #if precision in ["4bit"]:
110
- # return Precision.qt_4bit
111
- #if precision in ["GPTQ", "None"]:
112
- # return Precision.qt_GPTQ
113
- return Precision.Unknown
114
-
115
- # Column selection
116
- COLS = [c.name for c in fields(AutoEvalColumn) if not c.hidden]
117
- TYPES = [c.type for c in fields(AutoEvalColumn) if not c.hidden]
118
- COLS_LITE = [c.name for c in fields(AutoEvalColumn) if c.displayed_by_default and not c.hidden]
119
- TYPES_LITE = [c.type for c in fields(AutoEvalColumn) if c.displayed_by_default and not c.hidden]
120
-
121
- EVAL_COLS = [c.name for c in fields(EvalQueueColumn)]
122
- EVAL_TYPES = [c.type for c in fields(EvalQueueColumn)]
123
-
124
- BENCHMARK_COLS = [t.value.col_name for t in Tasks]
125
-
126
- NUMERIC_INTERVALS = {
127
- "?": pd.Interval(-1, 0, closed="right"),
128
- "~1.5": pd.Interval(0, 2, closed="right"),
129
- "~3": pd.Interval(2, 4, closed="right"),
130
- "~7": pd.Interval(4, 9, closed="right"),
131
- "~13": pd.Interval(9, 20, closed="right"),
132
- "~35": pd.Interval(20, 45, closed="right"),
133
- "~60": pd.Interval(45, 70, closed="right"),
134
- "70+": pd.Interval(70, 10000, closed="right"),
135
- }
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
src/dummy_leaderboard.py DELETED
@@ -1,5 +0,0 @@
1
- import pandas as pd
2
-
3
- # HEADERS = ["score1", "score2", "score3", "score4"]
4
- TYPES = ["str", "number"]
5
- DUMMY_LEADERBOARD = pd.DataFrame({"Model": ["gpt4", "gpt3"], "Score1": [0.1, 0.2], "Score2": [0.3, 0.4]})
 
 
 
 
 
 
src/envs.py CHANGED
@@ -6,20 +6,11 @@ from huggingface_hub import HfApi
6
  # ----------------------------------
7
  TOKEN = os.environ.get("TOKEN") # A read/write token for your org
8
 
9
- # OWNER = "demo-leaderboard-backend" # Change to your org - don't forget to create a results and request dataset, with the correct format!
10
  # ----------------------------------
11
 
12
  REPO_ID = f"yale-nlp/instrusumeval"
13
- QUEUE_REPO = f"demo-leaderboard-backend/requests"
14
- RESULTS_REPO = f"demo-leaderboard-backend/results"
15
 
16
  # If you setup a cache later, just change HF_HOME
17
  CACHE_PATH=os.getenv("HF_HOME", ".")
18
 
19
- # Local caches
20
- EVAL_REQUESTS_PATH = os.path.join(CACHE_PATH, "eval-queue")
21
- EVAL_RESULTS_PATH = os.path.join(CACHE_PATH, "eval-results")
22
- EVAL_REQUESTS_PATH_BACKEND = os.path.join(CACHE_PATH, "eval-queue-bk")
23
- EVAL_RESULTS_PATH_BACKEND = os.path.join(CACHE_PATH, "eval-results-bk")
24
-
25
  API = HfApi(token=TOKEN)
 
6
  # ----------------------------------
7
  TOKEN = os.environ.get("TOKEN") # A read/write token for your org
8
 
 
9
  # ----------------------------------
10
 
11
  REPO_ID = f"yale-nlp/instrusumeval"
 
 
12
 
13
  # If you setup a cache later, just change HF_HOME
14
  CACHE_PATH=os.getenv("HF_HOME", ".")
15
 
 
 
 
 
 
 
16
  API = HfApi(token=TOKEN)
src/leaderboard/read_evals.py DELETED
@@ -1,196 +0,0 @@
1
- import glob
2
- import json
3
- import math
4
- import os
5
- from dataclasses import dataclass
6
-
7
- import dateutil
8
- import numpy as np
9
-
10
- from src.display.formatting import make_clickable_model
11
- from src.display.utils import AutoEvalColumn, ModelType, Tasks, Precision, WeightType
12
- from src.submission.check_validity import is_model_on_hub
13
-
14
-
15
- @dataclass
16
- class EvalResult:
17
- """Represents one full evaluation. Built from a combination of the result and request file for a given run.
18
- """
19
- eval_name: str # org_model_precision (uid)
20
- full_model: str # org/model (path on hub)
21
- org: str
22
- model: str
23
- revision: str # commit hash, "" if main
24
- results: dict
25
- precision: Precision = Precision.Unknown
26
- model_type: ModelType = ModelType.Unknown # Pretrained, fine tuned, ...
27
- weight_type: WeightType = WeightType.Original # Original or Adapter
28
- architecture: str = "Unknown"
29
- license: str = "?"
30
- likes: int = 0
31
- num_params: int = 0
32
- date: str = "" # submission date of request file
33
- still_on_hub: bool = False
34
-
35
- @classmethod
36
- def init_from_json_file(self, json_filepath):
37
- """Inits the result from the specific model result file"""
38
- with open(json_filepath) as fp:
39
- data = json.load(fp)
40
-
41
- config = data.get("config")
42
-
43
- # Precision
44
- precision = Precision.from_str(config.get("model_dtype"))
45
-
46
- # Get model and org
47
- org_and_model = config.get("model_name", config.get("model_args", None))
48
- org_and_model = org_and_model.split("/", 1)
49
-
50
- if len(org_and_model) == 1:
51
- org = None
52
- model = org_and_model[0]
53
- result_key = f"{model}_{precision.value.name}"
54
- else:
55
- org = org_and_model[0]
56
- model = org_and_model[1]
57
- result_key = f"{org}_{model}_{precision.value.name}"
58
- full_model = "/".join(org_and_model)
59
-
60
- still_on_hub, _, model_config = is_model_on_hub(
61
- full_model, config.get("model_sha", "main"), trust_remote_code=True, test_tokenizer=False
62
- )
63
- architecture = "?"
64
- if model_config is not None:
65
- architectures = getattr(model_config, "architectures", None)
66
- if architectures:
67
- architecture = ";".join(architectures)
68
-
69
- # Extract results available in this file (some results are split in several files)
70
- results = {}
71
- for task in Tasks:
72
- task = task.value
73
-
74
- # We average all scores of a given metric (not all metrics are present in all files)
75
- accs = np.array([v.get(task.metric, None) for k, v in data["results"].items() if task.benchmark == k])
76
- if accs.size == 0 or any([acc is None for acc in accs]):
77
- continue
78
-
79
- mean_acc = np.mean(accs) * 100.0
80
- results[task.benchmark] = mean_acc
81
-
82
- return self(
83
- eval_name=result_key,
84
- full_model=full_model,
85
- org=org,
86
- model=model,
87
- results=results,
88
- precision=precision,
89
- revision= config.get("model_sha", ""),
90
- still_on_hub=still_on_hub,
91
- architecture=architecture
92
- )
93
-
94
- def update_with_request_file(self, requests_path):
95
- """Finds the relevant request file for the current model and updates info with it"""
96
- request_file = get_request_file_for_model(requests_path, self.full_model, self.precision.value.name)
97
-
98
- try:
99
- with open(request_file, "r") as f:
100
- request = json.load(f)
101
- self.model_type = ModelType.from_str(request.get("model_type", ""))
102
- self.weight_type = WeightType[request.get("weight_type", "Original")]
103
- self.license = request.get("license", "?")
104
- self.likes = request.get("likes", 0)
105
- self.num_params = request.get("params", 0)
106
- self.date = request.get("submitted_time", "")
107
- except Exception:
108
- print(f"Could not find request file for {self.org}/{self.model} with precision {self.precision.value.name}")
109
-
110
- def to_dict(self):
111
- """Converts the Eval Result to a dict compatible with our dataframe display"""
112
- average = sum([v for v in self.results.values() if v is not None]) / len(Tasks)
113
- data_dict = {
114
- "eval_name": self.eval_name, # not a column, just a save name,
115
- AutoEvalColumn.precision.name: self.precision.value.name,
116
- AutoEvalColumn.model_type.name: self.model_type.value.name,
117
- AutoEvalColumn.model_type_symbol.name: self.model_type.value.symbol,
118
- AutoEvalColumn.weight_type.name: self.weight_type.value.name,
119
- AutoEvalColumn.architecture.name: self.architecture,
120
- AutoEvalColumn.model.name: make_clickable_model(self.full_model),
121
- AutoEvalColumn.revision.name: self.revision,
122
- AutoEvalColumn.average.name: average,
123
- AutoEvalColumn.license.name: self.license,
124
- AutoEvalColumn.likes.name: self.likes,
125
- AutoEvalColumn.params.name: self.num_params,
126
- AutoEvalColumn.still_on_hub.name: self.still_on_hub,
127
- }
128
-
129
- for task in Tasks:
130
- data_dict[task.value.col_name] = self.results[task.value.benchmark]
131
-
132
- return data_dict
133
-
134
-
135
- def get_request_file_for_model(requests_path, model_name, precision):
136
- """Selects the correct request file for a given model. Only keeps runs tagged as FINISHED"""
137
- request_files = os.path.join(
138
- requests_path,
139
- f"{model_name}_eval_request_*.json",
140
- )
141
- request_files = glob.glob(request_files)
142
-
143
- # Select correct request file (precision)
144
- request_file = ""
145
- request_files = sorted(request_files, reverse=True)
146
- for tmp_request_file in request_files:
147
- with open(tmp_request_file, "r") as f:
148
- req_content = json.load(f)
149
- if (
150
- req_content["status"] in ["FINISHED"]
151
- and req_content["precision"] == precision.split(".")[-1]
152
- ):
153
- request_file = tmp_request_file
154
- return request_file
155
-
156
-
157
- def get_raw_eval_results(results_path: str, requests_path: str) -> list[EvalResult]:
158
- """From the path of the results folder root, extract all needed info for results"""
159
- model_result_filepaths = []
160
-
161
- for root, _, files in os.walk(results_path):
162
- # We should only have json files in model results
163
- if len(files) == 0 or any([not f.endswith(".json") for f in files]):
164
- continue
165
-
166
- # Sort the files by date
167
- try:
168
- files.sort(key=lambda x: x.removesuffix(".json").removeprefix("results_")[:-7])
169
- except dateutil.parser._parser.ParserError:
170
- files = [files[-1]]
171
-
172
- for file in files:
173
- model_result_filepaths.append(os.path.join(root, file))
174
-
175
- eval_results = {}
176
- for model_result_filepath in model_result_filepaths:
177
- # Creation of result
178
- eval_result = EvalResult.init_from_json_file(model_result_filepath)
179
- eval_result.update_with_request_file(requests_path)
180
-
181
- # Store results of same eval together
182
- eval_name = eval_result.eval_name
183
- if eval_name in eval_results.keys():
184
- eval_results[eval_name].results.update({k: v for k, v in eval_result.results.items() if v is not None})
185
- else:
186
- eval_results[eval_name] = eval_result
187
-
188
- results = []
189
- for v in eval_results.values():
190
- try:
191
- v.to_dict() # we test if the dict version is complete
192
- results.append(v)
193
- except KeyError: # not all eval values present
194
- continue
195
-
196
- return results
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
src/populate.py CHANGED
@@ -1,24 +1,22 @@
1
  import json
2
- import os
3
 
4
  import pandas as pd
5
 
6
- from src.display.formatting import has_no_nan_values, make_clickable_model
7
- from src.display.utils import AutoEvalColumn, EvalQueueColumn
8
- from src.leaderboard.read_evals import get_raw_eval_results
9
  import yaml
10
  from sklearn.metrics import cohen_kappa_score
11
  import numpy as np
 
12
 
13
  TYPES = ["str", "number", "number", "number", "number", "number"]
14
 
 
15
  def read_json(file_path: str) -> list[dict]:
16
  """
17
  Read a JSON/JSONL file and return its contents as a list of dictionaries.
18
-
19
  Parameters:
20
  file_path (str): The path to the JSON file.
21
-
22
  Returns:
23
  list[dict]: The contents of the JSON file as a list of dictionaries.
24
  """
@@ -31,75 +29,70 @@ def read_json(file_path: str) -> list[dict]:
31
  data = json.load(f)
32
  return data
33
 
 
34
  def pairwise_compare(
35
- evaluator1_dir: str,
36
- evaluator2_dir: str,
37
  ) -> tuple[float, float]:
38
  """
39
  Compare pairwise evaluators.
40
 
41
  Args:
42
- evaluator1_dir: The directory containing the responses from the first evaluator.
43
- evaluator2_dir: The directory containing the responses from the second evaluator.
44
-
45
  Returns:
46
  None
47
  """
48
 
49
- evaluator1_responses = read_json(evaluator1_dir)
50
- evaluator2_responses = read_json(evaluator2_dir)
51
  assert len(evaluator1_responses) == len(evaluator2_responses)
52
- evaluator1_winners = np.array(
53
- [response["winner"] for response in evaluator1_responses]
54
- )
55
- evaluator2_winners = np.array(
56
- [response["winner"] for response in evaluator2_responses]
57
- )
58
  acc = (evaluator1_winners == evaluator2_winners).mean().item()
59
  agreement = cohen_kappa_score(evaluator1_winners, evaluator2_winners)
60
  return acc, agreement
61
 
62
 
63
- def pairwise_meta_eval(
64
- human_dir: str,
65
- model_dir: str,
66
- model_dir_swap: str
67
- ) -> dict[float]:
68
  """
69
  Evaluate a pairwise evaluator.
70
 
71
  Args:
72
- human_dir: The directory containing the human responses.
73
  model_dir: The directory containing the model responses.
74
  model_dir_swap: The directory containing the model responses with swapped inputs.
75
 
76
  Returns:
77
  dict[float]: The accuracy and agreement.
78
  """
79
- acc, agr = pairwise_compare(human_dir, model_dir)
 
 
80
  swap_acc, swap_agr = pairwise_compare(
81
- human_dir, model_dir_swap,
 
82
  )
83
  acc = (acc + swap_acc) / 2
84
  agr = (agr + swap_agr) / 2
85
  models_acc, models_agr = pairwise_compare(
86
- model_dir, model_dir_swap,
 
87
  )
88
  return acc, agr, models_acc, models_agr
89
 
 
90
  def load_leaderboard() -> pd.DataFrame:
91
  """Loads the leaderboard from the file system"""
92
  with open("./data/models.yaml") as fp:
93
  models = yaml.safe_load(fp)
 
 
94
 
95
  predictions = {k: [] for k in ["Model", "Accuracy", "Agreement", "Self-Accuracy", "Self-Agreement"]}
96
 
97
  for model in models:
98
  fdir = model["fdir"]
99
  acc, agr, models_acc, models_agr = pairwise_meta_eval(
100
- f"./data/instrusum.json",
101
- f"./predictions/{fdir}.jsonl",
102
- f"./predictions/{fdir}_swap.jsonl"
103
  )
104
  predictions["Model"].append(model["name"])
105
  predictions["Accuracy"].append(acc)
@@ -107,55 +100,3 @@ def load_leaderboard() -> pd.DataFrame:
107
  predictions["Self-Accuracy"].append(models_acc)
108
  predictions["Self-Agreement"].append(models_agr)
109
  return pd.DataFrame(predictions).sort_values(by="Agreement", ascending=False).round(decimals=3)
110
-
111
-
112
-
113
-
114
- def get_leaderboard_df(results_path: str, requests_path: str, cols: list, benchmark_cols: list) -> pd.DataFrame:
115
- """Creates a dataframe from all the individual experiment results"""
116
- raw_data = get_raw_eval_results(results_path, requests_path)
117
- all_data_json = [v.to_dict() for v in raw_data]
118
-
119
- df = pd.DataFrame.from_records(all_data_json)
120
- df = df.sort_values(by=[AutoEvalColumn.average.name], ascending=False)
121
- df = df[cols].round(decimals=2)
122
-
123
- # filter out if any of the benchmarks have not been produced
124
- df = df[has_no_nan_values(df, benchmark_cols)]
125
- return raw_data, df
126
-
127
-
128
- def get_evaluation_queue_df(save_path: str, cols: list) -> list[pd.DataFrame]:
129
- """Creates the different dataframes for the evaluation queues requestes"""
130
- entries = [entry for entry in os.listdir(save_path) if not entry.startswith(".")]
131
- all_evals = []
132
-
133
- for entry in entries:
134
- if ".json" in entry:
135
- file_path = os.path.join(save_path, entry)
136
- with open(file_path) as fp:
137
- data = json.load(fp)
138
-
139
- data[EvalQueueColumn.model.name] = make_clickable_model(data["model"])
140
- data[EvalQueueColumn.revision.name] = data.get("revision", "main")
141
-
142
- all_evals.append(data)
143
- elif ".md" not in entry:
144
- # this is a folder
145
- sub_entries = [e for e in os.listdir(f"{save_path}/{entry}") if not e.startswith(".")]
146
- for sub_entry in sub_entries:
147
- file_path = os.path.join(save_path, entry, sub_entry)
148
- with open(file_path) as fp:
149
- data = json.load(fp)
150
-
151
- data[EvalQueueColumn.model.name] = make_clickable_model(data["model"])
152
- data[EvalQueueColumn.revision.name] = data.get("revision", "main")
153
- all_evals.append(data)
154
-
155
- pending_list = [e for e in all_evals if e["status"] in ["PENDING", "RERUN"]]
156
- running_list = [e for e in all_evals if e["status"] == "RUNNING"]
157
- finished_list = [e for e in all_evals if e["status"].startswith("FINISHED") or e["status"] == "PENDING_NEW_EVAL"]
158
- df_pending = pd.DataFrame.from_records(pending_list, columns=cols)
159
- df_running = pd.DataFrame.from_records(running_list, columns=cols)
160
- df_finished = pd.DataFrame.from_records(finished_list, columns=cols)
161
- return df_finished[cols], df_running[cols], df_pending[cols]
 
1
  import json
 
2
 
3
  import pandas as pd
4
 
 
 
 
5
  import yaml
6
  from sklearn.metrics import cohen_kappa_score
7
  import numpy as np
8
+ from datasets import load_dataset
9
 
10
  TYPES = ["str", "number", "number", "number", "number", "number"]
11
 
12
+
13
  def read_json(file_path: str) -> list[dict]:
14
  """
15
  Read a JSON/JSONL file and return its contents as a list of dictionaries.
16
+
17
  Parameters:
18
  file_path (str): The path to the JSON file.
19
+
20
  Returns:
21
  list[dict]: The contents of the JSON file as a list of dictionaries.
22
  """
 
29
  data = json.load(f)
30
  return data
31
 
32
+
33
  def pairwise_compare(
34
+ evaluator1_responses: list[dict],
35
+ evaluator2_responses: list[dict],
36
  ) -> tuple[float, float]:
37
  """
38
  Compare pairwise evaluators.
39
 
40
  Args:
41
+ evaluator1_responses: The responses from the first evaluator.
42
+ evaluator2_responses: The responses from the second evaluator.
 
43
  Returns:
44
  None
45
  """
46
 
 
 
47
  assert len(evaluator1_responses) == len(evaluator2_responses)
48
+ evaluator1_winners = np.array([response["winner"] for response in evaluator1_responses])
49
+ evaluator2_winners = np.array([response["winner"] for response in evaluator2_responses])
 
 
 
 
50
  acc = (evaluator1_winners == evaluator2_winners).mean().item()
51
  agreement = cohen_kappa_score(evaluator1_winners, evaluator2_winners)
52
  return acc, agreement
53
 
54
 
55
+ def pairwise_meta_eval(human_responses: list[dict], model_dir: str, model_dir_swap: str) -> dict[float]:
 
 
 
 
56
  """
57
  Evaluate a pairwise evaluator.
58
 
59
  Args:
60
+ human_responses: The responses from the human evaluator.
61
  model_dir: The directory containing the model responses.
62
  model_dir_swap: The directory containing the model responses with swapped inputs.
63
 
64
  Returns:
65
  dict[float]: The accuracy and agreement.
66
  """
67
+ model_responses = read_json(model_dir)
68
+ model_responses_swap = read_json(model_dir_swap)
69
+ acc, agr = pairwise_compare(human_responses, model_responses)
70
  swap_acc, swap_agr = pairwise_compare(
71
+ human_responses,
72
+ model_responses_swap,
73
  )
74
  acc = (acc + swap_acc) / 2
75
  agr = (agr + swap_agr) / 2
76
  models_acc, models_agr = pairwise_compare(
77
+ model_responses,
78
+ model_responses_swap,
79
  )
80
  return acc, agr, models_acc, models_agr
81
 
82
+
83
  def load_leaderboard() -> pd.DataFrame:
84
  """Loads the leaderboard from the file system"""
85
  with open("./data/models.yaml") as fp:
86
  models = yaml.safe_load(fp)
87
+ human_responses = load_dataset("salesforce/instrusum", "human_eval_pairwise")["data"]
88
+ human_responses = [x for x in human_responses]
89
 
90
  predictions = {k: [] for k in ["Model", "Accuracy", "Agreement", "Self-Accuracy", "Self-Agreement"]}
91
 
92
  for model in models:
93
  fdir = model["fdir"]
94
  acc, agr, models_acc, models_agr = pairwise_meta_eval(
95
+ human_responses, f"./predictions/{fdir}.jsonl", f"./predictions/{fdir}_swap.jsonl"
 
 
96
  )
97
  predictions["Model"].append(model["name"])
98
  predictions["Accuracy"].append(acc)
 
100
  predictions["Self-Accuracy"].append(models_acc)
101
  predictions["Self-Agreement"].append(models_agr)
102
  return pd.DataFrame(predictions).sort_values(by="Agreement", ascending=False).round(decimals=3)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
src/submission/check_validity.py DELETED
@@ -1,99 +0,0 @@
1
- import json
2
- import os
3
- import re
4
- from collections import defaultdict
5
- from datetime import datetime, timedelta, timezone
6
-
7
- import huggingface_hub
8
- from huggingface_hub import ModelCard
9
- from huggingface_hub.hf_api import ModelInfo
10
- from transformers import AutoConfig
11
- from transformers.models.auto.tokenization_auto import AutoTokenizer
12
-
13
- def check_model_card(repo_id: str) -> tuple[bool, str]:
14
- """Checks if the model card and license exist and have been filled"""
15
- try:
16
- card = ModelCard.load(repo_id)
17
- except huggingface_hub.utils.EntryNotFoundError:
18
- return False, "Please add a model card to your model to explain how you trained/fine-tuned it."
19
-
20
- # Enforce license metadata
21
- if card.data.license is None:
22
- if not ("license_name" in card.data and "license_link" in card.data):
23
- return False, (
24
- "License not found. Please add a license to your model card using the `license` metadata or a"
25
- " `license_name`/`license_link` pair."
26
- )
27
-
28
- # Enforce card content
29
- if len(card.text) < 200:
30
- return False, "Please add a description to your model card, it is too short."
31
-
32
- return True, ""
33
-
34
- def is_model_on_hub(model_name: str, revision: str, token: str = None, trust_remote_code=False, test_tokenizer=False) -> tuple[bool, str]:
35
- """Checks if the model model_name is on the hub, and whether it (and its tokenizer) can be loaded with AutoClasses."""
36
- try:
37
- config = AutoConfig.from_pretrained(model_name, revision=revision, trust_remote_code=trust_remote_code, token=token)
38
- if test_tokenizer:
39
- try:
40
- tk = AutoTokenizer.from_pretrained(model_name, revision=revision, trust_remote_code=trust_remote_code, token=token)
41
- except ValueError as e:
42
- return (
43
- False,
44
- f"uses a tokenizer which is not in a transformers release: {e}",
45
- None
46
- )
47
- except Exception as e:
48
- return (False, "'s tokenizer cannot be loaded. Is your tokenizer class in a stable transformers release, and correctly configured?", None)
49
- return True, None, config
50
-
51
- except ValueError:
52
- return (
53
- False,
54
- "needs to be launched with `trust_remote_code=True`. For safety reason, we do not allow these models to be automatically submitted to the leaderboard.",
55
- None
56
- )
57
-
58
- except Exception as e:
59
- return False, "was not found on hub!", None
60
-
61
-
62
- def get_model_size(model_info: ModelInfo, precision: str):
63
- """Gets the model size from the configuration, or the model name if the configuration does not contain the information."""
64
- try:
65
- model_size = round(model_info.safetensors["total"] / 1e9, 3)
66
- except (AttributeError, TypeError):
67
- return 0 # Unknown model sizes are indicated as 0, see NUMERIC_INTERVALS in app.py
68
-
69
- size_factor = 8 if (precision == "GPTQ" or "gptq" in model_info.modelId.lower()) else 1
70
- model_size = size_factor * model_size
71
- return model_size
72
-
73
- def get_model_arch(model_info: ModelInfo):
74
- """Gets the model architecture from the configuration"""
75
- return model_info.config.get("architectures", "Unknown")
76
-
77
- def already_submitted_models(requested_models_dir: str) -> set[str]:
78
- """Gather a list of already submitted models to avoid duplicates"""
79
- depth = 1
80
- file_names = []
81
- users_to_submission_dates = defaultdict(list)
82
-
83
- for root, _, files in os.walk(requested_models_dir):
84
- current_depth = root.count(os.sep) - requested_models_dir.count(os.sep)
85
- if current_depth == depth:
86
- for file in files:
87
- if not file.endswith(".json"):
88
- continue
89
- with open(os.path.join(root, file), "r") as f:
90
- info = json.load(f)
91
- file_names.append(f"{info['model']}_{info['revision']}_{info['precision']}")
92
-
93
- # Select organisation
94
- if info["model"].count("/") == 0 or "submitted_time" not in info:
95
- continue
96
- organisation, _ = info["model"].split("/")
97
- users_to_submission_dates[organisation].append(info["submitted_time"])
98
-
99
- return set(file_names), users_to_submission_dates
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
src/submission/submit.py DELETED
@@ -1,119 +0,0 @@
1
- import json
2
- import os
3
- from datetime import datetime, timezone
4
-
5
- from src.display.formatting import styled_error, styled_message, styled_warning
6
- from src.envs import API, EVAL_REQUESTS_PATH, TOKEN, QUEUE_REPO
7
- from src.submission.check_validity import (
8
- already_submitted_models,
9
- check_model_card,
10
- get_model_size,
11
- is_model_on_hub,
12
- )
13
-
14
- REQUESTED_MODELS = None
15
- USERS_TO_SUBMISSION_DATES = None
16
-
17
- def add_new_eval(
18
- model: str,
19
- base_model: str,
20
- revision: str,
21
- precision: str,
22
- weight_type: str,
23
- model_type: str,
24
- ):
25
- global REQUESTED_MODELS
26
- global USERS_TO_SUBMISSION_DATES
27
- if not REQUESTED_MODELS:
28
- REQUESTED_MODELS, USERS_TO_SUBMISSION_DATES = already_submitted_models(EVAL_REQUESTS_PATH)
29
-
30
- user_name = ""
31
- model_path = model
32
- if "/" in model:
33
- user_name = model.split("/")[0]
34
- model_path = model.split("/")[1]
35
-
36
- precision = precision.split(" ")[0]
37
- current_time = datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
38
-
39
- if model_type is None or model_type == "":
40
- return styled_error("Please select a model type.")
41
-
42
- # Does the model actually exist?
43
- if revision == "":
44
- revision = "main"
45
-
46
- # Is the model on the hub?
47
- if weight_type in ["Delta", "Adapter"]:
48
- base_model_on_hub, error, _ = is_model_on_hub(model_name=base_model, revision=revision, token=TOKEN, test_tokenizer=True)
49
- if not base_model_on_hub:
50
- return styled_error(f'Base model "{base_model}" {error}')
51
-
52
- if not weight_type == "Adapter":
53
- model_on_hub, error, _ = is_model_on_hub(model_name=model, revision=revision, token=TOKEN, test_tokenizer=True)
54
- if not model_on_hub:
55
- return styled_error(f'Model "{model}" {error}')
56
-
57
- # Is the model info correctly filled?
58
- try:
59
- model_info = API.model_info(repo_id=model, revision=revision)
60
- except Exception:
61
- return styled_error("Could not get your model information. Please fill it up properly.")
62
-
63
- model_size = get_model_size(model_info=model_info, precision=precision)
64
-
65
- # Were the model card and license filled?
66
- try:
67
- license = model_info.cardData["license"]
68
- except Exception:
69
- return styled_error("Please select a license for your model")
70
-
71
- modelcard_OK, error_msg = check_model_card(model)
72
- if not modelcard_OK:
73
- return styled_error(error_msg)
74
-
75
- # Seems good, creating the eval
76
- print("Adding new eval")
77
-
78
- eval_entry = {
79
- "model": model,
80
- "base_model": base_model,
81
- "revision": revision,
82
- "precision": precision,
83
- "weight_type": weight_type,
84
- "status": "PENDING",
85
- "submitted_time": current_time,
86
- "model_type": model_type,
87
- "likes": model_info.likes,
88
- "params": model_size,
89
- "license": license,
90
- "private": False,
91
- }
92
-
93
- # Check for duplicate submission
94
- if f"{model}_{revision}_{precision}" in REQUESTED_MODELS:
95
- return styled_warning("This model has been already submitted.")
96
-
97
- print("Creating eval file")
98
- OUT_DIR = f"{EVAL_REQUESTS_PATH}/{user_name}"
99
- os.makedirs(OUT_DIR, exist_ok=True)
100
- out_path = f"{OUT_DIR}/{model_path}_eval_request_False_{precision}_{weight_type}.json"
101
-
102
- with open(out_path, "w") as f:
103
- f.write(json.dumps(eval_entry))
104
-
105
- print("Uploading eval file")
106
- API.upload_file(
107
- path_or_fileobj=out_path,
108
- path_in_repo=out_path.split("eval-queue/")[1],
109
- repo_id=QUEUE_REPO,
110
- repo_type="dataset",
111
- commit_message=f"Add {model} to eval queue",
112
- )
113
-
114
- # Remove the local file
115
- os.remove(out_path)
116
-
117
- return styled_message(
118
- "Your request has been submitted to the evaluation queue!\nPlease wait for up to an hour for the model to show in the PENDING list."
119
- )