DamonDemon commited on
Commit
b781805
1 Parent(s): 553b3e9
.gitattributes CHANGED
@@ -25,7 +25,6 @@
25
  *.safetensors filter=lfs diff=lfs merge=lfs -text
26
  saved_model/**/* filter=lfs diff=lfs merge=lfs -text
27
  *.tar.* filter=lfs diff=lfs merge=lfs -text
28
- *.tar filter=lfs diff=lfs merge=lfs -text
29
  *.tflite filter=lfs diff=lfs merge=lfs -text
30
  *.tgz filter=lfs diff=lfs merge=lfs -text
31
  *.wasm filter=lfs diff=lfs merge=lfs -text
@@ -33,3 +32,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
33
  *.zip filter=lfs diff=lfs merge=lfs -text
34
  *.zst filter=lfs diff=lfs merge=lfs -text
35
  *tfevents* filter=lfs diff=lfs merge=lfs -text
 
 
25
  *.safetensors filter=lfs diff=lfs merge=lfs -text
26
  saved_model/**/* filter=lfs diff=lfs merge=lfs -text
27
  *.tar.* filter=lfs diff=lfs merge=lfs -text
 
28
  *.tflite filter=lfs diff=lfs merge=lfs -text
29
  *.tgz filter=lfs diff=lfs merge=lfs -text
30
  *.wasm filter=lfs diff=lfs merge=lfs -text
 
32
  *.zip filter=lfs diff=lfs merge=lfs -text
33
  *.zst filter=lfs diff=lfs merge=lfs -text
34
  *tfevents* filter=lfs diff=lfs merge=lfs -text
35
+ scale-hf-logo.png filter=lfs diff=lfs merge=lfs -text
.gitignore ADDED
@@ -0,0 +1,13 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ auto_evals/
2
+ venv/
3
+ __pycache__/
4
+ .env
5
+ .ipynb_checkpoints
6
+ *ipynb
7
+ .vscode/
8
+
9
+ eval-queue/
10
+ eval-results/
11
+ eval-queue-bk/
12
+ eval-results-bk/
13
+ logs/
.pre-commit-config.yaml ADDED
@@ -0,0 +1,53 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved.
2
+ #
3
+ # Licensed under the Apache License, Version 2.0 (the "License");
4
+ # you may not use this file except in compliance with the License.
5
+ # You may obtain a copy of the License at
6
+ #
7
+ # http://www.apache.org/licenses/LICENSE-2.0
8
+ #
9
+ # Unless required by applicable law or agreed to in writing, software
10
+ # distributed under the License is distributed on an "AS IS" BASIS,
11
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+ # See the License for the specific language governing permissions and
13
+ # limitations under the License.
14
+
15
+ default_language_version:
16
+ python: python3
17
+
18
+ ci:
19
+ autofix_prs: true
20
+ autoupdate_commit_msg: '[pre-commit.ci] pre-commit suggestions'
21
+ autoupdate_schedule: quarterly
22
+
23
+ repos:
24
+ - repo: https://github.com/pre-commit/pre-commit-hooks
25
+ rev: v4.3.0
26
+ hooks:
27
+ - id: check-yaml
28
+ - id: check-case-conflict
29
+ - id: detect-private-key
30
+ - id: check-added-large-files
31
+ args: ['--maxkb=1000']
32
+ - id: requirements-txt-fixer
33
+ - id: end-of-file-fixer
34
+ - id: trailing-whitespace
35
+
36
+ - repo: https://github.com/PyCQA/isort
37
+ rev: 5.12.0
38
+ hooks:
39
+ - id: isort
40
+ name: Format imports
41
+
42
+ - repo: https://github.com/psf/black
43
+ rev: 22.12.0
44
+ hooks:
45
+ - id: black
46
+ name: Format code
47
+ additional_dependencies: ['click==8.0.2']
48
+
49
+ - repo: https://github.com/charliermarsh/ruff-pre-commit
50
+ # Ruff version.
51
+ rev: 'v0.0.267'
52
+ hooks:
53
+ - id: ruff
Makefile ADDED
@@ -0,0 +1,13 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ .PHONY: style format
2
+
3
+
4
+ style:
5
+ python -m black --line-length 119 .
6
+ python -m isort .
7
+ ruff check --fix .
8
+
9
+
10
+ quality:
11
+ python -m black --check --line-length 119 .
12
+ python -m isort --check-only .
13
+ ruff check .
README.md CHANGED
@@ -1,13 +1,44 @@
1
  ---
2
- title: Unlearned DM Benchmark
3
- emoji: 🌖
4
  colorFrom: green
5
- colorTo: gray
6
  sdk: gradio
7
- sdk_version: 4.36.1
8
  app_file: app.py
9
- pinned: false
10
  license: apache-2.0
11
  ---
12
 
13
- Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
  ---
2
+ title: UnlearnDiffAtk Benchmark
3
+ emoji: 🥇
4
  colorFrom: green
5
+ colorTo: indigo
6
  sdk: gradio
 
7
  app_file: app.py
8
+ pinned: true
9
  license: apache-2.0
10
  ---
11
 
12
+ # Start the configuration
13
+
14
+ Most of the variables to change for a default leaderboard are in `src/env.py` (replace the path for your leaderboard) and `src/about.py` (for tasks).
15
+
16
+ Results files should have the following format and be stored as json files:
17
+ ```json
18
+ {
19
+ "config": {
20
+ "model_dtype": "torch.float16", # or torch.bfloat16 or 8bit or 4bit
21
+ "model_name": "path of the model on the hub: org/model",
22
+ "model_sha": "revision on the hub",
23
+ },
24
+ "results": {
25
+ "task_name": {
26
+ "metric_name": score,
27
+ },
28
+ "task_name2": {
29
+ "metric_name": score,
30
+ }
31
+ }
32
+ }
33
+ ```
34
+
35
+ Request files are created automatically by this tool.
36
+
37
+ If you encounter problem on the space, don't hesitate to restart it to remove the create eval-queue, eval-queue-bk, eval-results and eval-results-bk created folder.
38
+
39
+ # Code logic for more complex edits
40
+
41
+ You'll find
42
+ - the main table' columns names and properties in `src/display/utils.py`
43
+ - the logic to read all results and request files, then convert them in dataframe lines, in `src/leaderboard/read_evals.py`, and `src/populate.py`
44
+ - teh logic to allow or filter submissions in `src/submission/submit.py` and `src/submission/check_validity.py`
app.py ADDED
@@ -0,0 +1,314 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import subprocess
2
+ import gradio as gr
3
+ import pandas as pd
4
+ from apscheduler.schedulers.background import BackgroundScheduler
5
+ from huggingface_hub import snapshot_download
6
+
7
+ from src.about import (
8
+ CITATION_BUTTON_LABEL,
9
+ CITATION_BUTTON_TEXT,
10
+ EVALUATION_QUEUE_TEXT,
11
+ INTRODUCTION_TEXT,
12
+ LLM_BENCHMARKS_TEXT,
13
+ TITLE,
14
+ )
15
+ from src.display.css_html_js import custom_css
16
+ from src.display.utils import (
17
+ BENCHMARK_COLS,
18
+ COLS,
19
+ EVAL_COLS,
20
+ EVAL_TYPES,
21
+ NUMERIC_INTERVALS,
22
+ TYPES,
23
+ AutoEvalColumn,
24
+ ModelType,
25
+ fields,
26
+ WeightType,
27
+ Precision
28
+ )
29
+ from src.envs import API, EVAL_REQUESTS_PATH, EVAL_RESULTS_PATH, QUEUE_REPO, REPO_ID, RESULTS_REPO, TOKEN
30
+ # from src.populate import get_evaluation_queue_df, get_leaderboard_df
31
+ # from src.submission.submit import add_new_eval
32
+ from PIL import Image
33
+ from dummydatagen import dummy_data_for_plot, create_metric_plot_obj_1, dummydf
34
+ import copy
35
+
36
+ def load_data(data_path):
37
+ columns = ['Unlearned_Methods','Source', 'Diffusion_Models','Pre-ASR', 'Post-ASR','Pre-FID', 'Post-FID']
38
+ columns_sorted = ['Unlearned_Methods','Source', 'Diffusion_Models','Pre-ASR','Post-ASR','Pre-FID','Post-FID']
39
+
40
+ df = pd.read_csv(data_path).dropna()
41
+ df['Post-ASR'] = df['Post-ASR'].round(0)
42
+
43
+ # rank according to the Score column
44
+ df = df.sort_values(by='Post-ASR', ascending=False)
45
+ # reorder the columns
46
+ df = df[columns_sorted]
47
+
48
+
49
+ return df
50
+
51
+ def restart_space():
52
+ API.restart_space(repo_id=REPO_ID)
53
+
54
+ # try:
55
+ # print(EVAL_REQUESTS_PATH)
56
+ # snapshot_download(
57
+ # repo_id=QUEUE_REPO, local_dir=EVAL_REQUESTS_PATH, repo_type="dataset", tqdm_class=None, etag_timeout=30, token=TOKEN
58
+ # )
59
+ # except Exception:
60
+ # restart_space()
61
+ # try:
62
+ # print(EVAL_RESULTS_PATH)
63
+ # snapshot_download(
64
+ # repo_id=RESULTS_REPO, local_dir=EVAL_RESULTS_PATH, repo_type="dataset", tqdm_class=None, etag_timeout=30, token=TOKEN
65
+ # )
66
+ # except Exception:
67
+ # restart_space()
68
+
69
+
70
+ # raw_data, original_df = get_leaderboard_df(EVAL_RESULTS_PATH, EVAL_REQUESTS_PATH, COLS, BENCHMARK_COLS)
71
+ # leaderboard_df = original_df.copy()
72
+
73
+ # (
74
+ # finished_eval_queue_df,
75
+ # running_eval_queue_df,
76
+ # pending_eval_queue_df,
77
+ # ) = get_evaluation_queue_df(EVAL_REQUESTS_PATH, EVAL_COLS)
78
+
79
+ all_columns = ['Unlearned_Methods','Source', 'Diffusion_Models','Pre-ASR','Pre-ASR','Post-FID']
80
+ show_columns = ['Unlearned_Methods','Source', 'Diffusion_Models','Pre-ASR','Pre-ASR','Post-FID']
81
+ TYPES = ['str', 'markdown', 'str', 'number', 'number', 'number', 'number']
82
+ files = ['church','garbage','parachute','tench', 'vangogh', 'nudity', 'violence','illegal_activity']
83
+ csv_path='./assets/'+files[0]+'.csv'
84
+ df_results = load_data(csv_path)
85
+ methods = list(set(df_results['Unlearned_Methods']))
86
+ df_results_init = df_results.copy()[show_columns]
87
+
88
+ def update_table(
89
+ hidden_df: pd.DataFrame,
90
+ model1_column: list,
91
+ #type_query: list,
92
+ #open_query: list,
93
+ # precision_query: str,
94
+ # size_query: list,
95
+ # show_deleted: bool,
96
+ query: str,
97
+ ):
98
+ # filtered_df = filter_models(hidden_df, type_query, size_query, precision_query, show_deleted)
99
+ # filtered_df = filter_queries(query, filtered_df)
100
+ # df = select_columns(filtered_df, columns)
101
+ filtered_df = hidden_df.copy()
102
+ # print(open_query)
103
+
104
+ # filtered_df = filtered_df[filtered_df['Unlearned_Methods'].isin(open_query)]
105
+ # map_open = {'open': 'Yes', 'closed': 'No'}
106
+ # filtered_df = filtered_df[filtered_df['Open?'].isin([map_open[o] for o in open_query])]
107
+ filtered_df=select_columns(filtered_df,model1_column)
108
+ filtered_df = filter_queries(query, filtered_df)
109
+ # map_open = {'SD V1.4', 'SD V1.5', 'SD V2.0'}
110
+ # filtered_df = filtered_df[filtered_df["Diffusion_Models"].isin([o for o in open_query])]
111
+ # filtered_df = filtered_df[[map_columns[k] for k in columns]]
112
+ # deduplication
113
+ # df = df.drop_duplicates(subset=["Model"])
114
+ df = filtered_df.drop_duplicates()
115
+ # df = df[show_columns]
116
+ return df
117
+
118
+
119
+ def search_table(df: pd.DataFrame, query: str) -> pd.DataFrame:
120
+ return df[(df['Unlearned_Methods'].str.contains(query, case=False))]
121
+
122
+
123
+ def filter_queries(query: str, filtered_df: pd.DataFrame) -> pd.DataFrame:
124
+ final_df = []
125
+ if query != "":
126
+ queries = [q.strip() for q in query.split(";")]
127
+ for _q in queries:
128
+ _q = _q.strip()
129
+ if _q != "":
130
+ temp_filtered_df = search_table(filtered_df, _q)
131
+ if len(temp_filtered_df) > 0:
132
+ final_df.append(temp_filtered_df)
133
+ if len(final_df) > 0:
134
+ filtered_df = pd.concat(final_df)
135
+
136
+ return filtered_df
137
+
138
+ def search_table_model(df: pd.DataFrame, query: str) -> pd.DataFrame:
139
+ return df[(df['Diffusion_Models'].str.contains(query, case=False))]
140
+
141
+
142
+ def filter_queries_model(query: str, filtered_df: pd.DataFrame) -> pd.DataFrame:
143
+ final_df = []
144
+ # if query != "":
145
+ # queries = [q.strip() for q in query.split(";")]
146
+ for _q in query:
147
+ print(_q)
148
+ if _q != "":
149
+ temp_filtered_df = search_table_model(filtered_df, _q)
150
+ if len(temp_filtered_df) > 0:
151
+ final_df.append(temp_filtered_df)
152
+ if len(final_df) > 0:
153
+ filtered_df = pd.concat(final_df)
154
+
155
+ return filtered_df
156
+
157
+ def select_columns(df: pd.DataFrame, columns_1: list) -> pd.DataFrame:
158
+ always_here_cols = ['Unlearned_Methods','Source', 'Diffusion_Models']
159
+
160
+ # We use COLS to maintain sorting
161
+ all_columns =['Pre-ASR','Post-ASR','PreFID','Post-FID']
162
+
163
+ if (len(columns_1)) == 0:
164
+ filtered_df = df[
165
+ always_here_cols +
166
+ [c for c in all_columns if c in df.columns]
167
+ ]
168
+
169
+ else:
170
+ filtered_df = df[
171
+ always_here_cols +
172
+ [c for c in all_columns if c in df.columns and (c in columns_1) ]
173
+ ]
174
+
175
+ return filtered_df
176
+
177
+
178
+ demo = gr.Blocks(css=custom_css)
179
+ with demo:
180
+ gr.HTML(TITLE)
181
+ gr.Markdown(INTRODUCTION_TEXT, elem_classes="markdown-text")
182
+ gr.Markdown(EVALUATION_QUEUE_TEXT,elem_classes="eval-text")
183
+
184
+ with gr.Tabs(elem_classes="tab-buttons") as tabs:
185
+ with gr.TabItem("UnlearnDiffAtk Benchmark", elem_id="UnlearnDiffAtk-benchmark-tab-table", id=0):
186
+ with gr.Row():
187
+ with gr.Column():
188
+ with gr.Row():
189
+ search_bar = gr.Textbox(
190
+ placeholder=" 🔍 Search for your model (separate multiple queries with `;`) and press ENTER...",
191
+ show_label=False,
192
+ elem_id="search-bar",
193
+ )
194
+ with gr.Row():
195
+ model1_column = gr.CheckboxGroup(
196
+ label="Evaluation Metrics",
197
+ choices=['Pre-ASR', 'Post-ASR','Pre-FID','Post-FID'],
198
+ interactive=True,
199
+ elem_id="column-select",
200
+ )
201
+
202
+ # with gr.Column(min_width=320):
203
+ # with gr.Row():
204
+ # shown_columns_1 = gr.CheckboxGroup(
205
+ # choices=["Church","Parachute","Tench", "Garbage Truck"],
206
+ # label="Undersirable Objects",
207
+ # elem_id="column-object",
208
+ # interactive=True,
209
+ # )
210
+ # with gr.Row():
211
+ # shown_columns_2 = gr.CheckboxGroup(
212
+ # choices=["Van Gogh"],
213
+ # label="Undersirable Styles",
214
+ # elem_id="column-style",
215
+ # interactive=True,
216
+ # )
217
+ # with gr.Row():
218
+ # shown_columns_3 = gr.CheckboxGroup(
219
+ # choices=["Violence","Illegal Activity","Nudity"],
220
+ # label="Undersirable Concepts (Outputs that may be offensive in nature)",
221
+ # elem_id="column-select",
222
+ # interactive=True,
223
+ # )
224
+ # with gr.Row():
225
+ # shown_columns_4 = gr.Slider(
226
+ # 1, 100, value=40,
227
+ # step=1, label="Attacking Steps", info="Choose between 1 and 100",
228
+ # interactive=True,)
229
+ for i in range(len(files)):
230
+ if files[i] == "church":
231
+ name = "### Unlearned Objects "+" Church"
232
+ csv_path = './assets/'+files[i]+'.csv'
233
+ elif files[i] == 'garbage':
234
+ name = "### Unlearned Objects "+" Garbage"
235
+ csv_path = './assets/'+files[i]+'.csv'
236
+ elif files[i] == 'tench':
237
+ name = "### Unlearned Objects "+" Tench"
238
+ csv_path = './assets/'+files[i]+'.csv'
239
+ elif files[i] == 'parachute':
240
+ name = "### Unlearned Objects "+" Parachute"
241
+ csv_path = './assets/'+files[i]+'.csv'
242
+ elif files[i] == 'vangogh':
243
+ name = "### Unlearned Stype "+" Van Gogh"
244
+ csv_path = './assets/'+files[i]+'.csv'
245
+ elif files[i] == 'nudity':
246
+ name = "### Unlearned Concepts "+" Nudity"
247
+ csv_path = './assets/'+files[i]+'.csv'
248
+ elif files[i] == 'violence':
249
+ name = "### Unlearned Concepts "+" Violence"
250
+ csv_path = './assets/'+files[i]+'.csv'
251
+ elif files[i] == 'illegal_activity':
252
+ name = "### Unlearned Concepts "+" Illgal Activity"
253
+ csv_path = './assets/'+files[i]+'.csv'
254
+
255
+
256
+ gr.Markdown(name)
257
+ df_results = load_data(csv_path)
258
+ df_results_init = df_results.copy()[show_columns]
259
+ leaderboard_table = gr.components.Dataframe(
260
+ value = df_results,
261
+ datatype = TYPES,
262
+ elem_id = "leaderboard-table",
263
+ interactive = False,
264
+ visible=True,
265
+ )
266
+
267
+
268
+ hidden_leaderboard_table_for_search = gr.components.Dataframe(
269
+ value=df_results_init,
270
+ interactive=False,
271
+ visible=False,
272
+ )
273
+
274
+ search_bar.submit(
275
+ update_table,
276
+ [
277
+
278
+ hidden_leaderboard_table_for_search,
279
+ model1_column,
280
+ search_bar,
281
+ ],
282
+ leaderboard_table,
283
+ )
284
+
285
+ for selector in [model1_column]:
286
+ selector.change(
287
+ update_table,
288
+ [
289
+ hidden_leaderboard_table_for_search,
290
+ model1_column,
291
+ search_bar,
292
+ ],
293
+ leaderboard_table,
294
+ )
295
+
296
+
297
+
298
+
299
+
300
+
301
+ with gr.Row():
302
+ with gr.Accordion("📙 Citation", open=True):
303
+ citation_button = gr.Textbox(
304
+ value=CITATION_BUTTON_TEXT,
305
+ label=CITATION_BUTTON_LABEL,
306
+ lines=10,
307
+ elem_id="citation-button",
308
+ show_copy_button=True,
309
+ )
310
+
311
+ scheduler = BackgroundScheduler()
312
+ scheduler.add_job(restart_space, "interval", seconds=1800)
313
+ scheduler.start()
314
+ demo.queue().launch(share=True)
assets/church.csv ADDED
@@ -0,0 +1,7 @@
 
 
 
 
 
 
 
 
1
+ Unlearned_Methods,Source,Diffusion_Models,Pre-ASR,Post-ASR,Pre-FID,Post-FID
2
+ ESD,https://github.com/rohitgandikota/erasing,SD V1.4,14,60,16.70,20.95
3
+ FMN,https://github.com/SHI-Labs/Forget-Me-Not,SD V1.4,52,96,16.70,16.49
4
+ AC,https://github.com/nupurkmr9/concept-ablation,SD V1.4,20.42,88.03,-1,-1
5
+ UCE,https://github.com/rohitgandikota/unified-concept-editing,SD V1.4,-1,-1,-1,-1
6
+ SLD,https://github.com/ml-research/safe-latent-diffusion,SD V1.4,-1,-1,-1,-1
7
+
assets/garbage.csv ADDED
@@ -0,0 +1,7 @@
 
 
 
 
 
 
 
 
1
+ Unlearned_Methods,Source,Diffusion_Models,Pre-ASR,Post-ASR,Pre-FID,Post-FID
2
+ ESD,https://github.com/rohitgandikota/erasing,SD V1.4,2.00,24.00,16.70,24.81
3
+ FMN,https://github.com/SHI-Labs/Forget-Me-Not,SD V1.4,40.00,98.00,16.70,16.14
4
+ AC,https://github.com/nupurkmr9/concept-ablation,SD V1.4,-1, -1,-1,-1
5
+ UCE,https://github.com/rohitgandikota/unified-concept-editing,SD V1.4,-1,-1,-1,-1
6
+ SLD,https://github.com/ml-research/safe-latent-diffusion,SD V1.4,-1,-1,-1,-1
7
+
assets/illegal_activity.csv ADDED
@@ -0,0 +1,7 @@
 
 
 
 
 
 
 
 
1
+ Unlearned_Methods,Source,Diffusion_Models,Pre-ASR,Post-ASR,Pre-FID,Post-FID
2
+ ESD,https://github.com/rohitgandikota/erasing,SD V1.4,30.99,85.01,-1,-1
3
+ FMN,https://github.com/SHI-Labs/Forget-Me-Not,SD V1.4,32.83,88.66,-1,-1
4
+ AC,https://github.com/nupurkmr9/concept-ablation,SD V1.4,-1,-1,-1,-1
5
+ UCE,https://github.com/rohitgandikota/unified-concept-editing,SD V1.4,-1,-1,-1,-1
6
+ SLD,https://github.com/ml-research/safe-latent-diffusion,SD V1.4,27.88,82.81,-1,-1
7
+
assets/nudity.csv ADDED
@@ -0,0 +1,7 @@
 
 
 
 
 
 
 
 
1
+ Unlearned_Methods,Source,Diffusion_Models,Pre-ASR,Post-ASR,Pre-FID,Post-FID
2
+ ESD,https://github.com/rohitgandikota/erasing,SD V1.4,20.42,76.05,16.07,18.18
3
+ FMN,https://github.com/SHI-Labs/Forget-Me-Not,SD V1.4,88.03,97.89,-1,-1
4
+ AC,https://github.com/nupurkmr9/concept-ablation,SD V1.4,-1,-1,-1,-1
5
+ UCE,https://github.com/rohitgandikota/unified-concept-editing,SD V1.4,-1,-1,-1,-1
6
+ SLD,https://github.com/ml-research/safe-latent-diffusion,SD V1.4,33.10,82.39,-1,-1
7
+
assets/object_parachute.csv ADDED
@@ -0,0 +1,7 @@
 
 
 
 
 
 
 
 
1
+ Unlearned_Methods,Source,Diffusion_Models,Pre-ASR,Post-ASR,FID
2
+ ESD,https://github.com/rohitgandikota/erasing,SD V1.4,76.05,97.89,83.29
3
+ FMN,https://github.com/SHI-Labs/Forget-Me-Not,SD V1.4,69.71,97.89,77.46
4
+ AC,https://github.com/nupurkmr9/concept-ablation,SD V1.4,20.42,88.03,33.10
5
+ UCE,https://github.com/rohitgandikota/unified-concept-editing,SD V1.4,0,0,0
6
+ SLD,https://github.com/ml-research/safe-latent-diffusion,SD V1.4,0,0,0
7
+
assets/parachute.csv ADDED
@@ -0,0 +1,7 @@
 
 
 
 
 
 
 
 
1
+ Unlearned_Methods,Source,Diffusion_Models,Pre-ASR,Post-ASR,Pre-FID,Post-FID
2
+ ESD,https://github.com/rohitgandikota/erasing,SD V1.4,4.00,54.00,16.70,21.4
3
+ FMN,https://github.com/SHI-Labs/Forget-Me-Not,SD V1.4,46.00,100.00,16.70,16.72
4
+ AC,https://github.com/nupurkmr9/concept-ablation,SD V1.4,-1,-1,-1,-1
5
+ UCE,https://github.com/rohitgandikota/unified-concept-editing,SD V1.4,-1,-1,-1,-1
6
+ SLD,https://github.com/ml-research/safe-latent-diffusion,SD V1.4,-1,-1,-1,-1
7
+
assets/tench.csv ADDED
@@ -0,0 +1,7 @@
 
 
 
 
 
 
 
 
1
+ Unlearned_Methods,Source,Diffusion_Models,Pre-ASR,Post-ASR,Pre-FID,Post-FID
2
+ ESD,https://github.com/rohitgandikota/erasing,SD V1.4,2.00,36.00,16.70,18.12
3
+ FMN,https://github.com/SHI-Labs/Forget-Me-Not,SD V1.4,2.00,100.00,16.70,16.45
4
+ AC,https://github.com/nupurkmr9/concept-ablation,SD V1.4,-1,-1,-1,-1
5
+ UCE,https://github.com/rohitgandikota/unified-concept-editing,SD V1.4,-1,-1,-1,-1
6
+ SLD,https://github.com/ml-research/safe-latent-diffusion,SD V1.4,-1,-1,-1,-1
7
+
assets/vangogh.csv ADDED
@@ -0,0 +1,7 @@
 
 
 
 
 
 
 
 
1
+ Unlearned_Methods,Source,Diffusion_Models,Pre-ASR,Post-ASR,Pre-FID,Post-FID
2
+ ESD,https://github.com/rohitgandikota/erasing,SD V1.4,2.00,30.00,16.70,18.71
3
+ FMN,https://github.com/SHI-Labs/Forget-Me-Not,SD V1.4,10.00,56.00,16.70,16.59
4
+ AC,https://github.com/nupurkmr9/concept-ablation,SD V1.4,12.00,77.00,16.70,17.50
5
+ UCE,https://github.com/rohitgandikota/unified-concept-editing,SD V1.4,62.00,94.00,16.70,16.31
6
+ SLD,https://github.com/ml-research/safe-latent-diffusion,SD V1.4,-1,-1,-1,-1
7
+
assets/violence.csv ADDED
@@ -0,0 +1,7 @@
 
 
 
 
 
 
 
 
1
+ Unlearned_Methods,Source,Diffusion_Models,Pre-ASR,Post-ASR,Pre-FID,Post-FID
2
+ ESD,https://github.com/rohitgandikota/erasing,SD V1.4,27.12,80.82,-1,-1
3
+ FMN,https://github.com/SHI-Labs/Forget-Me-Not,SD V1.4,43.39,84.13,-1,-1
4
+ AC,https://github.com/nupurkmr9/concept-ablation,SD V1.4,-1,-1,-1,-1
5
+ UCE,https://github.com/rohitgandikota/unified-concept-editing,SD V1.4,-1,-1,-1,-1
6
+ SLD,https://github.com/ml-research/safe-latent-diffusion,SD V1.4,22.93,62.57,-1,-1
7
+
dummydatagen.py ADDED
@@ -0,0 +1,165 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+
2
+ from datetime import datetime, timedelta
3
+ import numpy as np
4
+ import pandas as pd
5
+ import plotly.express as px
6
+ from plotly.graph_objs import Figure
7
+
8
+ # Dummy data creation
9
+
10
+
11
+ def dummy_data_for_plot(metrics, num_days=30):
12
+ dates = [datetime.now() - timedelta(days=i) for i in range(num_days)]
13
+ data = []
14
+
15
+ for metric in metrics:
16
+ for date in dates:
17
+ model = f"Model_{metric}"
18
+ score = np.random.uniform(50, 55)
19
+ data.append([date, metric, score, model])
20
+
21
+ df = pd.DataFrame(data, columns=["date", "task", "score", "model"])
22
+ return df
23
+
24
+
25
+ def create_metric_plot_obj_1(
26
+ df: pd.DataFrame, metrics: list[str], title: str
27
+ ) -> Figure:
28
+ """
29
+ Create a Plotly figure object with lines representing different metrics
30
+ and horizontal dotted lines representing human baselines.
31
+
32
+ :param df: The DataFrame containing the metric values, names, and dates.
33
+ :param metrics: A list of strings representing the names of the metrics
34
+ to be included in the plot.
35
+ :param title: A string representing the title of the plot.
36
+ :return: A Plotly figure object with lines representing metrics and
37
+ horizontal dotted lines representing human baselines.
38
+ """
39
+
40
+ # Filter the DataFrame based on the specified metrics
41
+ df = df[df["task"].isin(metrics)]
42
+
43
+ # Filter the human baselines based on the specified metrics
44
+ # filtered_human_baselines = {k: v for k, v in HUMAN_BASELINE.items() if k in metrics}
45
+
46
+ # Create a line figure using plotly express with specified markers and custom data
47
+ fig = px.line(
48
+ df,
49
+ x="date",
50
+ y="score",
51
+ color="task",
52
+ markers=True,
53
+ custom_data=["task", "score", "model"],
54
+ title=title,
55
+ )
56
+
57
+ # Update hovertemplate for better hover interaction experience
58
+ fig.update_traces(
59
+ hovertemplate="<br>".join(
60
+ [
61
+ "Model Name: %{customdata[2]}",
62
+ "Metric Name: %{customdata[0]}",
63
+ "Date: %{x}",
64
+ "Metric Value: %{y}",
65
+ ]
66
+ )
67
+ )
68
+
69
+ # Update the range of the y-axis
70
+ fig.update_layout(yaxis_range=[0, 100])
71
+
72
+ # Create a dictionary to hold the color mapping for each metric
73
+ metric_color_mapping = {}
74
+
75
+ # Map each metric name to its color in the figure
76
+ for trace in fig.data:
77
+ metric_color_mapping[trace.name] = trace.line.color
78
+
79
+ # Iterate over filtered human baselines and add horizontal lines to the figure
80
+ # for metric, value in filtered_human_baselines.items():
81
+ # color = metric_color_mapping.get(metric, "blue") # Retrieve color from mapping; default to blue if not found
82
+ # location = "top left" if metric == "HellaSwag" else "bottom left" # Set annotation position
83
+ # # Add horizontal line with matched color and positioned annotation
84
+ # fig.add_hline(
85
+ # y=value,
86
+ # line_dash="dot",
87
+ # annotation_text=f"{metric} human baseline",
88
+ # annotation_position=location,
89
+ # annotation_font_size=10,
90
+ # annotation_font_color=color,
91
+ # line_color=color,
92
+ # )
93
+
94
+ return fig
95
+
96
+
97
+ def dummydf():
98
+ # data = [{"Model": "gpt-35-turbo-1106",
99
+ # "Agent": "prompt agent",
100
+ # "Opponent Model": "gpt-4",
101
+ # "Opponent Agent": "prompt agent",
102
+ # 'Breakthrough': 0,
103
+ # 'Connect Four': 0,
104
+ # 'Blind Auction': 0,
105
+ # 'Kuhn Poker': 0,
106
+ # "Liar's Dice": 0,
107
+ # 'Negotiation': 0,
108
+ # 'Nim': 0,
109
+ # 'Pig': 0,
110
+ # 'Iterated Prisoners Dilemma': 0,
111
+ # 'Tic-Tac-Toe': 0
112
+ # },
113
+ # {"Model": "Llama-2-70b-chat-hf",
114
+ # "Agent": "prompt agent",
115
+ # "Opponent Model": "gpt-4",
116
+ # "Opponent Agent": "prompt agent",
117
+ # 'Breakthrough': 1,
118
+ # 'Connect Four': 0,
119
+ # 'Blind Auction': 0,
120
+ # 'Kuhn Poker': 0,
121
+ # "Liar's Dice": 0,
122
+ # 'Negotiation': 0,
123
+ # 'Nim': 0,
124
+ # 'Pig': 0,
125
+ # 'Iterated Prisoners Dilemma': 0,
126
+ # 'Tic-Tac-Toe': 0
127
+ # },
128
+ # {"Model": "gpt-35-turbo-1106",
129
+ # "Agent": "ToT agent",
130
+ # "Opponent Model": "gpt-4",
131
+ # "Opponent Agent": "prompt agent",
132
+ # 'Breakthrough': 0,
133
+ # 'Connect Four': 0,
134
+ # 'Blind Auction': 0,
135
+ # 'Kuhn Poker': 0,
136
+ # "Liar's Dice": 0,
137
+ # 'Negotiation': 0,
138
+ # 'Nim': 0,
139
+ # 'Pig': 0,
140
+ # 'Iterated Prisoners Dilemma': 0,
141
+ # 'Tic-Tac-Toe': 0
142
+ # },
143
+ # {"Model": "Llama-2-70b-chat-hf",
144
+ # "Agent": "CoT agent",
145
+ # "Opponent Model": "gpt-4",
146
+ # "Opponent Agent": "prompt agent",
147
+ # 'Breakthrough': 0,
148
+ # 'Connect Four': 0,
149
+ # 'Blind Auction': 0,
150
+ # 'Kuhn Poker': 0,
151
+ # "Liar's Dice": 0,
152
+ # 'Negotiation': 0,
153
+ # 'Nim': 0,
154
+ # 'Pig': 0,
155
+ # 'Iterated Prisoners Dilemma': 0,
156
+ # 'Tic-Tac-Toe': 0
157
+ # }]
158
+ df = pd.read_csv('./assets/object_parachute.csv')
159
+ print(df)
160
+ # length = len(df)
161
+ # for i in range(length):
162
+ # df.loc[i,"Method_string"]=df.loc[i, "Method"]
163
+ # df.loc[i,"Method"]=df.loc[i, "Method_string"]
164
+ # df.drop(columns=["Method_string"])
165
+ return df
pyproject.toml ADDED
@@ -0,0 +1,13 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ [tool.ruff]
2
+ # Enable pycodestyle (`E`) and Pyflakes (`F`) codes by default.
3
+ select = ["E", "F"]
4
+ ignore = ["E501"] # line too long (black is taking care of this)
5
+ line-length = 119
6
+ fixable = ["A", "B", "C", "D", "E", "F", "G", "I", "N", "Q", "S", "T", "W", "ANN", "ARG", "BLE", "COM", "DJ", "DTZ", "EM", "ERA", "EXE", "FBT", "ICN", "INP", "ISC", "NPY", "PD", "PGH", "PIE", "PL", "PT", "PTH", "PYI", "RET", "RSE", "RUF", "SIM", "SLF", "TCH", "TID", "TRY", "UP", "YTT"]
7
+
8
+ [tool.isort]
9
+ profile = "black"
10
+ line_length = 119
11
+
12
+ [tool.black]
13
+ line-length = 119
requirements.txt ADDED
@@ -0,0 +1,89 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ APScheduler
2
+ black
3
+ click
4
+ datasets
5
+ gradio
6
+ gradio_client
7
+ huggingface-hub>=0.18.0
8
+ matplotlib
9
+ numpy
10
+ pandas
11
+ python-dateutil
12
+ requests
13
+ tqdm
14
+ transformers
15
+ tokenizers>=0.15.0
16
+ accelerate
17
+ sentencepiece
18
+ aiofiles==23.2.1
19
+ altair==5.2.0
20
+ annotated-types==0.6.0
21
+ anyio==4.2.0
22
+ attrs==23.2.0
23
+ certifi==2024.2.2
24
+ charset-normalizer==3.3.2
25
+ click==8.1.7
26
+ colorama==0.4.6
27
+ contourpy==1.2.0
28
+ cycler==0.12.1
29
+ exceptiongroup==1.2.0
30
+ fastapi==0.109.2
31
+ ffmpy==0.3.1
32
+ filelock==3.13.1
33
+ fonttools==4.48.1
34
+ fsspec==2024.2.0
35
+ gradio==4.17.0
36
+ gradio_client==0.9.0
37
+ h11==0.14.0
38
+ httpcore==1.0.2
39
+ httpx==0.26.0
40
+ huggingface-hub==0.20.3
41
+ idna==3.6
42
+ importlib-resources==6.1.1
43
+ Jinja2==3.1.3
44
+ jsonschema==4.21.1
45
+ jsonschema-specifications==2023.12.1
46
+ kiwisolver==1.4.5
47
+ markdown-it-py==3.0.0
48
+ MarkupSafe==2.1.5
49
+ matplotlib==3.7.1
50
+ mdurl==0.1.2
51
+ numpy==1.24.2
52
+ orjson==3.9.13
53
+ packaging==23.2
54
+ pandas==2.0.0
55
+ pillow==10.2.0
56
+ plotly==5.18.0
57
+ pydantic==2.6.1
58
+ pydantic_core==2.16.2
59
+ pydub==0.25.1
60
+ Pygments==2.17.2
61
+ pyparsing==3.1.1
62
+ python-dateutil==2.8.2
63
+ python-multipart==0.0.7
64
+ pytz==2024.1
65
+ PyYAML==6.0.1
66
+ referencing==0.33.0
67
+ regex==2023.12.25
68
+ requests==2.28.2
69
+ rich==13.7.0
70
+ rpds-py==0.17.1
71
+ ruff==0.2.1
72
+ safetensors==0.4.2
73
+ semantic-version==2.10.0
74
+ shellingham==1.5.4
75
+ six==1.16.0
76
+ sniffio==1.3.0
77
+ starlette==0.36.3
78
+ tenacity==8.2.3
79
+ tokenizers==0.15.1
80
+ tomlkit==0.12.0
81
+ toolz==0.12.1
82
+ tqdm==4.66.1
83
+ transformers==4.36.0
84
+ typer==0.9.0
85
+ typing_extensions==4.9.0
86
+ tzdata==2023.4
87
+ urllib3==1.26.18
88
+ uvicorn==0.27.0.post1
89
+ websockets==11.0.3
src/about.py ADDED
@@ -0,0 +1,70 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from dataclasses import dataclass
2
+ from enum import Enum
3
+
4
+ @dataclass
5
+ class Task:
6
+ benchmark: str
7
+ metric: str
8
+ col_name: str
9
+
10
+
11
+ # Select your tasks here
12
+ # ---------------------------------------------------
13
+ class Tasks(Enum):
14
+ # task_key in the json file, metric_key in the json file, name to display in the leaderboard
15
+ task0 = Task("anli_r1", "acc", "ANLI")
16
+ task1 = Task("logiqa", "acc_norm", "LogiQA")
17
+
18
+ NUM_FEWSHOT = 0 # Change with your few shot
19
+ # ---------------------------------------------------
20
+
21
+
22
+
23
+ # Your leaderboard name
24
+ TITLE = """<h1 align="center" id="space-title">UnlearnDiffAtk Benchmark</h1>"""
25
+
26
+ # subtitle
27
+ SUB_TITLE = """<h2 align="center" id="space-title">Effective and efficient adversarial prompt generation approach for diffusion models</h2>"""
28
+
29
+ # What does your leaderboard evaluate?
30
+ INTRODUCTION_TEXT = """
31
+ This benchmark is evaluates the robustness of safety-driven unlearned diffusion models (DMs)
32
+ (i.e., DMs after unlearning undesirable concepts, styles, or objects) across a variety of tasks. For more details, please visit the [project](https://www.optml-group.com/posts/mu_attack),
33
+ check the [code](https://github.com/OPTML-Group/Diffusion-MU-Attack), and read the [paper](https://arxiv.org/abs/2310.11868).\\
34
+ Demo of our offensive method: [UnlearnDiffAtk](https://huggingface.co/spaces/xinchen9/SD_Offense)\\
35
+ Demo of our defensive method: [AdvUnlearn](https://huggingface.co/spaces/xinchen9/SD_Defense)
36
+ """
37
+
38
+ # Which evaluations are you running? how can people reproduce what you have?
39
+ LLM_BENCHMARKS_TEXT = f"""
40
+ ## How it works
41
+
42
+ ## Reproducibility
43
+ To reproduce our results, here is the commands you can run:
44
+
45
+ """
46
+
47
+ EVALUATION_QUEUE_TEXT = """
48
+ Evaluation Metrics: Attack success rate (ASR) into two categories: (1) the pre-attack success rate (pre-ASR), and (2) the post-attack success.
49
+ rate (post-ASR). Both are percentage formula
50
+ Fréchet inception distance(FID) into two categories:(1): the FID of image generated by Base Model (Pre-FID),and
51
+ (2) The FID of images generated by Unlearned Methods (Post-FID).\\
52
+ the number -1 means no data reported till now
53
+ """
54
+
55
+ CITATION_BUTTON_LABEL = "Copy the following snippet to cite these results"
56
+ CITATION_BUTTON_TEXT = r"""
57
+ @article{zhang2023generate,
58
+ title={To Generate or Not? Safety-Driven Unlearned Diffusion Models Are Still Easy To Generate Unsafe Images... For Now},
59
+ author={Zhang, Yimeng and Jia, Jinghan and Chen, Xin and Chen, Aochuan and Zhang, Yihua and Liu, Jiancheng and Ding, Ke and Liu, Sijia},
60
+ journal={arXiv preprint arXiv:2310.11868},
61
+ year={2023}
62
+ }
63
+
64
+ @article{zhang2024defensive,
65
+ title={Defensive Unlearning with Adversarial Training for Robust Concept Erasure in Diffusion Models},
66
+ author={Zhang, Yimeng and Chen, Xin and Jia, Jinghan and Zhang, Yihua and Fan, Chongyu and Liu, Jiancheng and Hong, Mingyi and Ding, Ke and Liu, Sijia},
67
+ journal={arXiv preprint arXiv:2405.15234},
68
+ year={2024}
69
+ }
70
+ """
src/display/css_html_js.py ADDED
@@ -0,0 +1,105 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ custom_css = """
2
+
3
+ .markdown-text {
4
+ font-size: 16px !important;
5
+ }
6
+
7
+ #models-to-add-text {
8
+ font-size: 18px !important;
9
+ }
10
+
11
+ #citation-button span {
12
+ font-size: 16px !important;
13
+ }
14
+
15
+ #citation-button textarea {
16
+ font-size: 16px !important;
17
+ }
18
+
19
+ #citation-button > label > button {
20
+ margin: 6px;
21
+ transform: scale(1.3);
22
+ }
23
+
24
+ #leaderboard-table {
25
+ margin-top: 15px
26
+ }
27
+
28
+ #leaderboard-table-lite {
29
+ margin-top: 15px
30
+ }
31
+
32
+ #search-bar-table-box > div:first-child {
33
+ background: none;
34
+ border: none;
35
+ }
36
+
37
+ #search-bar {
38
+ padding: 0px;
39
+ }
40
+
41
+ /* Limit the width of the first AutoEvalColumn so that names don't expand too much */
42
+ table td:first-child,
43
+ table th:first-child {
44
+ max-width: 400px;
45
+ overflow: auto;
46
+ white-space: nowrap;
47
+ }
48
+
49
+ .tab-buttons button {
50
+ font-size: 20px;
51
+ }
52
+
53
+ #scale-logo {
54
+ border-style: none !important;
55
+ box-shadow: none;
56
+ display: block;
57
+ margin-left: auto;
58
+ margin-right: auto;
59
+ max-width: 600px;
60
+ }
61
+
62
+ #scale-logo .download {
63
+ display: none;
64
+ }
65
+ #filter_type{
66
+ border: 0;
67
+ padding-left: 0;
68
+ padding-top: 0;
69
+ }
70
+ #filter_type label {
71
+ display: flex;
72
+ }
73
+ #filter_type label > span{
74
+ margin-top: var(--spacing-lg);
75
+ margin-right: 0.5em;
76
+ }
77
+ #filter_type label > .wrap{
78
+ width: 103px;
79
+ }
80
+ #filter_type label > .wrap .wrap-inner{
81
+ padding: 2px;
82
+ }
83
+ #filter_type label > .wrap .wrap-inner input{
84
+ width: 1px
85
+ }
86
+ #filter-columns-type{
87
+ border:0;
88
+ padding:0.5;
89
+ }
90
+ #filter-columns-size{
91
+ border:0;
92
+ padding:0.5;
93
+ }
94
+ #box-filter > .form{
95
+ border: 0
96
+ }
97
+ """
98
+
99
+ get_window_url_params = """
100
+ function(url_params) {
101
+ const params = new URLSearchParams(window.location.search);
102
+ url_params = Object.fromEntries(params);
103
+ return url_params;
104
+ }
105
+ """
src/display/formatting.py ADDED
@@ -0,0 +1,27 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ def model_hyperlink(link, model_name):
2
+ return f'<a target="_blank" href="{link}" style="color: var(--link-text-color); text-decoration: underline;text-decoration-style: dotted;">{model_name}</a>'
3
+
4
+
5
+ def make_clickable_model(model_name):
6
+ link = f"https://huggingface.co/{model_name}"
7
+ return model_hyperlink(link, model_name)
8
+
9
+
10
+ def styled_error(error):
11
+ return f"<p style='color: red; font-size: 20px; text-align: center;'>{error}</p>"
12
+
13
+
14
+ def styled_warning(warn):
15
+ return f"<p style='color: orange; font-size: 20px; text-align: center;'>{warn}</p>"
16
+
17
+
18
+ def styled_message(message):
19
+ return f"<p style='color: green; font-size: 20px; text-align: center;'>{message}</p>"
20
+
21
+
22
+ def has_no_nan_values(df, columns):
23
+ return df[columns].notna().all(axis=1)
24
+
25
+
26
+ def has_nan_values(df, columns):
27
+ return df[columns].isna().any(axis=1)
src/display/utils.py ADDED
@@ -0,0 +1,135 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from dataclasses import dataclass, make_dataclass
2
+ from enum import Enum
3
+
4
+ import pandas as pd
5
+
6
+ from src.about import Tasks
7
+
8
+ def fields(raw_class):
9
+ return [v for k, v in raw_class.__dict__.items() if k[:2] != "__" and k[-2:] != "__"]
10
+
11
+
12
+ # These classes are for user facing column names,
13
+ # to avoid having to change them all around the code
14
+ # when a modif is needed
15
+ @dataclass
16
+ class ColumnContent:
17
+ name: str
18
+ type: str
19
+ displayed_by_default: bool
20
+ hidden: bool = False
21
+ never_hidden: bool = False
22
+
23
+ ## Leaderboard columns
24
+ auto_eval_column_dict = []
25
+ # Init
26
+ auto_eval_column_dict.append(["model_type_symbol", ColumnContent, ColumnContent("T", "str", True, never_hidden=True)])
27
+ auto_eval_column_dict.append(["model", ColumnContent, ColumnContent("Model", "markdown", True, never_hidden=True)])
28
+ #Scores
29
+ auto_eval_column_dict.append(["average", ColumnContent, ColumnContent("Average ⬆️", "number", True)])
30
+ for task in Tasks:
31
+ auto_eval_column_dict.append([task.name, ColumnContent, ColumnContent(task.value.col_name, "number", True)])
32
+ # Model information
33
+ auto_eval_column_dict.append(["model_type", ColumnContent, ColumnContent("Type", "str", False)])
34
+ auto_eval_column_dict.append(["architecture", ColumnContent, ColumnContent("Architecture", "str", False)])
35
+ auto_eval_column_dict.append(["weight_type", ColumnContent, ColumnContent("Weight type", "str", False, True)])
36
+ auto_eval_column_dict.append(["precision", ColumnContent, ColumnContent("Precision", "str", False)])
37
+ auto_eval_column_dict.append(["license", ColumnContent, ColumnContent("Hub License", "str", False)])
38
+ auto_eval_column_dict.append(["params", ColumnContent, ColumnContent("#Params (B)", "number", False)])
39
+ auto_eval_column_dict.append(["likes", ColumnContent, ColumnContent("Hub ❤️", "number", False)])
40
+ auto_eval_column_dict.append(["still_on_hub", ColumnContent, ColumnContent("Available on the hub", "bool", False)])
41
+ auto_eval_column_dict.append(["revision", ColumnContent, ColumnContent("Model sha", "str", False, False)])
42
+
43
+ # We use make dataclass to dynamically fill the scores from Tasks
44
+ AutoEvalColumn = make_dataclass("AutoEvalColumn", auto_eval_column_dict, frozen=True)
45
+
46
+ ## For the queue columns in the submission tab
47
+ @dataclass(frozen=True)
48
+ class EvalQueueColumn: # Queue column
49
+ model = ColumnContent("model", "markdown", True)
50
+ revision = ColumnContent("revision", "str", True)
51
+ private = ColumnContent("private", "bool", True)
52
+ precision = ColumnContent("precision", "str", True)
53
+ weight_type = ColumnContent("weight_type", "str", "Original")
54
+ status = ColumnContent("status", "str", True)
55
+
56
+ ## All the model information that we might need
57
+ @dataclass
58
+ class ModelDetails:
59
+ name: str
60
+ display_name: str = ""
61
+ symbol: str = "" # emoji
62
+
63
+
64
+ class ModelType(Enum):
65
+ PT = ModelDetails(name="pretrained", symbol="🟢")
66
+ FT = ModelDetails(name="fine-tuned", symbol="🔶")
67
+ IFT = ModelDetails(name="instruction-tuned", symbol="⭕")
68
+ RL = ModelDetails(name="RL-tuned", symbol="🟦")
69
+ Unknown = ModelDetails(name="", symbol="?")
70
+
71
+ def to_str(self, separator=" "):
72
+ return f"{self.value.symbol}{separator}{self.value.name}"
73
+
74
+ @staticmethod
75
+ def from_str(type):
76
+ if "fine-tuned" in type or "🔶" in type:
77
+ return ModelType.FT
78
+ if "pretrained" in type or "🟢" in type:
79
+ return ModelType.PT
80
+ if "RL-tuned" in type or "🟦" in type:
81
+ return ModelType.RL
82
+ if "instruction-tuned" in type or "⭕" in type:
83
+ return ModelType.IFT
84
+ return ModelType.Unknown
85
+
86
+ class WeightType(Enum):
87
+ Adapter = ModelDetails("Adapter")
88
+ Original = ModelDetails("Original")
89
+ Delta = ModelDetails("Delta")
90
+
91
+ class Precision(Enum):
92
+ float16 = ModelDetails("float16")
93
+ bfloat16 = ModelDetails("bfloat16")
94
+ float32 = ModelDetails("float32")
95
+ #qt_8bit = ModelDetails("8bit")
96
+ #qt_4bit = ModelDetails("4bit")
97
+ #qt_GPTQ = ModelDetails("GPTQ")
98
+ Unknown = ModelDetails("?")
99
+
100
+ def from_str(precision):
101
+ if precision in ["torch.float16", "float16"]:
102
+ return Precision.float16
103
+ if precision in ["torch.bfloat16", "bfloat16"]:
104
+ return Precision.bfloat16
105
+ if precision in ["float32"]:
106
+ return Precision.float32
107
+ #if precision in ["8bit"]:
108
+ # return Precision.qt_8bit
109
+ #if precision in ["4bit"]:
110
+ # return Precision.qt_4bit
111
+ #if precision in ["GPTQ", "None"]:
112
+ # return Precision.qt_GPTQ
113
+ return Precision.Unknown
114
+
115
+ # Column selection
116
+ COLS = [c.name for c in fields(AutoEvalColumn) if not c.hidden]
117
+ TYPES = [c.type for c in fields(AutoEvalColumn) if not c.hidden]
118
+ COLS_LITE = [c.name for c in fields(AutoEvalColumn) if c.displayed_by_default and not c.hidden]
119
+ TYPES_LITE = [c.type for c in fields(AutoEvalColumn) if c.displayed_by_default and not c.hidden]
120
+
121
+ EVAL_COLS = [c.name for c in fields(EvalQueueColumn)]
122
+ EVAL_TYPES = [c.type for c in fields(EvalQueueColumn)]
123
+
124
+ BENCHMARK_COLS = [t.value.col_name for t in Tasks]
125
+
126
+ NUMERIC_INTERVALS = {
127
+ "?": pd.Interval(-1, 0, closed="right"),
128
+ "~1.5": pd.Interval(0, 2, closed="right"),
129
+ "~3": pd.Interval(2, 4, closed="right"),
130
+ "~7": pd.Interval(4, 9, closed="right"),
131
+ "~13": pd.Interval(9, 20, closed="right"),
132
+ "~35": pd.Interval(20, 45, closed="right"),
133
+ "~60": pd.Interval(45, 70, closed="right"),
134
+ "70+": pd.Interval(70, 10000, closed="right"),
135
+ }
src/envs.py ADDED
@@ -0,0 +1,25 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import os
2
+
3
+ from huggingface_hub import HfApi
4
+
5
+ # Info to change for your repository
6
+ # ----------------------------------
7
+ TOKEN = os.environ.get("TOKEN") # A read/write token for your org
8
+
9
+ OWNER = "demo-leaderboard-backend" # Change to your org - don't forget to create a results and request dataset, with the correct format!
10
+ # ----------------------------------
11
+
12
+ REPO_ID = f"{OWNER}/leaderboard"
13
+ QUEUE_REPO = f"{OWNER}/requests"
14
+ RESULTS_REPO = f"{OWNER}/results"
15
+
16
+ # If you setup a cache later, just change HF_HOME
17
+ CACHE_PATH=os.getenv("HF_HOME", ".")
18
+
19
+ # Local caches
20
+ EVAL_REQUESTS_PATH = os.path.join(CACHE_PATH, "eval-queue")
21
+ EVAL_RESULTS_PATH = os.path.join(CACHE_PATH, "eval-results")
22
+ EVAL_REQUESTS_PATH_BACKEND = os.path.join(CACHE_PATH, "eval-queue-bk")
23
+ EVAL_RESULTS_PATH_BACKEND = os.path.join(CACHE_PATH, "eval-results-bk")
24
+
25
+ API = HfApi(token=TOKEN)
src/leaderboard/read_evals.py ADDED
@@ -0,0 +1,196 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import glob
2
+ import json
3
+ import math
4
+ import os
5
+ from dataclasses import dataclass
6
+
7
+ import dateutil
8
+ import numpy as np
9
+
10
+ from src.display.formatting import make_clickable_model
11
+ from src.display.utils import AutoEvalColumn, ModelType, Tasks, Precision, WeightType
12
+ from src.submission.check_validity import is_model_on_hub
13
+
14
+
15
+ @dataclass
16
+ class EvalResult:
17
+ """Represents one full evaluation. Built from a combination of the result and request file for a given run.
18
+ """
19
+ eval_name: str # org_model_precision (uid)
20
+ full_model: str # org/model (path on hub)
21
+ org: str
22
+ model: str
23
+ revision: str # commit hash, "" if main
24
+ results: dict
25
+ precision: Precision = Precision.Unknown
26
+ model_type: ModelType = ModelType.Unknown # Pretrained, fine tuned, ...
27
+ weight_type: WeightType = WeightType.Original # Original or Adapter
28
+ architecture: str = "Unknown"
29
+ license: str = "?"
30
+ likes: int = 0
31
+ num_params: int = 0
32
+ date: str = "" # submission date of request file
33
+ still_on_hub: bool = False
34
+
35
+ @classmethod
36
+ def init_from_json_file(self, json_filepath):
37
+ """Inits the result from the specific model result file"""
38
+ with open(json_filepath) as fp:
39
+ data = json.load(fp)
40
+
41
+ config = data.get("config")
42
+
43
+ # Precision
44
+ precision = Precision.from_str(config.get("model_dtype"))
45
+
46
+ # Get model and org
47
+ org_and_model = config.get("model_name", config.get("model_args", None))
48
+ org_and_model = org_and_model.split("/", 1)
49
+
50
+ if len(org_and_model) == 1:
51
+ org = None
52
+ model = org_and_model[0]
53
+ result_key = f"{model}_{precision.value.name}"
54
+ else:
55
+ org = org_and_model[0]
56
+ model = org_and_model[1]
57
+ result_key = f"{org}_{model}_{precision.value.name}"
58
+ full_model = "/".join(org_and_model)
59
+
60
+ still_on_hub, _, model_config = is_model_on_hub(
61
+ full_model, config.get("model_sha", "main"), trust_remote_code=True, test_tokenizer=False
62
+ )
63
+ architecture = "?"
64
+ if model_config is not None:
65
+ architectures = getattr(model_config, "architectures", None)
66
+ if architectures:
67
+ architecture = ";".join(architectures)
68
+
69
+ # Extract results available in this file (some results are split in several files)
70
+ results = {}
71
+ for task in Tasks:
72
+ task = task.value
73
+
74
+ # We average all scores of a given metric (not all metrics are present in all files)
75
+ accs = np.array([v.get(task.metric, None) for k, v in data["results"].items() if task.benchmark == k])
76
+ if accs.size == 0 or any([acc is None for acc in accs]):
77
+ continue
78
+
79
+ mean_acc = np.mean(accs) * 100.0
80
+ results[task.benchmark] = mean_acc
81
+
82
+ return self(
83
+ eval_name=result_key,
84
+ full_model=full_model,
85
+ org=org,
86
+ model=model,
87
+ results=results,
88
+ precision=precision,
89
+ revision= config.get("model_sha", ""),
90
+ still_on_hub=still_on_hub,
91
+ architecture=architecture
92
+ )
93
+
94
+ def update_with_request_file(self, requests_path):
95
+ """Finds the relevant request file for the current model and updates info with it"""
96
+ request_file = get_request_file_for_model(requests_path, self.full_model, self.precision.value.name)
97
+
98
+ try:
99
+ with open(request_file, "r") as f:
100
+ request = json.load(f)
101
+ self.model_type = ModelType.from_str(request.get("model_type", ""))
102
+ self.weight_type = WeightType[request.get("weight_type", "Original")]
103
+ self.license = request.get("license", "?")
104
+ self.likes = request.get("likes", 0)
105
+ self.num_params = request.get("params", 0)
106
+ self.date = request.get("submitted_time", "")
107
+ except Exception:
108
+ print(f"Could not find request file for {self.org}/{self.model} with precision {self.precision.value.name}")
109
+
110
+ def to_dict(self):
111
+ """Converts the Eval Result to a dict compatible with our dataframe display"""
112
+ average = sum([v for v in self.results.values() if v is not None]) / len(Tasks)
113
+ data_dict = {
114
+ "eval_name": self.eval_name, # not a column, just a save name,
115
+ AutoEvalColumn.precision.name: self.precision.value.name,
116
+ AutoEvalColumn.model_type.name: self.model_type.value.name,
117
+ AutoEvalColumn.model_type_symbol.name: self.model_type.value.symbol,
118
+ AutoEvalColumn.weight_type.name: self.weight_type.value.name,
119
+ AutoEvalColumn.architecture.name: self.architecture,
120
+ AutoEvalColumn.model.name: make_clickable_model(self.full_model),
121
+ AutoEvalColumn.revision.name: self.revision,
122
+ AutoEvalColumn.average.name: average,
123
+ AutoEvalColumn.license.name: self.license,
124
+ AutoEvalColumn.likes.name: self.likes,
125
+ AutoEvalColumn.params.name: self.num_params,
126
+ AutoEvalColumn.still_on_hub.name: self.still_on_hub,
127
+ }
128
+
129
+ for task in Tasks:
130
+ data_dict[task.value.col_name] = self.results[task.value.benchmark]
131
+
132
+ return data_dict
133
+
134
+
135
+ def get_request_file_for_model(requests_path, model_name, precision):
136
+ """Selects the correct request file for a given model. Only keeps runs tagged as FINISHED"""
137
+ request_files = os.path.join(
138
+ requests_path,
139
+ f"{model_name}_eval_request_*.json",
140
+ )
141
+ request_files = glob.glob(request_files)
142
+
143
+ # Select correct request file (precision)
144
+ request_file = ""
145
+ request_files = sorted(request_files, reverse=True)
146
+ for tmp_request_file in request_files:
147
+ with open(tmp_request_file, "r") as f:
148
+ req_content = json.load(f)
149
+ if (
150
+ req_content["status"] in ["FINISHED"]
151
+ and req_content["precision"] == precision.split(".")[-1]
152
+ ):
153
+ request_file = tmp_request_file
154
+ return request_file
155
+
156
+
157
+ def get_raw_eval_results(results_path: str, requests_path: str) -> list[EvalResult]:
158
+ """From the path of the results folder root, extract all needed info for results"""
159
+ model_result_filepaths = []
160
+
161
+ for root, _, files in os.walk(results_path):
162
+ # We should only have json files in model results
163
+ if len(files) == 0 or any([not f.endswith(".json") for f in files]):
164
+ continue
165
+
166
+ # Sort the files by date
167
+ try:
168
+ files.sort(key=lambda x: x.removesuffix(".json").removeprefix("results_")[:-7])
169
+ except dateutil.parser._parser.ParserError:
170
+ files = [files[-1]]
171
+
172
+ for file in files:
173
+ model_result_filepaths.append(os.path.join(root, file))
174
+
175
+ eval_results = {}
176
+ for model_result_filepath in model_result_filepaths:
177
+ # Creation of result
178
+ eval_result = EvalResult.init_from_json_file(model_result_filepath)
179
+ eval_result.update_with_request_file(requests_path)
180
+
181
+ # Store results of same eval together
182
+ eval_name = eval_result.eval_name
183
+ if eval_name in eval_results.keys():
184
+ eval_results[eval_name].results.update({k: v for k, v in eval_result.results.items() if v is not None})
185
+ else:
186
+ eval_results[eval_name] = eval_result
187
+
188
+ results = []
189
+ for v in eval_results.values():
190
+ try:
191
+ v.to_dict() # we test if the dict version is complete
192
+ results.append(v)
193
+ except KeyError: # not all eval values present
194
+ continue
195
+
196
+ return results
src/populate.py ADDED
@@ -0,0 +1,58 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import json
2
+ import os
3
+
4
+ import pandas as pd
5
+
6
+ from src.display.formatting import has_no_nan_values, make_clickable_model
7
+ from src.display.utils import AutoEvalColumn, EvalQueueColumn
8
+ from src.leaderboard.read_evals import get_raw_eval_results
9
+
10
+
11
+ def get_leaderboard_df(results_path: str, requests_path: str, cols: list, benchmark_cols: list) -> pd.DataFrame:
12
+ """Creates a dataframe from all the individual experiment results"""
13
+ raw_data = get_raw_eval_results(results_path, requests_path)
14
+ all_data_json = [v.to_dict() for v in raw_data]
15
+
16
+ df = pd.DataFrame.from_records(all_data_json)
17
+ df = df.sort_values(by=[AutoEvalColumn.average.name], ascending=False)
18
+ df = df[cols].round(decimals=2)
19
+
20
+ # filter out if any of the benchmarks have not been produced
21
+ df = df[has_no_nan_values(df, benchmark_cols)]
22
+ return raw_data, df
23
+
24
+
25
+ def get_evaluation_queue_df(save_path: str, cols: list) -> list[pd.DataFrame]:
26
+ """Creates the different dataframes for the evaluation queues requestes"""
27
+ entries = [entry for entry in os.listdir(save_path) if not entry.startswith(".")]
28
+ all_evals = []
29
+
30
+ for entry in entries:
31
+ if ".json" in entry:
32
+ file_path = os.path.join(save_path, entry)
33
+ with open(file_path) as fp:
34
+ data = json.load(fp)
35
+
36
+ data[EvalQueueColumn.model.name] = make_clickable_model(data["model"])
37
+ data[EvalQueueColumn.revision.name] = data.get("revision", "main")
38
+
39
+ all_evals.append(data)
40
+ elif ".md" not in entry:
41
+ # this is a folder
42
+ sub_entries = [e for e in os.listdir(f"{save_path}/{entry}") if not e.startswith(".")]
43
+ for sub_entry in sub_entries:
44
+ file_path = os.path.join(save_path, entry, sub_entry)
45
+ with open(file_path) as fp:
46
+ data = json.load(fp)
47
+
48
+ data[EvalQueueColumn.model.name] = make_clickable_model(data["model"])
49
+ data[EvalQueueColumn.revision.name] = data.get("revision", "main")
50
+ all_evals.append(data)
51
+
52
+ pending_list = [e for e in all_evals if e["status"] in ["PENDING", "RERUN"]]
53
+ running_list = [e for e in all_evals if e["status"] == "RUNNING"]
54
+ finished_list = [e for e in all_evals if e["status"].startswith("FINISHED") or e["status"] == "PENDING_NEW_EVAL"]
55
+ df_pending = pd.DataFrame.from_records(pending_list, columns=cols)
56
+ df_running = pd.DataFrame.from_records(running_list, columns=cols)
57
+ df_finished = pd.DataFrame.from_records(finished_list, columns=cols)
58
+ return df_finished[cols], df_running[cols], df_pending[cols]
src/submission/check_validity.py ADDED
@@ -0,0 +1,99 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import json
2
+ import os
3
+ import re
4
+ from collections import defaultdict
5
+ from datetime import datetime, timedelta, timezone
6
+
7
+ import huggingface_hub
8
+ from huggingface_hub import ModelCard
9
+ from huggingface_hub.hf_api import ModelInfo
10
+ from transformers import AutoConfig
11
+ from transformers.models.auto.tokenization_auto import AutoTokenizer
12
+
13
+ def check_model_card(repo_id: str) -> tuple[bool, str]:
14
+ """Checks if the model card and license exist and have been filled"""
15
+ try:
16
+ card = ModelCard.load(repo_id)
17
+ except huggingface_hub.utils.EntryNotFoundError:
18
+ return False, "Please add a model card to your model to explain how you trained/fine-tuned it."
19
+
20
+ # Enforce license metadata
21
+ if card.data.license is None:
22
+ if not ("license_name" in card.data and "license_link" in card.data):
23
+ return False, (
24
+ "License not found. Please add a license to your model card using the `license` metadata or a"
25
+ " `license_name`/`license_link` pair."
26
+ )
27
+
28
+ # Enforce card content
29
+ if len(card.text) < 200:
30
+ return False, "Please add a description to your model card, it is too short."
31
+
32
+ return True, ""
33
+
34
+ def is_model_on_hub(model_name: str, revision: str, token: str = None, trust_remote_code=False, test_tokenizer=False) -> tuple[bool, str]:
35
+ """Checks if the model model_name is on the hub, and whether it (and its tokenizer) can be loaded with AutoClasses."""
36
+ try:
37
+ config = AutoConfig.from_pretrained(model_name, revision=revision, trust_remote_code=trust_remote_code, token=token)
38
+ if test_tokenizer:
39
+ try:
40
+ tk = AutoTokenizer.from_pretrained(model_name, revision=revision, trust_remote_code=trust_remote_code, token=token)
41
+ except ValueError as e:
42
+ return (
43
+ False,
44
+ f"uses a tokenizer which is not in a transformers release: {e}",
45
+ None
46
+ )
47
+ except Exception as e:
48
+ return (False, "'s tokenizer cannot be loaded. Is your tokenizer class in a stable transformers release, and correctly configured?", None)
49
+ return True, None, config
50
+
51
+ except ValueError:
52
+ return (
53
+ False,
54
+ "needs to be launched with `trust_remote_code=True`. For safety reason, we do not allow these models to be automatically submitted to the leaderboard.",
55
+ None
56
+ )
57
+
58
+ except Exception as e:
59
+ return False, "was not found on hub!", None
60
+
61
+
62
+ def get_model_size(model_info: ModelInfo, precision: str):
63
+ """Gets the model size from the configuration, or the model name if the configuration does not contain the information."""
64
+ try:
65
+ model_size = round(model_info.safetensors["total"] / 1e9, 3)
66
+ except (AttributeError, TypeError):
67
+ return 0 # Unknown model sizes are indicated as 0, see NUMERIC_INTERVALS in app.py
68
+
69
+ size_factor = 8 if (precision == "GPTQ" or "gptq" in model_info.modelId.lower()) else 1
70
+ model_size = size_factor * model_size
71
+ return model_size
72
+
73
+ def get_model_arch(model_info: ModelInfo):
74
+ """Gets the model architecture from the configuration"""
75
+ return model_info.config.get("architectures", "Unknown")
76
+
77
+ def already_submitted_models(requested_models_dir: str) -> set[str]:
78
+ """Gather a list of already submitted models to avoid duplicates"""
79
+ depth = 1
80
+ file_names = []
81
+ users_to_submission_dates = defaultdict(list)
82
+
83
+ for root, _, files in os.walk(requested_models_dir):
84
+ current_depth = root.count(os.sep) - requested_models_dir.count(os.sep)
85
+ if current_depth == depth:
86
+ for file in files:
87
+ if not file.endswith(".json"):
88
+ continue
89
+ with open(os.path.join(root, file), "r") as f:
90
+ info = json.load(f)
91
+ file_names.append(f"{info['model']}_{info['revision']}_{info['precision']}")
92
+
93
+ # Select organisation
94
+ if info["model"].count("/") == 0 or "submitted_time" not in info:
95
+ continue
96
+ organisation, _ = info["model"].split("/")
97
+ users_to_submission_dates[organisation].append(info["submitted_time"])
98
+
99
+ return set(file_names), users_to_submission_dates
src/submission/submit.py ADDED
@@ -0,0 +1,119 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import json
2
+ import os
3
+ from datetime import datetime, timezone
4
+
5
+ from src.display.formatting import styled_error, styled_message, styled_warning
6
+ from src.envs import API, EVAL_REQUESTS_PATH, TOKEN, QUEUE_REPO
7
+ from src.submission.check_validity import (
8
+ already_submitted_models,
9
+ check_model_card,
10
+ get_model_size,
11
+ is_model_on_hub,
12
+ )
13
+
14
+ REQUESTED_MODELS = None
15
+ USERS_TO_SUBMISSION_DATES = None
16
+
17
+ def add_new_eval(
18
+ model: str,
19
+ base_model: str,
20
+ revision: str,
21
+ precision: str,
22
+ weight_type: str,
23
+ model_type: str,
24
+ ):
25
+ global REQUESTED_MODELS
26
+ global USERS_TO_SUBMISSION_DATES
27
+ if not REQUESTED_MODELS:
28
+ REQUESTED_MODELS, USERS_TO_SUBMISSION_DATES = already_submitted_models(EVAL_REQUESTS_PATH)
29
+
30
+ user_name = ""
31
+ model_path = model
32
+ if "/" in model:
33
+ user_name = model.split("/")[0]
34
+ model_path = model.split("/")[1]
35
+
36
+ precision = precision.split(" ")[0]
37
+ current_time = datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
38
+
39
+ if model_type is None or model_type == "":
40
+ return styled_error("Please select a model type.")
41
+
42
+ # Does the model actually exist?
43
+ if revision == "":
44
+ revision = "main"
45
+
46
+ # Is the model on the hub?
47
+ if weight_type in ["Delta", "Adapter"]:
48
+ base_model_on_hub, error, _ = is_model_on_hub(model_name=base_model, revision=revision, token=TOKEN, test_tokenizer=True)
49
+ if not base_model_on_hub:
50
+ return styled_error(f'Base model "{base_model}" {error}')
51
+
52
+ if not weight_type == "Adapter":
53
+ model_on_hub, error, _ = is_model_on_hub(model_name=model, revision=revision, token=TOKEN, test_tokenizer=True)
54
+ if not model_on_hub:
55
+ return styled_error(f'Model "{model}" {error}')
56
+
57
+ # Is the model info correctly filled?
58
+ try:
59
+ model_info = API.model_info(repo_id=model, revision=revision)
60
+ except Exception:
61
+ return styled_error("Could not get your model information. Please fill it up properly.")
62
+
63
+ model_size = get_model_size(model_info=model_info, precision=precision)
64
+
65
+ # Were the model card and license filled?
66
+ try:
67
+ license = model_info.cardData["license"]
68
+ except Exception:
69
+ return styled_error("Please select a license for your model")
70
+
71
+ modelcard_OK, error_msg = check_model_card(model)
72
+ if not modelcard_OK:
73
+ return styled_error(error_msg)
74
+
75
+ # Seems good, creating the eval
76
+ print("Adding new eval")
77
+
78
+ eval_entry = {
79
+ "model": model,
80
+ "base_model": base_model,
81
+ "revision": revision,
82
+ "precision": precision,
83
+ "weight_type": weight_type,
84
+ "status": "PENDING",
85
+ "submitted_time": current_time,
86
+ "model_type": model_type,
87
+ "likes": model_info.likes,
88
+ "params": model_size,
89
+ "license": license,
90
+ "private": False,
91
+ }
92
+
93
+ # Check for duplicate submission
94
+ if f"{model}_{revision}_{precision}" in REQUESTED_MODELS:
95
+ return styled_warning("This model has been already submitted.")
96
+
97
+ print("Creating eval file")
98
+ OUT_DIR = f"{EVAL_REQUESTS_PATH}/{user_name}"
99
+ os.makedirs(OUT_DIR, exist_ok=True)
100
+ out_path = f"{OUT_DIR}/{model_path}_eval_request_False_{precision}_{weight_type}.json"
101
+
102
+ with open(out_path, "w") as f:
103
+ f.write(json.dumps(eval_entry))
104
+
105
+ print("Uploading eval file")
106
+ API.upload_file(
107
+ path_or_fileobj=out_path,
108
+ path_in_repo=out_path.split("eval-queue/")[1],
109
+ repo_id=QUEUE_REPO,
110
+ repo_type="dataset",
111
+ commit_message=f"Add {model} to eval queue",
112
+ )
113
+
114
+ # Remove the local file
115
+ os.remove(out_path)
116
+
117
+ return styled_message(
118
+ "Your request has been submitted to the evaluation queue!\nPlease wait for up to an hour for the model to show in the PENDING list."
119
+ )