alozowski HF staff commited on
Commit
efed7dc
1 Parent(s): 3b94cb4

submit-system-update (#838)

Browse files

- add login at submit and user info (0b31d4e1ffcd10bcdf9603d588d3b1d28c77fdc2)
- use gradio-leaderboard 0.0.11 (c8c479205c76ee4c961253c5909ffaa1eb6b03ed)
- update and revised submission info (8013f916ae1e9df83a51c72a1b736f51f10c8bc5)

Files changed (5) hide show
  1. app.py +1 -0
  2. pyproject.toml +1 -1
  3. requirements.txt +1 -1
  4. src/display/about.py +26 -14
  5. src/submission/submit.py +22 -7
app.py CHANGED
@@ -235,6 +235,7 @@ with main_block:
235
 
236
  with gr.Row():
237
  gr.Markdown("# ✉️✨ Submit your model here!", elem_classes="markdown-text")
 
238
 
239
  with gr.Row():
240
  with gr.Column():
 
235
 
236
  with gr.Row():
237
  gr.Markdown("# ✉️✨ Submit your model here!", elem_classes="markdown-text")
238
+ login_button = gr.LoginButton(elem_id="oauth-button")
239
 
240
  with gr.Row():
241
  with gr.Column():
pyproject.toml CHANGED
@@ -45,7 +45,7 @@ tokenizers = ">=0.15.0"
45
  gradio-space-ci = {git = "https://huggingface.co/spaces/Wauplin/gradio-space-ci", rev = "0.2.3"}
46
  isort = "^5.13.2"
47
  ruff = "^0.3.5"
48
- gradio-leaderboard = "0.0.11rc0"
49
  gradio = {extras = ["oauth"], version = "^4.36.1"}
50
  requests = "^2.31.0"
51
  requests-oauthlib = "^1.3.1"
 
45
  gradio-space-ci = {git = "https://huggingface.co/spaces/Wauplin/gradio-space-ci", rev = "0.2.3"}
46
  isort = "^5.13.2"
47
  ruff = "^0.3.5"
48
+ gradio-leaderboard = "0.0.11"
49
  gradio = {extras = ["oauth"], version = "^4.36.1"}
50
  requests = "^2.31.0"
51
  requests-oauthlib = "^1.3.1"
requirements.txt CHANGED
@@ -17,7 +17,7 @@ isort
17
  ruff
18
  gradio==4.31.0
19
  gradio[oauth]
20
- gradio_leaderboard==0.0.11rc0
21
  requests==2.31.0
22
  requests-oauthlib== 1.3.1
23
  schedule == 1.2.2
 
17
  ruff
18
  gradio==4.31.0
19
  gradio[oauth]
20
+ gradio_leaderboard==0.0.11
21
  requests==2.31.0
22
  requests-oauthlib== 1.3.1
23
  schedule == 1.2.2
src/display/about.py CHANGED
@@ -179,35 +179,47 @@ EVALUATION_QUEUE_TEXT = f"""
179
 
180
  Models added here will be automatically evaluated on the 🤗 cluster.
181
 
182
- ## Don't forget to read the FAQ and the About tabs for more information!
183
 
184
- ## First steps before submitting a model
 
 
 
 
 
 
 
 
 
 
185
 
186
- ### 1) Make sure you can load your model and tokenizer using AutoClasses:
187
  ```python
188
  from transformers import AutoConfig, AutoModel, AutoTokenizer
189
  config = AutoConfig.from_pretrained("your model name", revision=revision)
190
  model = AutoModel.from_pretrained("your model name", revision=revision)
191
  tokenizer = AutoTokenizer.from_pretrained("your model name", revision=revision)
192
  ```
193
- If this step fails, follow the error messages to debug your model before submitting it. It's likely your model has been improperly uploaded.
 
 
 
194
 
195
- Note: make sure your model is public!
196
- Note: if your model needs `use_remote_code=True`, we do not support this option yet but we are working on adding it, stay posted!
197
 
198
- ### 2) Convert your model weights to [safetensors](https://huggingface.co/docs/safetensors/index)
199
- It's a new format for storing weights which is safer and faster to load and use. It will also allow us to add the number of parameters of your model to the `Extended Viewer`!
200
-
201
- ### 3) Make sure your model has an open license!
202
  This is a leaderboard for Open LLMs, and we'd love for as many people as possible to know they can use your model 🤗
203
 
204
- ### 4) Fill up your model card
205
  When we add extra information about models to the leaderboard, it will be automatically taken from the model card
206
 
207
- ### 5) Select the correct precision
208
- Not all models are converted properly from `float16` to `bfloat16`, and selecting the wrong precision can sometimes cause evaluation error (as loading a `bf16` model in `fp16` can sometimes generate NaNs, depending on the weight range).
 
 
 
 
209
 
210
- <b>Note:</b> Please be advised that when submitting, git <b>branches</b> and <b>tags</b> will be strictly tied to the <b>specific commit</b> present at the time of submission. This ensures revision consistency.
211
  ## Model types
212
  {icons}
213
  """
 
179
 
180
  Models added here will be automatically evaluated on the 🤗 cluster.
181
 
182
+ > **Important:** Don't forget to read the [FAQ](https://huggingface.co/docs/leaderboards/open_llm_leaderboard/faq) and [documentation](https://huggingface.co/docs/leaderboards/open_llm_leaderboard/about) for more information! 📄
183
 
184
+ ## Submission Disclaimer
185
+ **By submitting a model, you acknowledge that:**
186
+ - We store information about who submitted each model in [Requests dataset](https://huggingface.co/datasets/open-llm-leaderboard/requests).
187
+ - This practice helps maintain the integrity of our leaderboard, prevent spam, and ensure responsible submissions.
188
+ - Your submission will be visible to the community and you may be contacted regarding your model.
189
+ - Please submit carefully and responsibly 💛
190
+
191
+ ## First Steps Before Submitting a Model
192
+
193
+ ### 1. Ensure Your Model Loads with AutoClasses
194
+ Verify that you can load your model and tokenizer using AutoClasses:
195
 
 
196
  ```python
197
  from transformers import AutoConfig, AutoModel, AutoTokenizer
198
  config = AutoConfig.from_pretrained("your model name", revision=revision)
199
  model = AutoModel.from_pretrained("your model name", revision=revision)
200
  tokenizer = AutoTokenizer.from_pretrained("your model name", revision=revision)
201
  ```
202
+ Note:
203
+ - If this step fails, debug your model before submitting.
204
+ - Ensure your model is public.
205
+ - We are working on adding support for models requiring `use_remote_code=True`.
206
 
207
+ ### 2. Convert Weights to Safetensors
208
+ [Safetensors](https://huggingface.co/docs/safetensors/index) is a new format for storing weights which is safer and faster to load and use. It will also allow us to add the number of parameters of your model to the `Extended Viewer`!
209
 
210
+ ### 3. Verify Your Model Open License
 
 
 
211
  This is a leaderboard for Open LLMs, and we'd love for as many people as possible to know they can use your model 🤗
212
 
213
+ ### 4. Complete Your Model Card
214
  When we add extra information about models to the leaderboard, it will be automatically taken from the model card
215
 
216
+ ### 5. Select Correct Precision
217
+ Choose the right precision to avoid evaluation errors:
218
+ - Not all models convert properly from float16 to bfloat16.
219
+ - Incorrect precision can cause issues (e.g., loading a bf16 model in fp16 may generate NaNs).
220
+
221
+ > **Important:** When submitting, git branches and tags will be strictly tied to the specific commit present at the time of submission to ensure revision consistency.
222
 
 
223
  ## Model types
224
  {icons}
225
  """
src/submission/submit.py CHANGED
@@ -1,5 +1,6 @@
1
  import json
2
  import os
 
3
  from datetime import datetime, timezone
4
 
5
  from dataclasses import dataclass
@@ -59,16 +60,25 @@ def add_new_eval(
59
  weight_type: str,
60
  model_type: str,
61
  use_chat_template: bool,
62
- ):
 
 
 
 
 
 
 
 
63
  global REQUESTED_MODELS
64
  global USERS_TO_SUBMISSION_DATES
65
  if not REQUESTED_MODELS:
66
  REQUESTED_MODELS, USERS_TO_SUBMISSION_DATES = already_submitted_models(EVAL_REQUESTS_PATH)
67
 
68
- user_name = ""
 
69
  model_path = model
70
  if "/" in model:
71
- user_name = model.split("/")[0]
72
  model_path = model.split("/")[1]
73
 
74
  precision = precision.split(" ")[0]
@@ -77,10 +87,14 @@ def add_new_eval(
77
  if model_type is None or model_type == "":
78
  return styled_error("Please select a model type.")
79
 
 
 
 
 
80
  # Is the user rate limited?
81
- if user_name != "":
82
  user_can_submit, error_msg = user_submission_permission(
83
- user_name, USERS_TO_SUBMISSION_DATES, RATE_LIMIT_PERIOD, RATE_LIMIT_QUOTA
84
  )
85
  if not user_can_submit:
86
  return styled_error(error_msg)
@@ -144,7 +158,6 @@ def add_new_eval(
144
 
145
  # Seems good, creating the eval
146
  print("Adding new eval")
147
-
148
  eval_entry = {
149
  "model": model,
150
  "base_model": base_model,
@@ -159,10 +172,11 @@ def add_new_eval(
159
  "job_id": -1,
160
  "job_start_time": None,
161
  "use_chat_template": use_chat_template,
 
162
  }
163
 
164
  print("Creating eval file")
165
- OUT_DIR = f"{EVAL_REQUESTS_PATH}/{user_name}"
166
  os.makedirs(OUT_DIR, exist_ok=True)
167
  out_path = f"{OUT_DIR}/{model_path}_eval_request_False_{precision}_{weight_type}.json"
168
 
@@ -170,6 +184,7 @@ def add_new_eval(
170
  f.write(json.dumps(eval_entry))
171
 
172
  print("Uploading eval file")
 
173
  API.upload_file(
174
  path_or_fileobj=out_path,
175
  path_in_repo=out_path.split("eval-queue/")[1],
 
1
  import json
2
  import os
3
+ import gradio as gr
4
  from datetime import datetime, timezone
5
 
6
  from dataclasses import dataclass
 
60
  weight_type: str,
61
  model_type: str,
62
  use_chat_template: bool,
63
+ profile: gr.OAuthProfile | None
64
+ ):
65
+ # Login require
66
+ if profile is None:
67
+ return styled_error("Hub Login Required")
68
+
69
+ # Name of the actual user who sent the request
70
+ username = profile.username
71
+
72
  global REQUESTED_MODELS
73
  global USERS_TO_SUBMISSION_DATES
74
  if not REQUESTED_MODELS:
75
  REQUESTED_MODELS, USERS_TO_SUBMISSION_DATES = already_submitted_models(EVAL_REQUESTS_PATH)
76
 
77
+
78
+ org_or_user = ""
79
  model_path = model
80
  if "/" in model:
81
+ org_or_user = model.split("/")[0]
82
  model_path = model.split("/")[1]
83
 
84
  precision = precision.split(" ")[0]
 
87
  if model_type is None or model_type == "":
88
  return styled_error("Please select a model type.")
89
 
90
+ # Is user submitting own model?
91
+ # Check that username in the org.
92
+ # if org_or_user != profile.username:
93
+
94
  # Is the user rate limited?
95
+ if org_or_user != "":
96
  user_can_submit, error_msg = user_submission_permission(
97
+ org_or_user, USERS_TO_SUBMISSION_DATES, RATE_LIMIT_PERIOD, RATE_LIMIT_QUOTA
98
  )
99
  if not user_can_submit:
100
  return styled_error(error_msg)
 
158
 
159
  # Seems good, creating the eval
160
  print("Adding new eval")
 
161
  eval_entry = {
162
  "model": model,
163
  "base_model": base_model,
 
172
  "job_id": -1,
173
  "job_start_time": None,
174
  "use_chat_template": use_chat_template,
175
+ "sender": username
176
  }
177
 
178
  print("Creating eval file")
179
+ OUT_DIR = f"{EVAL_REQUESTS_PATH}/{org_or_user}"
180
  os.makedirs(OUT_DIR, exist_ok=True)
181
  out_path = f"{OUT_DIR}/{model_path}_eval_request_False_{precision}_{weight_type}.json"
182
 
 
184
  f.write(json.dumps(eval_entry))
185
 
186
  print("Uploading eval file")
187
+ print(eval_entry)
188
  API.upload_file(
189
  path_or_fileobj=out_path,
190
  path_in_repo=out_path.split("eval-queue/")[1],