add env variables: REQUIRE_MODEL_CARD and REQUIRE_MODEL_LICENSE

src/envs.py

@@ -66,5 +66,7 @@ ORIGINAL_HF_LEADERBOARD_RESULTS_REPO = get_config("ORIGINAL_HF_LEADERBOARD_RESUL
 ORIGINAL_HF_LEADERBOARD_EVAL_RESULTS_PATH = os.path.join(CACHE_PATH, 'original_results')
 
 SHOW_INCOMPLETE_EVALS = str2bool(get_config("SHOW_INCOMPLETE_EVALS", False))
+REQUIRE_MODEL_CARD = str2bool(get_config("REQUIRE_MODEL_CARD", True))
+REQUIRE_MODEL_LICENSE = str2bool(get_config("REQUIRE_MODEL_LICENSE", True))
 
 API = HfApi(token=H4_TOKEN)

src/scripts/update_all_request_files.py

@@ -46,9 +46,9 @@ def update_models(file_path, models, original_leaderboard_files=None):
 
             # Is the model still on the hub?
             model_name = model_id
-
             if model_cfg.card_data is not None and hasattr(model_cfg.card_data, "base_model") and model_cfg.card_data.base_model is not None:
-                model_name = model_cfg.card_data.base_model # for adapters, we look at the parent model
+                if isinstance(model_cfg.card_data.base_model, str):
+                    model_name = model_cfg.card_data.base_model # for adapters, we look at the parent model
             still_on_hub, _, _ = is_model_on_hub(
                 model_name=model_name, revision=data.get("revision"), trust_remote_code=True, test_tokenizer=False, token=H4_TOKEN
             )
@@ -57,7 +57,7 @@ def update_models(file_path, models, original_leaderboard_files=None):
             tags = []
 
             if still_on_hub:
-                status, _, model_card = check_model_card(model_id)
+                status, _, _, model_card = check_model_card(model_id)
                 tags = get_model_tags(model_card, model_id)
 
             

src/submission/check_validity.py

@@ -20,23 +20,27 @@ def check_model_card(repo_id: str) -> tuple[bool, str]:
     try:
         card = ModelCard.load(repo_id)
     except huggingface_hub.utils.EntryNotFoundError:
-        return False, "Please add a model card to your model to explain how you trained/fine-tuned it.", None
+        return False, False, "Please add a model card to your model to explain how you trained/fine-tuned it.", None
     except Exception as e:
-        return False, f"Error while loading the model card. Exception: {str(e)}", None
+        return False, False, f"Error while loading the model card. Exception: {str(e)}", None
     
-    # Enforce license metadata
+    license = True
     if card.data.license is None:
         if not ("license_name" in card.data and "license_link" in card.data):
-            return False, (
-                "License not found. Please add a license to your model card using the `license` metadata or a"
-                " `license_name`/`license_link` pair."
-            ), None
+            license = False
 
     # Enforce card content
     if len(card.text) < 200:
-        return False, "Please add a description to your model card, it is too short.", None
+        return False, license, "Please add a description to your model card bigger than 200 characters, it is too short.", None
 
-    return True, "", card
+    # Enforce license metadata
+    if not license:
+        return True, False, (
+            "License not found. Please add a license to your model card using the `license` metadata or a"
+            " `license_name`/`license_link` pair."
+        ), None
+ 
+    return True, True, "", card
 
 
 def is_model_on_hub(model_name: str, revision: str, token: str = None, trust_remote_code=TRUST_REMOTE_CODE, test_tokenizer=False) -> tuple[bool, str, AutoConfig]:
@@ -149,19 +153,19 @@ def get_model_tags(model_card, model: str):
 
     # Storing the model tags
     tags = []
-    moe_keywords = ["moe", "mixture of experts", "mixtral"]
+    merge_keywords = ["merged model", "merge model"]
+    moe_keywords = ["moe", "mixtral"]
+
     if model_card is not None:
         if model_card.data.tags:
             is_merge_from_metadata = "merge" in model_card.data.tags
             is_moe_from_metadata = "moe" in model_card.data.tags
-        merge_keywords = ["merged model", "merge model"]
         # If the model is a merge but not saying it in the metadata, we flag it
         is_merge_from_model_card = any(keyword in model_card.text.lower() for keyword in merge_keywords)
         if is_merge_from_model_card or is_merge_from_metadata:
             tags.append("merge")
             if not is_merge_from_metadata:
                 tags.append("flagged:undisclosed_merge")
-        moe_keywords = ["moe", "mixtral"]
         is_moe_from_model_card = any(keyword in model_card.text.lower() for keyword in moe_keywords)
     is_moe_from_name = "moe" in model.lower().replace("/", "-").replace("_", "-").split("-")
     if is_moe_from_model_card or is_moe_from_name or is_moe_from_metadata:

src/submission/submit.py

@@ -5,7 +5,7 @@ from datetime import datetime, timezone
 from huggingface_hub import ModelCard, snapshot_download
 
 from src.display.formatting import styled_error, styled_message, styled_warning
-from src.envs import API, EVAL_REQUESTS_PATH, DYNAMIC_INFO_PATH, DYNAMIC_INFO_FILE_PATH, DYNAMIC_INFO_REPO, H4_TOKEN, QUEUE_REPO, RATE_LIMIT_PERIOD, RATE_LIMIT_QUOTA
+from src.envs import API, EVAL_REQUESTS_PATH, DYNAMIC_INFO_PATH, DYNAMIC_INFO_FILE_PATH, DYNAMIC_INFO_REPO, H4_TOKEN, QUEUE_REPO, RATE_LIMIT_PERIOD, RATE_LIMIT_QUOTA, REQUIRE_MODEL_LICENSE, REQUIRE_MODEL_CARD
 from src.leaderboard.filter_models import DO_NOT_SUBMIT_MODELS
 from src.submission.check_validity import (
     already_submitted_models,
@@ -99,9 +99,11 @@ def add_new_eval(
         license = None
         #return styled_error("Please select a license for your model")
 
-    modelcard_OK, error_msg, model_card = check_model_card(model)
-    #if not modelcard_OK:
-    #    return styled_error(error_msg)
+    modelcard_OK, license_OK, error_msg, model_card = check_model_card(model)
+    if not modelcard_OK and REQUIRE_MODEL_CARD:
+        return styled_error(error_msg)
+    if not license_OK and REQUIRE_MODEL_LICENSE:
+        return styled_error(error_msg)
     
     tags = get_model_tags(model_card, model)
 

tasks_config/pt_config.yaml

@@ -11,6 +11,8 @@ config:
   GET_ORIGINAL_HF_LEADERBOARD_EVAL_RESULTS: true
   TRUST_REMOTE_CODE: true
   SHOW_INCOMPLETE_EVALS: false
+  REQUIRE_MODEL_CARD: true
+  REQUIRE_MODEL_LICENSE: false
 readme:
   general_description: |
     📐 The 🚀 Open PT LLM Leaderboard aims to provide a benchmark for the evaluation of