TRUST_REMOTE_CODE env variable

src/display/about.py

@@ -1,6 +1,6 @@
 from src.display.utils import ModelType
 from src.display.utils import Tasks
-from src.envs import REPO_ID, QUEUE_REPO, RESULTS_REPO, PATH_TO_COLLECTION, LEADERBOARD_NAME
+from src.envs import REPO_ID, QUEUE_REPO, RESULTS_REPO, PATH_TO_COLLECTION, LEADERBOARD_NAME, TRUST_REMOTE_CODE
 
 LM_EVAL_URL = "https://github.com/eduagarcia/lm-evaluation-harness-pt"
 
@@ -81,6 +81,10 @@ To get more information about quantization, see:
 - [Collection of best models](https://huggingface.co/collections/{PATH_TO_COLLECTION})
 """
 
+REMOTE_CODE_EXAPLANATION = f"- *Yes.*"
+if not TRUST_REMOTE_CODE:
+      REMOTE_CODE_EXAPLANATION = f"- *We only support models that have been integrated in a stable version of the `transformers` library for automatic submission, as we don't want to run possibly unsage code on our cluster.*"
+
 FAQ_TEXT = f"""
 ---------------------------
 # FAQ
@@ -88,7 +92,7 @@ Below are some common questions - if this FAQ does not answer you, feel free to
 
 ## 1) Submitting a model
 My model requires `trust_remote_code=True`, can I submit it?
-- *We only support models that have been integrated in a stable version of the `transformers` library for automatic submission, as we don't want to run possibly unsage code on our cluster.*
+{REMOTE_CODE_EXAPLANATION}
 
 What about models of type X? 
 - *We only support models that have been integrated in a stable version of the `transformers` library for automatic submission.*
@@ -132,6 +136,7 @@ I have an issue about accessing the leaderboard through the Gradio API
 - *Since this is not the recommended way to access the leaderboard, we won't provide support for this, but you can look at tools provided by the community for inspiration!*
 """
 
+REMOTE_CODE_NOTE = "Note: if your model needs `use_remote_code=True`, we do not support this option yet but we are working on adding it, stay posted!"
 
 EVALUATION_QUEUE_TEXT = f"""
 # Evaluation Queue for the 🚀 {LEADERBOARD_NAME}
@@ -149,8 +154,8 @@ tokenizer = AutoTokenizer.from_pretrained("your model name", revision=revision)
 ```
 If this step fails, follow the error messages to debug your model before submitting it. It's likely your model has been improperly uploaded.
 
-Note: make sure your model is public!  
-Note: if your model needs `use_remote_code=True`, we do not support this option yet but we are working on adding it, stay posted!
+Note: make sure your model is public! 
+{REMOTE_CODE_NOTE if not TRUST_REMOTE_CODE else ""}
 
 ### 2) Convert your model weights to [safetensors](https://huggingface.co/docs/safetensors/index)
 It's a new format for storing weights which is safer and faster to load and use. It will also allow us to add the number of parameters of your model to the `Extended Viewer`!

src/envs.py

@@ -36,4 +36,6 @@ RATE_LIMIT_PERIOD = int(os.getenv("RATE_LIMIT_PERIOD", 7))
 RATE_LIMIT_QUOTA = int(os.getenv("RATE_LIMIT_QUOTA", 5))
 HAS_HIGHER_RATE_LIMIT = os.environ.get("HAS_HIGHER_RATE_LIMIT", "TheBloke").split(',')
 
+TRUST_REMOTE_CODE = bool(os.getenv("TRUST_REMOTE_CODE", False))
+
 API = HfApi(token=H4_TOKEN)

src/submission/check_validity.py

@@ -9,7 +9,7 @@ from huggingface_hub import ModelCard
 from huggingface_hub.hf_api import ModelInfo, get_safetensors_metadata
 from transformers import AutoConfig, AutoTokenizer
 
-from src.envs import HAS_HIGHER_RATE_LIMIT
+from src.envs import HAS_HIGHER_RATE_LIMIT, TRUST_REMOTE_CODE
 
 
 # ht to @Wauplin, thank you for the snippet!
@@ -36,7 +36,7 @@ def check_model_card(repo_id: str) -> tuple[bool, str]:
     return True, ""
 
 
-def is_model_on_hub(model_name: str, revision: str, token: str = None, trust_remote_code=False, test_tokenizer=False) -> tuple[bool, str, AutoConfig]:
+def is_model_on_hub(model_name: str, revision: str, token: str = None, trust_remote_code=TRUST_REMOTE_CODE, test_tokenizer=False) -> tuple[bool, str, AutoConfig]:
     try:
         config = AutoConfig.from_pretrained(model_name, revision=revision, trust_remote_code=trust_remote_code, token=token) #, force_download=True)
         if test_tokenizer: