n-shot filter

src/about.py

@@ -37,7 +37,7 @@ NUM_FEWSHOT = 0 # Change with your few shot
 
 
 # Your leaderboard name
-TITLE = """<h1 align="center" id="space-title">Open PL LLM Leaderboard (0-shot)</h1>"""
+TITLE = """<h1 align="center" id="space-title">Open PL LLM Leaderboard (0-shot and 5-shot)</h1>"""
 
 # What does your leaderboard evaluate?
 INTRODUCTION_TEXT = """

src/display/utils.py

@@ -26,7 +26,7 @@ auto_eval_column_dict = []
 # Init
 auto_eval_column_dict.append(["model_type_symbol", ColumnContent, ColumnContent("T", "str", True, never_hidden=True)])
 auto_eval_column_dict.append(["model", ColumnContent, ColumnContent("Model", "markdown", True, never_hidden=True)])
-auto_eval_column_dict.append(["n_shot", ColumnContent, ColumnContent("n_shot", "number", True)])
+auto_eval_column_dict.append(["n_shot", ColumnContent, ColumnContent("n_shot", "str", True)])
 #Scores
 auto_eval_column_dict.append(["average", ColumnContent, ColumnContent("Average ⬆️", "number", True)])
 for task in Tasks:
@@ -96,6 +96,14 @@ class NShotType(Enum):
     n0 = ModelDetails("0")
     n5 = ModelDetails("5")
 
+    @staticmethod
+    def from_str(n):
+        if n in ["0", 0]:
+            return NShotType.n0
+        if n in ["5", 5]:
+            return NShotType.n5
+        return NShotType.Unknown
+
 class Precision(Enum):
     float16 = ModelDetails("float16")
     bfloat16 = ModelDetails("bfloat16")

src/leaderboard/read_evals.py

@@ -9,7 +9,7 @@ import dateutil
 import numpy as np
 
 from src.display.formatting import make_clickable_model
-from src.display.utils import AutoEvalColumn, ModelType, Tasks, Precision, WeightType
+from src.display.utils import AutoEvalColumn, ModelType, Tasks, Precision, WeightType, NShotType
 from src.submission.check_validity import is_model_on_hub
 
 NUM_FEWSHOT = 0
@@ -31,7 +31,7 @@ class EvalResult:
     num_params: int = 0
     date: str = "" # submission date of request file
     still_on_hub: bool = False
-    n_shot: int = 0
+    n_shot: NShotType = NShotType.n0
 
     @classmethod
     def init_from_json_file(self, json_filepath, n_shot_num):
@@ -92,7 +92,7 @@ class EvalResult:
             revision= config.get("model_sha", ""),
             still_on_hub=still_on_hub,
             architecture=architecture,
-            n_shot=n_shot_num
+            n_shot=NShotType.from_str(n_shot_num)
         )
 
     def update_with_request_file(self, requests_path):
@@ -203,7 +203,7 @@ class EvalResult:
             print(f"Could not find still on hub")
 
         try:
-            data_dict[AutoEvalColumn.n_shot.name] = self.n_shot
+            data_dict[AutoEvalColumn.n_shot.name] = self.n_shot.value.name
         except KeyError:
             print(f"Could not find still on hub")