refactor columns

app.py

@@ -350,6 +350,6 @@ with demo:
             )
 
 scheduler = BackgroundScheduler()
-scheduler.add_job(restart_space, "interval", seconds=1800)
+scheduler.add_job(restart_space, "interval", seconds=3600)
 scheduler.start()
 demo.queue(default_concurrency_limit=40).launch()

src/about.py

@@ -7,6 +7,8 @@ class Task:
     benchmark: str
     metric: str
     col_name: str
+    higher_is_better: bool = True
+    scale_by_100: bool = True
 
 
 # Select your tasks here
@@ -16,11 +18,11 @@ class Tasks(Enum):
     task0 = Task("arc_challenge_ita", "acc_norm,none", "ARC-C")
     task1 = Task("ami_2020_aggressiveness", "f1,none", "AMI 2020 Agg")
     task2 = Task("ami_2020_misogyny", "f1,none", "AMI 2020 Miso")
-    task3 = Task("gente_rephrasing", "acc,none", "GeNTE Rephr")
+    task3 = Task("gente_rephrasing", "acc,none", "GeNTE Neutralizing")
     task4 = Task("belebele_ita", "acc_norm,none", "Belebele")
     task5 = Task("hatecheck_ita", "f1,none", "HateCheck")
-    task6 = Task("honest_ita", "acc,none", "HONEST")
-    task7 = Task("itacola", "mcc,none", "ItaCoLA")
+    task6 = Task("honest_ita", "acc,none", "HONEST", higher_is_better=False)
+    task7 = Task("itacola", "mcc,none", "ItaCoLA", scale_by_100=False)
     task8 = Task("news_sum", "bertscore,none", "News Sum")
     task9 = Task("squad_it", "squad_f1,get-answer", "SQuAD it")
     task10 = Task("truthfulqa_gen_ita", "rouge1_max,none", "TruthfulQA")
@@ -36,6 +38,9 @@ TITLE = """<h1 align="center" id="space-title">ItaEval leaderboard</h1>"""
 # What does your leaderboard evaluate?
 INTRODUCTION_TEXT = """
 This leaderboard evaluates language models on <b>ItaEval</b>, a new unified benchmark for Italian.
+
+Some information:
+- compared to other leaderboard you may found online, we do not support automatic evaluation for new model submissions
 """
 
 ITA_EVAL_REPO = "https://github.com/g8a9/ita-eval"

src/display/utils.py

@@ -5,6 +5,7 @@ import pandas as pd
 
 from src.about import Tasks
 
+
 def fields(raw_class):
     return [v for k, v in raw_class.__dict__.items() if k[:2] != "__" and k[-2:] != "__"]
 
@@ -20,29 +21,36 @@ class ColumnContent:
     hidden: bool = False
     never_hidden: bool = False
 
+
 ## Leaderboard columns
 auto_eval_column_dict = []
+
 # Init
 auto_eval_column_dict.append(["model_type_symbol", ColumnContent, ColumnContent("T", "str", True, never_hidden=True)])
 auto_eval_column_dict.append(["model", ColumnContent, ColumnContent("Model", "markdown", True, never_hidden=True)])
-#Scores
+
+# Scores
 auto_eval_column_dict.append(["average", ColumnContent, ColumnContent("Average ⬆️", "number", True)])
 for task in Tasks:
     auto_eval_column_dict.append([task.name, ColumnContent, ColumnContent(task.value.col_name, "number", True)])
+
 # Model information
 auto_eval_column_dict.append(["model_type", ColumnContent, ColumnContent("Type", "str", False)])
-auto_eval_column_dict.append(["architecture", ColumnContent, ColumnContent("Architecture", "str", False)])
-auto_eval_column_dict.append(["weight_type", ColumnContent, ColumnContent("Weight type", "str", False, True)])
 auto_eval_column_dict.append(["precision", ColumnContent, ColumnContent("Precision", "str", False)])
 auto_eval_column_dict.append(["license", ColumnContent, ColumnContent("Hub License", "str", False)])
 auto_eval_column_dict.append(["params", ColumnContent, ColumnContent("#Params (B)", "number", False)])
+auto_eval_column_dict.append(["weight_type", ColumnContent, ColumnContent("Weight type", "str", False, True)])
+auto_eval_column_dict.append(["architecture", ColumnContent, ColumnContent("Architecture", "str", False)])
 auto_eval_column_dict.append(["likes", ColumnContent, ColumnContent("Hub ❤️", "number", False)])
 auto_eval_column_dict.append(["still_on_hub", ColumnContent, ColumnContent("Available on the hub", "bool", False)])
 auto_eval_column_dict.append(["revision", ColumnContent, ColumnContent("Model sha", "str", False, False)])
+auto_eval_column_dict.append(["training_codebase", ColumnContent, ColumnContent("Training code", "str", False, False)])
+auto_eval_column_dict.append(["training_data", ColumnContent, ColumnContent("Training data", "str", False, False)])
 
 # We use make dataclass to dynamically fill the scores from Tasks
 AutoEvalColumn = make_dataclass("AutoEvalColumn", auto_eval_column_dict, frozen=True)
 
+
 ## For the queue columns in the submission tab
 @dataclass(frozen=True)
 class EvalQueueColumn:  # Queue column
@@ -53,12 +61,13 @@ class EvalQueueColumn:  # Queue column
     weight_type = ColumnContent("weight_type", "str", "Original")
     status = ColumnContent("status", "str", True)
 
+
 ## All the model information that we might need
 @dataclass
 class ModelDetails:
     name: str
     display_name: str = ""
-    symbol: str = "" # emoji
+    symbol: str = ""  # emoji
 
 
 class ModelType(Enum):
@@ -83,18 +92,37 @@ class ModelType(Enum):
             return ModelType.IFT
         return ModelType.Unknown
 
+
+class DisclosedType(Enum):
+    D = ModelDetails(name="disclosed", symbol="🟢")
+    UD = ModelDetails(name="undisclosed", symbol="⭕")
+    Unknown = ModelDetails(name="", symbol="?")
+
+    def to_str(self, separator=" "):
+        return f"{self.value.symbol}{separator}{self.value.name}"
+
+    @staticmethod
+    def from_str(type):
+        if "disclosed" in type or "🟢" in type:
+            return DisclosedType.D
+        if "undisclosed" in type or "⭕" in type:
+            return DisclosedType.UD
+        return DisclosedType.Unknown
+
+
 class WeightType(Enum):
     Adapter = ModelDetails("Adapter")
     Original = ModelDetails("Original")
     Delta = ModelDetails("Delta")
 
+
 class Precision(Enum):
     float16 = ModelDetails("float16")
     bfloat16 = ModelDetails("bfloat16")
     float32 = ModelDetails("float32")
-    #qt_8bit = ModelDetails("8bit")
-    #qt_4bit = ModelDetails("4bit")
-    #qt_GPTQ = ModelDetails("GPTQ")
+    # qt_8bit = ModelDetails("8bit")
+    # qt_4bit = ModelDetails("4bit")
+    # qt_GPTQ = ModelDetails("GPTQ")
     Unknown = ModelDetails("?")
 
     def from_str(precision):
@@ -104,14 +132,15 @@ class Precision(Enum):
             return Precision.bfloat16
         if precision in ["float32"]:
             return Precision.float32
-        #if precision in ["8bit"]:
+        # if precision in ["8bit"]:
         #    return Precision.qt_8bit
-        #if precision in ["4bit"]:
+        # if precision in ["4bit"]:
         #    return Precision.qt_4bit
-        #if precision in ["GPTQ", "None"]:
+        # if precision in ["GPTQ", "None"]:
         #    return Precision.qt_GPTQ
         return Precision.Unknown
 
+
 # Column selection
 COLS = [c.name for c in fields(AutoEvalColumn) if not c.hidden]
 TYPES = [c.type for c in fields(AutoEvalColumn) if not c.hidden]

src/leaderboard/read_evals.py

@@ -8,7 +8,7 @@ import dateutil
 import numpy as np
 
 from src.display.formatting import make_clickable_model
-from src.display.utils import AutoEvalColumn, ModelType, Tasks, Precision, WeightType
+from src.display.utils import AutoEvalColumn, ModelType, Tasks, Precision, WeightType, DisclosedType
 from src.submission.check_validity import is_model_on_hub
 
 
@@ -47,8 +47,10 @@ class EvalResult:
             "license": config.get("license", None),
             "num_params": config.get("params", None),
             "base_model": config.get("base_model", None),
-            "training_codebase": config.get("training_codebase", None),
-            "training_data": config.get("training_data", None),
+            "model_type": ModelType.from_str(config.get("model_type", "")),
+            "weight_type": WeightType[config.get("weight_type", "")],
+            "training_codebase": DisclosedType.from_str(config.get("training_codebase", "")),
+            "training_data": DisclosedType.from_str(config.get("training_data", "")),
         }
 
         # Precision
@@ -83,11 +85,21 @@ class EvalResult:
             task = task.value
 
             # We average all scores of a given metric (not all metrics are present in all files)
-            accs = np.array([v.get(task.metric, None) for k, v in data["results"].items() if task.benchmark == k])
+            accs = np.array(
+                [
+                    v.get(task.metric, None) if task.higher_is_better else 1 - v.get(task.metric, None)
+                    for k, v in data["results"].items()
+                    if task.benchmark == k
+                ]
+            )
+
             if accs.size == 0 or any([acc is None for acc in accs]):
                 continue
 
-            mean_acc = np.mean(accs) * 100.0
+            mean_acc = np.mean(accs)
+            if task.scale_by_100:
+                mean_acc *= 100.0
+
             results[task.benchmark] = mean_acc
 
         return self(
@@ -132,12 +144,14 @@ class EvalResult:
             AutoEvalColumn.weight_type.name: self.weight_type.value.name,
             AutoEvalColumn.architecture.name: self.architecture,
             AutoEvalColumn.model.name: make_clickable_model(self.full_model),
-            AutoEvalColumn.revision.name: self.revision,
             AutoEvalColumn.average.name: average,
             AutoEvalColumn.license.name: self.license,
-            AutoEvalColumn.likes.name: self.likes,
             AutoEvalColumn.params.name: self.num_params,
+            AutoEvalColumn.revision.name: self.revision,
+            AutoEvalColumn.likes.name: self.likes,
             AutoEvalColumn.still_on_hub.name: self.still_on_hub,
+            AutoEvalColumn.training_codebase.name: self.training_codebase.value.symbol,
+            AutoEvalColumn.training_data.name: self.training_data.value.symbol,
         }
 
         for task in Tasks: