bugfix

requirements.txt

@@ -13,6 +13,5 @@ requests==2.28.2
 tqdm==4.65.0
 transformers==4.35.2
 tokenizers>=0.15.0
-git+https://github.com/EleutherAI/lm-evaluation-harness.git@b281b0921b636bc36ad05c0b0b0763bd6dd43463#egg=lm-eval
 accelerate==0.24.1
-sentencepiece
+sentencepiece

src/about.py

@@ -25,7 +25,8 @@ class Tasks(Enum):
     task7 = Task("itacola", "mcc,none", "ItaCoLA", scale_by_100=False)
     task8 = Task("news_sum", "bertscore,none", "News Sum")
     task9 = Task("squad_it", "squad_f1,get-answer", "SQuAD it")
-    task10 = Task("truthfulqa_gen_ita", "rouge1_max,none", "TruthfulQA")
+    task10 = Task("truthfulqa_mc2_ita", "acc,none", "TruthfulQA")
+    task11 = Task("xcopa_it", "acc,none", "TruthfulQA")
 
 
 NUM_FEWSHOT = 0  # Change with your few shot

src/display/utils.py

@@ -13,7 +13,7 @@ def fields(raw_class):
 # These classes are for user facing column names,
 # to avoid having to change them all around the code
 # when a modif is needed
-@dataclass
+@dataclass(frozen=True)
 class ColumnContent:
     name: str
     type: str
@@ -114,6 +114,18 @@ class WeightType(Enum):
     Adapter = ModelDetails("Adapter")
     Original = ModelDetails("Original")
     Delta = ModelDetails("Delta")
+    Unknown = ModelDetails("Unknown")
+
+    @staticmethod
+    def from_str(type):
+        if type == "adapter":
+            return WeightType.Adapter
+        elif type == "original":
+            return WeightType.Original
+        elif type == "delta":
+            return WeightType.Delta
+        else:
+            return WeightType.Unknown
 
 
 class Precision(Enum):

src/leaderboard/read_evals.py

@@ -48,7 +48,7 @@ class EvalResult:
             "num_params": config.get("params", None),
             "base_model": config.get("base_model", None),
             "model_type": ModelType.from_str(config.get("model_type", "")),
-            "weight_type": WeightType[config.get("weight_type", "")],
+            "weight_type": WeightType.from_str(config.get("weight_type", "")),
             "training_codebase": DisclosedType.from_str(config.get("training_codebase", "")),
             "training_data": DisclosedType.from_str(config.get("training_data", "")),
         }
@@ -57,7 +57,7 @@ class EvalResult:
         precision = Precision.from_str(config.get("model_dtype"))
 
         # Get model and org
-        org_and_model = config.get("model_name", config.get("model_args", None))
+        org_and_model = config.get("model_name", data.get("model_name", config.get("model_args", None)))
         org_and_model = org_and_model.split("/", 1)
 
         if len(org_and_model) == 1:
@@ -197,6 +197,9 @@ def get_raw_eval_results(results_path: str, requests_path: str) -> list[EvalResu
         for file in files:
             model_result_filepaths.append(os.path.join(root, file))
 
+    # Exclude any "samples_* file"
+    model_result_filepaths = [m for m in model_result_filepaths if not os.path.basename(m).startswith("samples_")]
+
     eval_results = {}
     for model_result_filepath in model_result_filepaths:
         # Creation of result