Fix GSM8k key change issue

(get-answer -> strict-match)

Signed-off-by: jinsol-neubla <[email protected]>

app.py

@@ -80,7 +80,7 @@ leaderboard_df, original_df, plot_df = init_space()
 def update_table(
     hidden_df: pd.DataFrame,
     columns: list,
-    type_query: list,
+    # type_query: list,
     weight_precision_query: str,
     activation_precision_query: str,
     size_query: list,
@@ -90,7 +90,7 @@ def update_table(
 ):
     filtered_df = filter_models(
         df=hidden_df,
-        type_query=type_query,
+        # type_query=type_query,
         size_query=size_query,
         weight_precision_query=weight_precision_query,
         activation_precision_query=activation_precision_query,
@@ -151,7 +151,7 @@ def filter_queries(query: str, filtered_df: pd.DataFrame):
 
 def filter_models(
     df: pd.DataFrame,
-    type_query: list,
+    # type_query: list,
     size_query: list,
     weight_precision_query: list,
     activation_precision_query: list,
@@ -173,8 +173,8 @@ def filter_models(
     if "Flagged" in hide_models:
         filtered_df = filtered_df[filtered_df[AutoEvalColumn.flagged.name] == False]
 
-    type_emoji = [t[0] for t in type_query]
-    filtered_df = filtered_df.loc[df[AutoEvalColumn.model_type_symbol.name].isin(type_emoji)]
+    # type_emoji = [t[0] for t in type_query]
+    # filtered_df = filtered_df.loc[df[AutoEvalColumn.model_type_symbol.name].isin(type_emoji)]
     filtered_df = filtered_df.loc[df[AutoEvalColumn.weight_precision.name].isin(weight_precision_query + ["None"])]
     filtered_df = filtered_df.loc[
         df[AutoEvalColumn.activation_precision.name].isin(activation_precision_query + ["None"])
@@ -191,7 +191,7 @@ def filter_models(
 
 leaderboard_df = filter_models(
     df=leaderboard_df,
-    type_query=[t.to_str(" : ") for t in ModelType],
+    # type_query=[t.to_str(" : ") for t in ModelType],
     size_query=list(NUMERIC_INTERVALS.keys()),
     weight_precision_query=[i.value.name for i in Precision],
     activation_precision_query=[i.value.name for i in Precision],
@@ -239,13 +239,13 @@ with demo:
                         )
                 with gr.Column(min_width=320):
                     # with gr.Box(elem_id="box-filter"):
-                    filter_columns_type = gr.CheckboxGroup(
-                        label="Model types",
-                        choices=[t.to_str() for t in ModelType],
-                        value=[t.to_str() for t in ModelType],
-                        interactive=True,
-                        elem_id="filter-columns-type",
-                    )
+                    # filter_columns_type = gr.CheckboxGroup(
+                    #     label="Model types",
+                    #     choices=[t.to_str() for t in ModelType],
+                    #     value=[t.to_str() for t in ModelType],
+                    #     interactive=True,
+                    #     elem_id="filter-columns-type",
+                    # )
                     filter_columns_weight_precision = gr.CheckboxGroup(
                         label="Weight Precision",
                         choices=[i.value.name for i in Precision],
@@ -301,7 +301,7 @@ with demo:
                 [
                     hidden_leaderboard_table_for_search,
                     shown_columns,
-                    filter_columns_type,
+                    # filter_columns_type,
                     filter_columns_weight_precision,
                     filter_columns_activation_precision,
                     filter_columns_size,
@@ -319,7 +319,7 @@ with demo:
                 [
                     hidden_leaderboard_table_for_search,
                     shown_columns,
-                    filter_columns_type,
+                    # filter_columns_type,
                     filter_columns_weight_precision,
                     filter_columns_activation_precision,
                     filter_columns_size,
@@ -334,7 +334,7 @@ with demo:
 
             for selector in [
                 shown_columns,
-                filter_columns_type,
+                # filter_columns_type,
                 filter_columns_weight_precision,
                 filter_columns_activation_precision,
                 filter_columns_size,
@@ -346,7 +346,7 @@ with demo:
                     [
                         hidden_leaderboard_table_for_search,
                         shown_columns,
-                        filter_columns_type,
+                        # filter_columns_type,
                         filter_columns_weight_precision,
                         filter_columns_activation_precision,
                         filter_columns_size,
@@ -391,4 +391,4 @@ scheduler = BackgroundScheduler()
 scheduler.add_job(restart_space, "interval", seconds=1800)  # restarted every 3h
 scheduler.start()
 
-demo.queue(default_concurrency_limit=40).launch(share=True)
+demo.queue(default_concurrency_limit=40).launch()

requirements.txt

@@ -2,15 +2,15 @@ APScheduler==3.10.1
 black==23.11.0
 click==8.1.3
 datasets==2.14.5
-gradio==4.9.0
-gradio_client==0.7.2
+gradio==4.29.0
+gradio_client
 huggingface-hub>=0.18.0
 matplotlib==3.7.1
 numpy==1.24.2
 pandas==2.0.0
 plotly==5.14.1
 python-dateutil==2.8.2
-requests==2.28.2
+requests
 sentencepiece
 tqdm==4.65.0
 transformers==4.37.0

src/display/utils.py

@@ -1,6 +1,7 @@
 from dataclasses import dataclass, make_dataclass
 from enum import Enum
 from altair import Column
+from typing import Union, List, Dict
 
 import pandas as pd
 
@@ -12,7 +13,7 @@ def fields(raw_class):
 @dataclass
 class Task:
     benchmark: str
-    metric: str
+    metric: Union[str, List[str]]
     col_name: str
 
 
@@ -22,7 +23,17 @@ class Tasks(Enum):
     mmlu = Task("mmlu", "acc", "MMLU")
     truthfulqa = Task("truthfulqa_mc2", "acc", "TruthfulQA")
     winogrande = Task("winogrande", "acc", "Winogrande")
-    gsm8k = Task("gsm8k", "exact_match,get-answer", "GSM8K")
+    gsm8k = Task("gsm8k", ["exact_match,get-answer", "exact_match,strict-match"], "GSM8K")
+
+    @staticmethod
+    def get_metric(task: Task, dict_results: Dict[str, float]):
+        if isinstance(task.metric, str):
+            return dict_results[task.metric]
+        else:
+            for metric in task.metric:
+                if metric in dict_results:
+                    return dict_results[metric]
+            return None
 
 
 # These classes are for user facing column names,
@@ -40,7 +51,7 @@ class ColumnContent:
 
 auto_eval_column_dict = []
 # Init
-auto_eval_column_dict.append(["model_type_symbol", ColumnContent, ColumnContent("T", "str", True, never_hidden=True)])
+# auto_eval_column_dict.append(["model_type_symbol", ColumnContent, ColumnContent("T", "str", True, never_hidden=True)])
 auto_eval_column_dict.append(["model", ColumnContent, ColumnContent("Model", "markdown", True, never_hidden=True)])
 # Scores
 auto_eval_column_dict.append(["average", ColumnContent, ColumnContent("Average ⬆️", "number", True)])
@@ -173,7 +184,7 @@ class Precision(Enum):
     Unknown = ModelDetails("?")
 
     def from_str(precision):
-        if precision in ["torch.float16", "float16"]:
+        if precision in ["torch.float16", "float16", "fp16"]:
             return Precision.float16
         if precision in ["torch.bfloat16", "bfloat16"]:
             return Precision.bfloat16

src/leaderboard/read_evals.py

@@ -94,7 +94,7 @@ class EvalResult:
                 if task.benchmark == "mmlu":
                     accs = np.array([data["results"].get(task.benchmark, {}).get(task.metric, None)])
                 else:
-                    accs = np.array([v.get(task.metric, None) for k, v in data["results"].items() if task.benchmark in k])
+                    accs = np.array([Tasks.get_metric(task, v) for k, v in data["results"].items() if task.benchmark in k])
                 if accs.size == 0 or any([acc is None for acc in accs]):
                     continue
 
@@ -154,7 +154,7 @@ class EvalResult:
             AutoEvalColumn.weight_precision.name: self.weight_precision.value.name,
             AutoEvalColumn.activation_precision.name: self.activation_precision.value.name,
             AutoEvalColumn.model_type.name: self.model_type.value.name,
-            AutoEvalColumn.model_type_symbol.name: self.model_type.value.symbol,
+            # AutoEvalColumn.model_type_symbol.name: self.model_type.value.symbol,
             AutoEvalColumn.weight_type.name: self.weight_type.value.name,
             AutoEvalColumn.architecture.name: self.architecture,
             AutoEvalColumn.model.name: make_clickable_model(self.full_model),
@@ -216,6 +216,7 @@ def get_raw_eval_results(results_path: str) -> list[EvalResult]:
 
     eval_results = {}
     for model_result_filepath in model_result_filepaths:
+        print(f"Read {model_result_filepath}")
         # Creation of result
         eval_result = EvalResult.init_from_json_file(model_result_filepath)
 
@@ -232,7 +233,9 @@ def get_raw_eval_results(results_path: str) -> list[EvalResult]:
             if v.status == "FINISHED":
                 v.to_dict()  # we test if the dict version is complete
                 results.append(v)
-        except KeyError:  # not all eval values present
+        except KeyError as e:  # not all eval values present
+            print(f"Fail to get results from {v.eval_name} with the error {e}")
+            print(v)
             continue
 
     return results