fix

src/control_panel.py

@@ -40,7 +40,7 @@ def create_control_panel(machine: str):
         with gr.Row():
             with gr.Column(scale=1, variant="panel"):
                 datatype_checkboxes = gr.CheckboxGroup(
-                    label="DTypes 📥",
+                    label="Precision 📥",
                     choices=["float32", "float16", "bfloat16"],
                     value=["float32", "float16", "bfloat16"],
                     info="☑️ Select the load data types",
@@ -49,8 +49,8 @@ def create_control_panel(machine: str):
             with gr.Column(scale=1, variant="panel"):
                 optimization_checkboxes = gr.CheckboxGroup(
                     label="Attentions 👁️",
-                    choices=["None", "BetterTransformer", "FlashAttentionV2"],
-                    value=["None", "BetterTransformer", "FlashAttentionV2"],
+                    choices=["Eager", "SDPA", "FAv2"],
+                    value=["Eager", "SDPA", "FAv2"],
                     info="☑️ Select the optimization",
                     elem_id="optimization-checkboxes",
                 )
@@ -61,21 +61,15 @@ def create_control_panel(machine: str):
                         "None",
                         "BnB.4bit",
                         "BnB.8bit",
+                        "AWQ.4bit",
                         "GPTQ.4bit",
-                        "GPTQ.4bit+ExllamaV1",
-                        "GPTQ.4bit+ExllamaV2",
-                        "AWQ.4bit+GEMM",
-                        "AWQ.4bit+GEMV",
                     ],
                     value=[
                         "None",
                         "BnB.4bit",
                         "BnB.8bit",
+                        "AWQ.4bit",
                         "GPTQ.4bit",
-                        "GPTQ.4bit+ExllamaV1",
-                        "GPTQ.4bit+ExllamaV2",
-                        "AWQ.4bit+GEMM",
-                        "AWQ.4bit+GEMV",
                     ],
                     info="☑️ Select the quantization schemes",
                     elem_id="quantization-checkboxes",
@@ -100,31 +94,35 @@ def create_control_panel(machine: str):
     )
 
 
-def filter_fn(
+def filter_rows_fn(
     machine,
     # inputs
     score,
     memory,
     backends,
-    datatypes,
-    optimizations,
+    precisions,
+    attentions,
     quantizations,
     # interactive
     columns,
     search,
 ):
-    raw_df = get_llm_perf_df(machine=machine)
-    filtered_df = raw_df[
-        # raw_df["Model 🤗"].str.contains(model, case=False)
-        raw_df["Backend 🏭"].isin(backends)
-        & raw_df["DType 📥"].isin(datatypes)
-        & raw_df["Attention 👁️"].isin(optimizations)
-        & raw_df["Quantization 🗜️"].isin(quantizations)
-        & (raw_df["Open LLM Score (%)"] >= score)
-        & (raw_df["Allocated Memory (MB)"] <= memory)
+    llm_perf_df = get_llm_perf_df(machine=machine)
+    # print(attentions)
+    # print(llm_perf_df["Attention 👁️"].unique())
+    filtered_llm_perf_df = llm_perf_df[
+        llm_perf_df["Model 🤗"].str.contains(search, case=False)
+        & llm_perf_df["Backend 🏭"].isin(backends)
+        & llm_perf_df["Precision 📥"].isin(precisions)
+        & llm_perf_df["Attention 👁️"].isin(attentions)
+        & llm_perf_df["Quantization 🗜️"].isin(quantizations)
+        & (llm_perf_df["Open LLM Score (%)"] >= score)
+        & (llm_perf_df["Memory (MB)"] <= memory)
     ]
-    filtered_leaderboard_df = select_fn(machine, columns, search)
-    filtered_lat_score_mem_fig = get_lat_score_mem_fig(filtered_df)
+    selected_filtered_llm_perf_df = select_columns_fn(
+        machine, columns, search, filtered_llm_perf_df
+    )
+    selected_filtered_lat_score_mem_fig = get_lat_score_mem_fig(filtered_llm_perf_df)
     # filtered_bt_prefill_fig = get_bt_prefill_fig(filtered_df)
     # filtered_bt_decode_fig = get_bt_decode_fig(filtered_df)
     # filtered_fa2_prefill_fig = get_fa2_prefill_fig(filtered_df)
@@ -133,8 +131,8 @@ def filter_fn(
     # filtered_quant_decode_fig = get_quant_decode_fig(filtered_df)
 
     return [
-        filtered_leaderboard_df,
-        filtered_lat_score_mem_fig,
+        selected_filtered_llm_perf_df,
+        selected_filtered_lat_score_mem_fig,
         # filtered_bt_prefill_fig,
         # filtered_bt_decode_fig,
         # filtered_fa2_prefill_fig,
@@ -170,7 +168,7 @@ def create_control_callback(
     # quant_decode_plot,
 ):
     filter_button.click(
-        fn=filter_fn,
+        fn=filter_rows_fn,
         inputs=[
             # fixed
             machine_textbox,
@@ -198,8 +196,10 @@ def create_control_callback(
     )
 
 
-def select_fn(machine, columns, search):
-    llm_perf_df = get_llm_perf_df(machine=machine)
+def select_columns_fn(machine, columns, search, llm_perf_df=None):
+    if llm_perf_df is None:
+        llm_perf_df = get_llm_perf_df(machine=machine)
+
     selected_leaderboard_df = get_leaderboard_df(llm_perf_df)
     selected_leaderboard_df = selected_leaderboard_df[
         selected_leaderboard_df["Model 🤗"].str.contains(search, case=False)
@@ -219,12 +219,12 @@ def create_select_callback(
     leaderboard_table,
 ):
     columns_checkboxes.change(
-        fn=select_fn,
+        fn=select_columns_fn,
         inputs=[machine_textbox, columns_checkboxes, search_bar],
         outputs=[leaderboard_table],
     )
     search_bar.change(
-        fn=select_fn,
+        fn=select_columns_fn,
         inputs=[machine_textbox, columns_checkboxes, search_bar],
         outputs=[leaderboard_table],
     )

src/llm_perf.py

@@ -36,19 +36,19 @@ def get_raw_llm_perf_df(machine: str = "1xA10"):
         try:
             dfs.append(
                 pd.read_csv(
-                    f"hf://datasets/optimum-benchmark/llm-perf-leaderboard/llm-perf-leaderboard-{subset}-{machine}.csv"
+                    f"hf://datasets/optimum-benchmark/llm-perf-leaderboard/perf-df-{subset}-{machine}.csv"
                 )
             )
         except Exception:
             print(f"Subset {subset} for machine {machine} not found")
 
-    llm_perf_df = pd.concat(dfs)
-    open_llm_df = pd.read_csv(
-        "hf://datasets/optimum-benchmark/open-llm-leaderboard/open-llm-leaderboard.csv"
+    perf_df = pd.concat(dfs)
+    llm_df = pd.read_csv(
+        "hf://datasets/optimum-benchmark/llm-perf-leaderboard/llm-df.csv"
     )
 
     llm_perf_df = pd.merge(
-        open_llm_df, llm_perf_df, left_on="Model", right_on="config.backend.model"
+        llm_df, perf_df, left_on="Model", right_on="config.backend.model"
     )
 
     return llm_perf_df