Merge pull request #15 from huggingface/fix-app

app.py

@@ -42,15 +42,9 @@ TASK_TO_ID = {
 TASK_TO_DEFAULT_METRICS = {
     "binary_classification": ["f1", "precision", "recall", "auc", "accuracy"],
     "multi_class_classification": [
-        "f1_micro",
-        "f1_macro",
-        "f1_weighted",
-        "precision_macro",
-        "precision_micro",
-        "precision_weighted",
-        "recall_macro",
-        "recall_micro",
-        "recall_weighted",
+        "f1",
+        "precision",
+        "recall",
         "accuracy",
     ],
     "entity_extraction": ["precision", "recall", "f1", "accuracy"],
@@ -72,6 +66,7 @@ def get_supported_metrics():
         except Exception as e:
             print(e)
             print("Skipping the following metric, which cannot load:", metric)
+            continue
 
         argspec = inspect.getfullargspec(metric_func.compute)
         if "references" in argspec.kwonlyargs and "predictions" in argspec.kwonlyargs:
@@ -307,9 +302,7 @@ with st.expander("Advanced configuration"):
             col_mapping[answers_text_col] = "answers.text"
             col_mapping[answers_start_col] = "answers.answer_start"
 
-with st.form(key="form"):
-
-    compatible_models = get_compatible_models(selected_task, selected_dataset)
+    st.markdown("**Select metrics**")
     st.markdown("The following metrics will be computed")
     html_string = " ".join(
         [
@@ -328,26 +321,31 @@ with st.form(key="form"):
     )
     st.info(
         "Note: user-selected metrics will be run with their default arguments from "
-        + "[here](https://github.com/huggingface/datasets/tree/master/metrics)"
+        + "[here](https://github.com/huggingface/evaluate/tree/main/metrics)"
     )
 
+with st.form(key="form"):
+
+    compatible_models = get_compatible_models(selected_task, selected_dataset)
+
     selected_models = st.multiselect("Select the models you wish to evaluate", compatible_models)
     print("Selected models:", selected_models)
 
-    selected_models = filter_evaluated_models(
-        selected_models,
-        selected_task,
-        selected_dataset,
-        selected_config,
-        selected_split,
-    )
-    print("Selected models:", selected_models)
+    if len(selected_models) > 0:
+        selected_models = filter_evaluated_models(
+            selected_models,
+            selected_task,
+            selected_dataset,
+            selected_config,
+            selected_split,
+        )
+        print("Selected models:", selected_models)
 
     submit_button = st.form_submit_button("Make submission")
 
     if submit_button:
         if len(selected_models) > 0:
-            project_id = str(uuid.uuid4())[:3]
+            project_id = str(uuid.uuid4())
             payload = {
                 "username": AUTOTRAIN_USERNAME,
                 "proj_name": f"eval-project-{project_id}",

evaluation.py

@@ -15,10 +15,13 @@ class EvaluationInfo:
 
 
 def compute_evaluation_id(dataset_info: DatasetInfo) -> int:
-    metadata = dataset_info.cardData["eval_info"]
-    metadata.pop("col_mapping", None)
-    evaluation_info = EvaluationInfo(**metadata)
-    return hash(evaluation_info)
+    if dataset_info.cardData is not None:
+        metadata = dataset_info.cardData["eval_info"]
+        metadata.pop("col_mapping", None)
+        evaluation_info = EvaluationInfo(**metadata)
+        return hash(evaluation_info)
+    else:
+        return None
 
 
 def get_evaluation_ids():