Merge pull request #12 from huggingface/fix-qa-cols

README.md

@@ -1,6 +1,6 @@
 ---
 title: AutoEvaluate
-emoji: 🏢
+emoji: 📊
 colorFrom: red
 colorTo: red
 sdk: streamlit
@@ -22,6 +22,6 @@ The table below shows which tasks are currently supported for evaluation in the
 | `multi_class_classification`    |     ✅     | [`eval-staging-822`](https://huggingface.co/datasets/autoevaluate/eval-staging-822) |
 | `multi_label_classification`    |     ❌     |                                                                                     |
 | `entity_extraction`             |     ✅     | [`eval-staging-838`](https://huggingface.co/datasets/autoevaluate/eval-staging-838) |
-| `extractive_question_answering` |     ❌     |                                                                                     |
+| `extractive_question_answering` |     ✅     |                                                                                     |
 | `translation`                   |     ❌     |                                                                                     |
 | `summarization`                 |     ❌     |                                                                                     |

app.py

@@ -9,7 +9,14 @@ from dotenv import load_dotenv
 from huggingface_hub import list_datasets
 
 from evaluation import filter_evaluated_models
-from utils import get_compatible_models, get_key, get_metadata, http_get, http_post
+from utils import (
+    format_col_mapping,
+    get_compatible_models,
+    get_key,
+    get_metadata,
+    http_get,
+    http_post,
+)
 
 if Path(".env").is_file():
     load_dotenv(".env")
@@ -26,7 +33,7 @@ TASK_TO_ID = {
     # "multi_label_classification": 3, # Not fully supported in AutoTrain
     "entity_extraction": 4,
     "extractive_question_answering": 5,
-    # "translation": 6, $ Not fully supported in AutoTrain evaluation
+    # "translation": 6, # Not fully supported in AutoTrain evaluation
     "summarization": 8,
 }
 
@@ -202,7 +209,7 @@ with st.expander("Advanced configuration"):
     elif selected_task == "extractive_question_answering":
         col_mapping = metadata[0]["col_mapping"]
         # Hub YAML parser converts periods to hyphens, so we remap them here
-        col_mapping = {k.replace("-", "."): v.replace("-", ".") for k, v in col_mapping.items()}
+        col_mapping = format_col_mapping(col_mapping)
         with col1:
             st.markdown("`context` column")
             st.text("")
@@ -332,6 +339,6 @@ with st.form(key="form"):
                         """
                         )
                     else:
-                        st.error("🙈 Oh noes, there was an error submitting your evaluation job!")
+                        st.error("🙈 Oh no, there was an error submitting your evaluation job!")
         else:
             st.warning("⚠️ No models were selected for evaluation!")

utils.py

@@ -70,7 +70,7 @@ def get_compatible_models(task, dataset_name):
         library=["transformers", "pytorch"],
     )
     compatible_models = api.list_models(filter=filt)
-    return [model.modelId for model in compatible_models]
+    return sorted([model.modelId for model in compatible_models])
 
 
 def get_key(col_mapping, val):
@@ -79,3 +79,10 @@ def get_key(col_mapping, val):
             return key
 
     return "key doesn't exist"
+
+
+def format_col_mapping(col_mapping: dict) -> dict:
+    for k, v in col_mapping["answers"].items():
+        col_mapping[f"answers.{k}"] = f"answers.{v}"
+    del col_mapping["answers"]
+    return col_mapping