license:cc-by-4.0 region:us
This dataset is designed for Annotation Error Detection.
@inproceedings{larson-etal-2020-inconsistencies,
    title = "Inconsistencies in Crowdsourced Slot-Filling Annotations: A Typology and Identification Methods",
    author = "Larson, Stefan  and
      Cheung, Adrian  and
      Mahendran, Anish  and
      Leach, Kevin  and
      Kummerfeld, Jonathan K.",
    booktitle = "Proceedings of the 28th International Conference on Computational Linguistics",
    month = dec,
    year = "2020",
    address = "Barcelona, Spain (Online)",
    publisher = "International Committee on Computational Linguistics",
    url = "https://aclanthology.org/2020.coling-main.442",
    doi = "10.18653/v1/2020.coling-main.442",
    pages = "5035--5046",
    abstract = "Slot-filling models in task-driven dialog systems rely on carefully annotated training data. However, annotations by crowd workers are often inconsistent or contain errors. Simple solutions like manually checking annotations or having multiple workers label each sample are expensive and waste effort on samples that are correct. If we can identify inconsistencies, we can focus effort where it is needed. Toward this end, we define six inconsistency types in slot-filling annotations. Using three new noisy crowd-annotated datasets, we show that a wide range of inconsistencies occur and can impact system performance if not addressed. We then introduce automatic methods of identifying inconsistencies. Experiments on our new datasets show that these methods effectively reveal inconsistencies in data, though there is further scope for improvement.",
}
---
license: cc-by-4.0
---
