task_categories:token-classification language:fa region:us
"""

_DOWNLOAD_URLS = {
    "train": "https://huggingface.co/datasets/hezarai/parstwiner/resolve/main/parstwiner_train.csv",
    "test": "https://huggingface.co/datasets/hezarai/parstwiner/resolve/main/parstwiner_test.csv",
}


class ParsTwiNERConfig(datasets.BuilderConfig):
    def __init__(self, **kwargs):
        super(ParsTwiNERConfig, self).__init__(**kwargs)


class ParsTwiNER(datasets.GeneratorBasedBuilder):
    BUILDER_CONFIGS = [
        ParsTwiNERConfig(
            name="ParsTwiNER",
            version=datasets.Version("1.0.0"),
            description=_DESCRIPTION,
        ),
    ]

    def _info(self):
        return datasets.DatasetInfo(
            description=_DESCRIPTION,
            features=datasets.Features(
                {
                    "tokens": datasets.Sequence(datasets.Value("string")),
                    "ner_tags": datasets.Sequence(
                        datasets.features.ClassLabel(
                            names=[
                                "O",
                                "B-POG",
                                "I-POG",
                                "B-PER",
                                "I-PER",
                                "B-ORG",
                                "I-ORG",
                                "B-NAT",
                                "I-NAT",
                                "B-LOC",
                                "I-LOC",
                                "B-EVE",
                                "I-EVE",
                            ]
                        )
                    ),
                }
            ),
            homepage="https://github.com/overfit-ir/parstwiner",
            citation=_CITATION,
        )

    def _split_generators(self, dl_manager):
@inproceedings{aghajani-etal-2021-parstwiner,
    title = "{P}ars{T}wi{NER}: A Corpus for Named Entity Recognition at Informal {P}ersian",
    author = "Aghajani, MohammadMahdi  and
      Badri, AliAkbar  and
      Beigy, Hamid",
    booktitle = "Proceedings of the Seventh Workshop on Noisy User-generated Text (W-NUT 2021)",
    month = nov,
    year = "2021",
    address = "Online",
    publisher = "Association for Computational Linguistics",
    url = "https://aclanthology.org/2021.wnut-1.16",
    pages = "131--136",
    abstract = "As a result of unstructured sentences and some misspellings and errors, finding named entities in a noisy environment such as social media takes much more effort. ParsTwiNER contains about 250k tokens, based on standard instructions like MUC-6 or CoNLL 2003, gathered from Persian Twitter. Using Cohen{'}s Kappa coefficient, the consistency of annotators is 0.95, a high score. In this study, we demonstrate that some state-of-the-art models degrade on these corpora, and trained a new model using parallel transfer learning based on the BERT architecture. Experimental results show that the model works well in informal Persian as well as in formal Persian.",
}
---
task_categories:
- token-classification
language:
- fa
pretty_name: ParsTwiNER
---
ParsTwiNER dataset created by Aghajani et al. [Paper](https://paperswithcode.com/paper/parstwiner-a-corpus-for-named-entity)

> As a result of unstructured sentences and some misspellings and errors, finding named entities in a noisy environment such as social media takes much more effort.
> ParsTwiNER contains about 250k tokens, based on standard instructions like MUC-6 or CoNLL 2003, gathered from Persian Twitter. Using Cohen’s Kappa coefficient, the consistency of annotators is 0.95, a high score.
> In this study, we demonstrate that some state-of-the-art models degrade on these corpora, and trained a new model using parallel transfer learning based on the BERT architecture.
> Experimental results show that the model works well in informal Persian as well as in formal Persian.