natural language toolkit tagset mapping c 20012023 nltk project nathan schneider nathancmu edu steven bird stevenbird1gmail com url https www nltk org for license information see license txt interface for converting pos tags from various treebanks to the universal tagset of petrov das mcdonald the tagset consists of the following 12 coarse tags verb verbs all tenses and modes noun nouns common and proper pron pronouns adj adjectives adv adverbs adp adpositions prepositions and postpositions conj conjunctions det determiners num cardinal numbers prt particles or other function words x other foreign words typos abbreviations punctuation see https arxiv orgabs1104 2086 and https code google compuniversalpostags mappings defaultdictlambda defaultdictdict the mapping between tagset t1 and t2 returns unk if applied to an unrecognized tag when mapping to the universal tagset map unknown inputs to x not unk retrieve the mapping dictionary between tagsets tagsetmapping rurnc universal a adj c conj ad adv nn noun vg verb comp conj nc num vp verb p adp ij x v verb z x vi verb yesnosent x ptcl prt true added the new russian national corpus mappings because the russian model for nltk postag uses it maps the tag from the source tagset to the target tagset maptag enptb universal vbz verb maptag enptb universal vbp verb maptag enptb universal we need a systematic approach to naming natural language toolkit tagset mapping c 2001 2023 nltk project nathan schneider nathan cmu edu steven bird stevenbird1 gmail com url https www nltk org for license information see license txt interface for converting pos tags from various treebanks to the universal tagset of petrov das mcdonald the tagset consists of the following 12 coarse tags verb verbs all tenses and modes noun nouns common and proper pron pronouns adj adjectives adv adverbs adp adpositions prepositions and postpositions conj conjunctions det determiners num cardinal numbers prt particles or other function words x other foreign words typos abbreviations punctuation see https arxiv org abs 1104 2086 and https code google com p universal pos tags _mappings defaultdict lambda defaultdict dict the mapping between tagset t1 and t2 returns unk if applied to an unrecognized tag when mapping to the universal tagset map unknown inputs to x not unk retrieve the mapping dictionary between tagsets tagset_mapping ru rnc universal a adj c conj ad adv nn noun vg verb comp conj nc num vp verb p adp ij x v verb z x vi verb yes_no_sent x ptcl prt true added the new russian national corpus mappings because the russian model for nltk pos_tag uses it maps the tag from the source tagset to the target tagset map_tag en ptb universal vbz verb map_tag en ptb universal vbp verb map_tag en ptb universal we need a systematic approach to naming
from collections import defaultdict
from os.path import join

from nltk.data import load

_UNIVERSAL_DATA = "taggers/universal_tagset"
_UNIVERSAL_TAGS = (
    "VERB",
    "NOUN",
    "PRON",
    "ADJ",
    "ADV",
    "ADP",
    "CONJ",
    "DET",
    "NUM",
    "PRT",
    "X",
    ".",
)



_MAPPINGS = defaultdict(lambda: defaultdict(lambda: defaultdict(lambda: "UNK")))


def _load_universal_map(fileid):
    contents = load(join(_UNIVERSAL_DATA, fileid + ".map"), format="text")

    
    
    _MAPPINGS[fileid]["universal"].default_factory = lambda: "X"

    for line in contents.splitlines():
        line = line.strip()
        if line == "":
            continue
        fine, coarse = line.split("\t")

        assert coarse in _UNIVERSAL_TAGS, f"Unexpected coarse tag: {coarse}"
        assert (
            fine not in _MAPPINGS[fileid]["universal"]
        ), f"Multiple entries for original tag: {fine}"

        _MAPPINGS[fileid]["universal"][fine] = coarse


def tagset_mapping(source, target):
    

    if source not in _MAPPINGS or target not in _MAPPINGS[source]:
        if target == "universal":
            _load_universal_map(source)
            
            
            _MAPPINGS["ru-rnc-new"]["universal"] = {
                "A": "ADJ",
                "A-PRO": "PRON",
                "ADV": "ADV",
                "ADV-PRO": "PRON",
                "ANUM": "ADJ",
                "CONJ": "CONJ",
                "INTJ": "X",
                "NONLEX": ".",
                "NUM": "NUM",
                "PARENTH": "PRT",
                "PART": "PRT",
                "PR": "ADP",
                "PRAEDIC": "PRT",
                "PRAEDIC-PRO": "PRON",
                "S": "NOUN",
                "S-PRO": "PRON",
                "V": "VERB",
            }

    return _MAPPINGS[source][target]


def map_tag(source, target, source_tag):
    

    
    if target == "universal":
        if source == "wsj":
            source = "en-ptb"
        if source == "brown":
            source = "en-brown"

    return tagset_mapping(source, target)[source_tag]