natural language toolkit taggers c 20012023 nltk project edward loper edlopergmail com steven bird stevenbird1gmail com minor additions url https www nltk org for license information see license txt nltk taggers this package contains classes and interfaces for partofspeech tagging or simply tagging a tag is a casesensitive string that specifies some property of a token such as its part of speech tagged tokens are encoded as tuples tag token for example the following tagged token combines the word fly with a noun part of speech tag nn taggedtok fly nn an offtheshelf tagger is available for english it uses the penn treebank tagset from nltk import postag wordtokenize postagwordtokenizejohn s big idea isn t all that bad doctest normalizewhitespace john nnp s pos big jj idea nn is vbz n t rb all pdt that dt bad jj a russian tagger is also available if you specify langrus it uses the russian national corpus tagset postagwordtokenize lang rus doctest skip s v conj adv v s nonlex this package defines several taggers which take a list of tokens assign a tag to each one and return the resulting list of tagged tokens most of the taggers are built automatically based on a training corpus for example the unigram tagger tags each word w by checking what the most frequent tag for w was in a training corpus from nltk corpus import brown from nltk tag import unigramtagger tagger unigramtaggerbrown taggedsentscategories news 500 sent mitchell decried the high rate of unemployment for word tag in tagger tagsent printword tag mitchell np decried none the at high jj rate nn of in unemployment none note that words that the tagger has not seen during training receive a tag of none we evaluate a tagger on data that was not seen during training roundtagger accuracybrown taggedsentscategories news 500 600 3 0 735 for more information please consult chapter 5 of the nltk book isort skipfile currently only supports english and russian throws error if tokens is of string type note that the new russian pos tags from the model contains suffixes see https github comnltknltkissues2151issuecomment430709018 use nltk s currently recommended part of speech tagger to tag the given list of tokens from nltk tag import postag from nltk tokenize import wordtokenize postagwordtokenizejohn s big idea isn t all that bad doctest normalizewhitespace john nnp s pos big jj idea nn is vbz n t rb all pdt that dt bad jj postagwordtokenizejohn s big idea isn t all that bad tagset universal doctest normalizewhitespace john noun s prt big adj idea noun is verb n t adv all det that det bad adj nb use postagsents for efficient tagging of more than one sentence param tokens sequence of tokens to be tagged type tokens liststr param tagset the tagset to be used e g universal wsj brown type tagset str param lang the iso 639 code of the language e g eng for english rus for russian type lang str return the tagged tokens rtype listtuplestr str use nltk s currently recommended part of speech tagger to tag the given list of sentences each consisting of a list of tokens param sentences list of sentences to be tagged type sentences listliststr param tagset the tagset to be used e g universal wsj brown type tagset str param lang the iso 639 code of the language e g eng for english rus for russian type lang str return the list of tagged sentences rtype listlisttuplestr str natural language toolkit taggers c 2001 2023 nltk project edward loper edloper gmail com steven bird stevenbird1 gmail com minor additions url https www nltk org for license information see license txt nltk taggers this package contains classes and interfaces for part of speech tagging or simply tagging a tag is a case sensitive string that specifies some property of a token such as its part of speech tagged tokens are encoded as tuples tag token for example the following tagged token combines the word fly with a noun part of speech tag nn tagged_tok fly nn an off the shelf tagger is available for english it uses the penn treebank tagset from nltk import pos_tag word_tokenize pos_tag word_tokenize john s big idea isn t all that bad doctest normalize_whitespace john nnp s pos big jj idea nn is vbz n t rb all pdt that dt bad jj a russian tagger is also available if you specify lang rus it uses the russian national corpus tagset pos_tag word_tokenize илья оторопел и дважды перечитал бумажку lang rus doctest skip илья s оторопел v и conj дважды adv перечитал v бумажку s nonlex this package defines several taggers which take a list of tokens assign a tag to each one and return the resulting list of tagged tokens most of the taggers are built automatically based on a training corpus for example the unigram tagger tags each word w by checking what the most frequent tag for w was in a training corpus from nltk corpus import brown from nltk tag import unigramtagger tagger unigramtagger brown tagged_sents categories news 500 sent mitchell decried the high rate of unemployment for word tag in tagger tag sent print word tag mitchell np decried none the at high jj rate nn of in unemployment none note that words that the tagger has not seen during training receive a tag of none we evaluate a tagger on data that was not seen during training round tagger accuracy brown tagged_sents categories news 500 600 3 0 735 for more information please consult chapter 5 of the nltk book isort skip_file currently only supports english and russian throws error if tokens is of string type maps to the specified tagset note that the new russian pos tags from the model contains suffixes see https github com nltk nltk issues 2151 issuecomment 430709018 use nltk s currently recommended part of speech tagger to tag the given list of tokens from nltk tag import pos_tag from nltk tokenize import word_tokenize pos_tag word_tokenize john s big idea isn t all that bad doctest normalize_whitespace john nnp s pos big jj idea nn is vbz n t rb all pdt that dt bad jj pos_tag word_tokenize john s big idea isn t all that bad tagset universal doctest normalize_whitespace john noun s prt big adj idea noun is verb n t adv all det that det bad adj nb use pos_tag_sents for efficient tagging of more than one sentence param tokens sequence of tokens to be tagged type tokens list str param tagset the tagset to be used e g universal wsj brown type tagset str param lang the iso 639 code of the language e g eng for english rus for russian type lang str return the tagged tokens rtype list tuple str str use nltk s currently recommended part of speech tagger to tag the given list of sentences each consisting of a list of tokens param sentences list of sentences to be tagged type sentences list list str param tagset the tagset to be used e g universal wsj brown type tagset str param lang the iso 639 code of the language e g eng for english rus for russian type lang str return the list of tagged sentences rtype list list tuple str str
from nltk.tag.api import TaggerI
from nltk.tag.util import str2tuple, tuple2str, untag
from nltk.tag.sequential import (
    SequentialBackoffTagger,
    ContextTagger,
    DefaultTagger,
    NgramTagger,
    UnigramTagger,
    BigramTagger,
    TrigramTagger,
    AffixTagger,
    RegexpTagger,
    ClassifierBasedTagger,
    ClassifierBasedPOSTagger,
)
from nltk.tag.brill import BrillTagger
from nltk.tag.brill_trainer import BrillTaggerTrainer
from nltk.tag.tnt import TnT
from nltk.tag.hunpos import HunposTagger
from nltk.tag.stanford import StanfordTagger, StanfordPOSTagger, StanfordNERTagger
from nltk.tag.hmm import HiddenMarkovModelTagger, HiddenMarkovModelTrainer
from nltk.tag.senna import SennaTagger, SennaChunkTagger, SennaNERTagger
from nltk.tag.mapping import tagset_mapping, map_tag
from nltk.tag.crf import CRFTagger
from nltk.tag.perceptron import PerceptronTagger

from nltk.data import load, find

RUS_PICKLE = (
    "taggers/averaged_perceptron_tagger_ru/averaged_perceptron_tagger_ru.pickle"
)


def _get_tagger(lang=None):
    if lang == "rus":
        tagger = PerceptronTagger(False)
        ap_russian_model_loc = "file:" + str(find(RUS_PICKLE))
        tagger.load(ap_russian_model_loc)
    else:
        tagger = PerceptronTagger()
    return tagger


def _pos_tag(tokens, tagset=None, tagger=None, lang=None):
    
    if lang not in ["eng", "rus"]:
        raise NotImplementedError(
            "Currently, NLTK pos_tag only supports English and Russian "
            "(i.e. lang='eng' or lang='rus')"
        )
    
    elif isinstance(tokens, str):
        raise TypeError("tokens: expected a list of strings, got a string")

    else:
        tagged_tokens = tagger.tag(tokens)
        if tagset:  
            if lang == "eng":
                tagged_tokens = [
                    (token, map_tag("en-ptb", tagset, tag))
                    for (token, tag) in tagged_tokens
                ]
            elif lang == "rus":
                
                
                tagged_tokens = [
                    (token, map_tag("ru-rnc-new", tagset, tag.partition("=")[0]))
                    for (token, tag) in tagged_tokens
                ]
        return tagged_tokens


def pos_tag(tokens, tagset=None, lang="eng"):
    
    tagger = _get_tagger(lang)
    return _pos_tag(tokens, tagset, tagger, lang)


def pos_tag_sents(sentences, tagset=None, lang="eng"):
    
    tagger = _get_tagger(lang)
    return [_pos_tag(sent, tagset, tagger, lang) for sent in sentences]