natural language toolkit senna pos tagger c 20012023 nltk project rami alrfou ralrfoucs stonybrook edu url https www nltk org for license information see license txt senna pos tagger ner tagger chunk tagger the input is path to the directory that contains senna executables if the path is incorrect sennatagger will automatically search for executable file specified in senna environment variable optionally the encoding of the input data default utf8 note unit tests for this module can be found in testunittestsenna py from nltk tag import sennatagger tagger sennatagger usrsharesennav3 0 doctest skip tagger tag what is the airspeed of an unladen swallow split doctest skip what wp is vbz the dt airspeed nn of in an dt unladen nn swallow nn from nltk tag import sennachunktagger chktagger sennachunktagger usrsharesennav3 0 doctest skip chktagger tag what is the airspeed of an unladen swallow split doctest skip what bnp is bvp the bnp airspeed inp of bpp an bnp unladen inp swallow inp o from nltk tag import sennanertagger nertagger sennanertagger usrsharesennav3 0 doctest skip nertagger tag shakespeare theatre was in london split doctest skip shakespeare bper theatre o was o in o london bloc o nertagger tag un headquarters are in ny usa split doctest skip un borg headquarters o are o in o ny bloc o usa bloc o applies the tag method over a list of sentences this method will return for each sentence a list of tuples of word tag applies the tag method over a list of sentences this method will return for each sentence a list of tuples of word tag extracts the chunks in a bio chunktagged sentence from nltk tag import sennachunktagger chktagger sennachunktagger usrsharesennav3 0 doctest skip sent what is the airspeed of an unladen swallow split taggedsent chktagger tagsent doctest skip taggedsent doctest skip what bnp is bvp the bnp airspeed inp of bpp an bnp unladen inp swallow inp o listchktagger biotochunkstaggedsent chunktype np doctest skip what 0 the airspeed 23 an unladen swallow 567 param taggedsent a list of tuples of word and bio chunk tag type taggedsent listtuple param taggedsent the chunk tag that users want to extract e g np or vp type taggedsent str return an iterable of tuples of chunks that users want to extract and their corresponding indices rtype itertuplestr applies the tag method over a list of sentences this method will return for each sentence a list of tuples of word tag natural language toolkit senna pos tagger c 2001 2023 nltk project rami al rfou ralrfou cs stonybrook edu url https www nltk org for license information see license txt senna pos tagger ner tagger chunk tagger the input is path to the directory that contains senna executables if the path is incorrect sennatagger will automatically search for executable file specified in senna environment variable optionally the encoding of the input data default utf 8 note unit tests for this module can be found in test unit test_senna py from nltk tag import sennatagger tagger sennatagger usr share senna v3 0 doctest skip tagger tag what is the airspeed of an unladen swallow split doctest skip what wp is vbz the dt airspeed nn of in an dt unladen nn swallow nn from nltk tag import sennachunktagger chktagger sennachunktagger usr share senna v3 0 doctest skip chktagger tag what is the airspeed of an unladen swallow split doctest skip what b np is b vp the b np airspeed i np of b pp an b np unladen i np swallow i np o from nltk tag import sennanertagger nertagger sennanertagger usr share senna v3 0 doctest skip nertagger tag shakespeare theatre was in london split doctest skip shakespeare b per theatre o was o in o london b loc o nertagger tag un headquarters are in ny usa split doctest skip un b org headquarters o are o in o ny b loc o usa b loc o applies the tag method over a list of sentences this method will return for each sentence a list of tuples of word tag applies the tag method over a list of sentences this method will return for each sentence a list of tuples of word tag extracts the chunks in a bio chunk tagged sentence from nltk tag import sennachunktagger chktagger sennachunktagger usr share senna v3 0 doctest skip sent what is the airspeed of an unladen swallow split tagged_sent chktagger tag sent doctest skip tagged_sent doctest skip what b np is b vp the b np airspeed i np of b pp an b np unladen i np swallow i np o list chktagger bio_to_chunks tagged_sent chunk_type np doctest skip what 0 the airspeed 2 3 an unladen swallow 5 6 7 param tagged_sent a list of tuples of word and bio chunk tag type tagged_sent list tuple param tagged_sent the chunk tag that users want to extract e g np or vp type tagged_sent str return an iterable of tuples of chunks that users want to extract and their corresponding indices rtype iter tuple str append the word to the current_chunk flush the full chunk when out of an np flush the last chunk applies the tag method over a list of sentences this method will return for each sentence a list of tuples of word tag
from nltk.classify import Senna


class SennaTagger(Senna):
    def __init__(self, path, encoding="utf-8"):
        super().__init__(path, ["pos"], encoding)

    def tag_sents(self, sentences):
        
        tagged_sents = super().tag_sents(sentences)
        for i in range(len(tagged_sents)):
            for j in range(len(tagged_sents[i])):
                annotations = tagged_sents[i][j]
                tagged_sents[i][j] = (annotations["word"], annotations["pos"])
        return tagged_sents


class SennaChunkTagger(Senna):
    def __init__(self, path, encoding="utf-8"):
        super().__init__(path, ["chk"], encoding)

    def tag_sents(self, sentences):
        
        tagged_sents = super().tag_sents(sentences)
        for i in range(len(tagged_sents)):
            for j in range(len(tagged_sents[i])):
                annotations = tagged_sents[i][j]
                tagged_sents[i][j] = (annotations["word"], annotations["chk"])
        return tagged_sents

    def bio_to_chunks(self, tagged_sent, chunk_type):
        
        current_chunk = []
        current_chunk_position = []
        for idx, word_pos in enumerate(tagged_sent):
            word, pos = word_pos
            if "-" + chunk_type in pos:  
                current_chunk.append(word)
                current_chunk_position.append(idx)
            else:
                if current_chunk:  
                    _chunk_str = " ".join(current_chunk)
                    _chunk_pos_str = "-".join(map(str, current_chunk_position))
                    yield _chunk_str, _chunk_pos_str
                    current_chunk = []
                    current_chunk_position = []
        if current_chunk:  
            yield " ".join(current_chunk), "-".join(map(str, current_chunk_position))


class SennaNERTagger(Senna):
    def __init__(self, path, encoding="utf-8"):
        super().__init__(path, ["ner"], encoding)

    def tag_sents(self, sentences):
        
        tagged_sents = super().tag_sents(sentences)
        for i in range(len(tagged_sents)):
            for j in range(len(tagged_sents[i])):
                annotations = tagged_sents[i][j]
                tagged_sents[i][j] = (annotations["word"], annotations["ner"])
        return tagged_sents