natural language toolkit senna interface c 20012023 nltk project rami alrfou ralrfoucs stonybrook edu url https www nltk org for license information see license txt a general interface to the senna pipeline that supports any of the operations specified in supportedoperations applying multiple operations at once has the speed advantage for example senna will automatically determine pos tags if you are extracting named entities applying both of the operations will cost only the time of extracting the named entities the senna pipeline has a fixed maximum size of the sentences that it can read by default it is 1024 tokensentence if you have larger sentences changing the maxsentencesize value in sennamain c should be considered and your system specific binary should be rebuilt otherwise this could introduce misalignment errors the input is path to the directory that contains senna executables if the path is incorrect senna will automatically search for executable file specified in senna environment variable list of the operations needed to be performed optionally the encoding of the input data default utf8 note unit tests for this module can be found in testunittestsenna py from nltk classify import senna pipeline senna usrsharesennav3 0 pos chk ner doctest skip sent dusseldorf is an international business center split token word token chk token ner token pos for token in pipeline tagsent doctest skip dusseldorf bnp bloc nnp is bvp o vbz an bnp o dt international inp o jj business inp o nn center inp o nn verifies the existence of the executable on the self path first sennabinaryfile1 self executableself path check for the system environment self path path joinenviron senna the function that determines the system specific binary that should be used in the pipeline in case the system is not known the default senna binary will be used a method that calculates the order of the columns that senna pipeline will output the tags into this depends on the operations being ordered applies the specified operations on a list of tokens applies the tag method over a list of sentences this method will return a list of dictionaries every dictionary will contain a word with its calculated annotationstags build the senna command to run the tagger serialize the actual sentences to a temporary string run the tagger and get the output check the return code output the tagged sentences natural language toolkit senna interface c 2001 2023 nltk project rami al rfou ralrfou cs stonybrook edu url https www nltk org for license information see license txt a general interface to the senna pipeline that supports any of the operations specified in supported_operations applying multiple operations at once has the speed advantage for example senna will automatically determine pos tags if you are extracting named entities applying both of the operations will cost only the time of extracting the named entities the senna pipeline has a fixed maximum size of the sentences that it can read by default it is 1024 token sentence if you have larger sentences changing the max_sentence_size value in senna_main c should be considered and your system specific binary should be rebuilt otherwise this could introduce misalignment errors the input is path to the directory that contains senna executables if the path is incorrect senna will automatically search for executable file specified in senna environment variable list of the operations needed to be performed optionally the encoding of the input data default utf 8 note unit tests for this module can be found in test unit test_senna py from nltk classify import senna pipeline senna usr share senna v3 0 pos chk ner doctest skip sent dusseldorf is an international business center split token word token chk token ner token pos for token in pipeline tag sent doctest skip dusseldorf b np b loc nnp is b vp o vbz an b np o dt international i np o jj business i np o nn center i np o nn verifies the existence of the executable on the self _path first senna_binary_file_1 self executable self _path check for the system environment self _path path join environ senna the function that determines the system specific binary that should be used in the pipeline in case the system is not known the default senna binary will be used a method that calculates the order of the columns that senna pipeline will output the tags into this depends on the operations being ordered applies the specified operation s on a list of tokens applies the tag method over a list of sentences this method will return a list of dictionaries every dictionary will contain a word with its calculated annotations tags build the senna command to run the tagger serialize the actual sentences to a temporary string run the tagger and get the output check the return code output the tagged sentences
from os import environ, path, sep
from platform import architecture, system
from subprocess import PIPE, Popen

from nltk.tag.api import TaggerI


class Senna(TaggerI):

    SUPPORTED_OPERATIONS = ["pos", "chk", "ner"]

    def __init__(self, senna_path, operations, encoding="utf-8"):
        self._encoding = encoding
        self._path = path.normpath(senna_path) + sep

        
        
        exe_file_1 = self.executable(self._path)
        if not path.isfile(exe_file_1):
            
            if "SENNA" in environ:
                
                self._path = path.normpath(environ["SENNA"]) + sep
                exe_file_2 = self.executable(self._path)
                if not path.isfile(exe_file_2):
                    raise LookupError(
                        "Senna executable expected at %s or %s but not found"
                        % (exe_file_1, exe_file_2)
                    )

        self.operations = operations

    def executable(self, base_path):
        
        os_name = system()
        if os_name == "Linux":
            bits = architecture()[0]
            if bits == "64bit":
                return path.join(base_path, "senna-linux64")
            return path.join(base_path, "senna-linux32")
        if os_name == "Windows":
            return path.join(base_path, "senna-win32.exe")
        if os_name == "Darwin":
            return path.join(base_path, "senna-osx")
        return path.join(base_path, "senna")

    def _map(self):
        
        _map = {}
        i = 1
        for operation in Senna.SUPPORTED_OPERATIONS:
            if operation in self.operations:
                _map[operation] = i
                i += 1
        return _map

    def tag(self, tokens):
        
        return self.tag_sents([tokens])[0]

    def tag_sents(self, sentences):
        
        encoding = self._encoding

        if not path.isfile(self.executable(self._path)):
            raise LookupError(
                "Senna executable expected at %s but not found"
                % self.executable(self._path)
            )

        
        _senna_cmd = [
            self.executable(self._path),
            "-path",
            self._path,
            "-usrtokens",
            "-iobtags",
        ]
        _senna_cmd.extend(["-" + op for op in self.operations])

        
        _input = "\n".join(" ".join(x) for x in sentences) + "\n"
        if isinstance(_input, str) and encoding:
            _input = _input.encode(encoding)

        
        p = Popen(_senna_cmd, stdin=PIPE, stdout=PIPE, stderr=PIPE)
        (stdout, stderr) = p.communicate(input=_input)
        senna_output = stdout

        
        if p.returncode != 0:
            raise RuntimeError("Senna command failed! Details: %s" % stderr)

        if encoding:
            senna_output = stdout.decode(encoding)

        
        map_ = self._map()
        tagged_sentences = [[]]
        sentence_index = 0
        token_index = 0
        for tagged_word in senna_output.strip().split("\n"):
            if not tagged_word:
                tagged_sentences.append([])
                sentence_index += 1
                token_index = 0
                continue
            tags = tagged_word.split("\t")
            result = {}
            for tag in map_:
                result[tag] = tags[map_[tag]].strip()
            try:
                result["word"] = sentences[sentence_index][token_index]
            except IndexError as e:
                raise IndexError(
                    "Misalignment error occurred at sentence number %d. Possible reason"
                    " is that the sentence size exceeded the maximum size. Check the "
                    "documentation of Senna class for more information."
                    % sentence_index
                ) from e
            tagged_sentences[-1].append(result)
            token_index += 1
        return tagged_sentences