natural language toolkit interface to the hunpos postagger c 20012023 nltk project peter ljunglf peter ljunglofheatherleaf se dvid mrk nemeskey nemeskeydgmail com modifications attila zsder zsedergmail com modifications url https www nltk org for license information see license txt a module for interfacing with the hunpos opensource postagger the default encoding used by hunpos iso88591 class hunpostaggertaggeri def init self pathtomodel pathtobinnone encodinghunposcharset verbosefalse self closed true hunpospaths usrbin usrlocalbin optlocalbin applicationsbin bin applicationsbin hunpospaths listmapos path expanduser hunpospaths self hunposbin findbinary hunpostag pathtobin envvarshunpostagger searchpathhunpospaths urlhunposurl verboseverbose self hunposmodel findfile pathtomodel envvarshunpostagger verboseverbose self encoding encoding self hunpos popen self hunposbin self hunposmodel shellfalse stdinpipe stdoutpipe stderrpipe self closed false def delself self close def closeself tags a single sentence a list of words the tokens should not contain any newline characters we write a final empty line to tell hunpos that the sentence is finished we have to read and dismiss the final empty line natural language toolkit interface to the hunpos pos tagger c 2001 2023 nltk project peter ljunglöf peter ljunglof heatherleaf se dávid márk nemeskey nemeskeyd gmail com modifications attila zséder zseder gmail com modifications url https www nltk org for license information see license txt a module for interfacing with the hunpos open source pos tagger the default encoding used by hunpos iso 8859 1 a class for pos tagging with hunpos the input is the paths to a model trained on training data optionally the path to the hunpos tag binary optionally the encoding of the training data default iso 8859 1 check whether the required hunpos tag binary is available from nltk test setup_fixt import check_binary check_binary hunpos tag example from nltk tag import hunpostagger ht hunpostagger en_wsj model ht tag what is the airspeed of an unladen swallow split what wp is vbz the dt airspeed nn of in an dt unladen nn swallow vb ht close this class communicates with the hunpos tag binary via pipes when the tagger object is no longer needed the close method should be called to free system resources the class supports the context manager interface if used in a with statement the close method is invoked automatically with hunpostagger en_wsj model as ht ht tag what is the airspeed of an unladen swallow split what wp is vbz the dt airspeed nn of in an dt unladen nn swallow vb starts the hunpos tag executable and establishes a connection with it param path_to_model the model file param path_to_bin the hunpos tag binary param encoding the encoding used by the model unicode tokens passed to the tag and tag_sents methods are converted to this charset when they are sent to hunpos tag the default is iso 8859 1 latin 1 this parameter is ignored for str tokens which are sent as is the caller must ensure that tokens are encoded in the right charset closes the pipe to the hunpos executable tags a single sentence a list of words the tokens should not contain any newline characters we write a final empty line to tell hunpos that the sentence is finished we have to read and dismiss the final empty line
import os
from subprocess import PIPE, Popen

from nltk.internals import find_binary, find_file
from nltk.tag.api import TaggerI

_hunpos_url = "https://code.google.com/p/hunpos/"

_hunpos_charset = "ISO-8859-1"



class HunposTagger(TaggerI):
    

    def __init__(
        self, path_to_model, path_to_bin=None, encoding=_hunpos_charset, verbose=False
    ):
        
        self._closed = True
        hunpos_paths = [
            ".",
            "/usr/bin",
            "/usr/local/bin",
            "/opt/local/bin",
            "/Applications/bin",
            "~/bin",
            "~/Applications/bin",
        ]
        hunpos_paths = list(map(os.path.expanduser, hunpos_paths))

        self._hunpos_bin = find_binary(
            "hunpos-tag",
            path_to_bin,
            env_vars=("HUNPOS_TAGGER",),
            searchpath=hunpos_paths,
            url=_hunpos_url,
            verbose=verbose,
        )

        self._hunpos_model = find_file(
            path_to_model, env_vars=("HUNPOS_TAGGER",), verbose=verbose
        )
        self._encoding = encoding
        self._hunpos = Popen(
            [self._hunpos_bin, self._hunpos_model],
            shell=False,
            stdin=PIPE,
            stdout=PIPE,
            stderr=PIPE,
        )
        self._closed = False

    def __del__(self):
        self.close()

    def close(self):
        
        if not self._closed:
            self._hunpos.communicate()
            self._closed = True

    def __enter__(self):
        return self

    def __exit__(self, exc_type, exc_value, traceback):
        self.close()

    def tag(self, tokens):
        
        for token in tokens:
            assert "\n" not in token, "Tokens should not contain newlines"
            if isinstance(token, str):
                token = token.encode(self._encoding)
            self._hunpos.stdin.write(token + b"\n")
        
        self._hunpos.stdin.write(b"\n")
        self._hunpos.stdin.flush()

        tagged_tokens = []
        for token in tokens:
            tagged = self._hunpos.stdout.readline().strip().split(b"\t")
            tag = tagged[1] if len(tagged) > 1 else None
            tagged_tokens.append((token, tag))
        
        self._hunpos.stdout.readline()

        return tagged_tokens