natural language toolkit interface to the stanford tokenizer c 20012023 nltk project steven xu xxustudent unimelb edu au url https www nltk org for license information see license txt jar stanfordpostagger jar def init self pathtojarnone encodingutf8 optionsnone verbosefalse javaoptionsmx1000m raise deprecation warning warnings warn str nthe stanfordtokenizer will be deprecated in version 3 2 5 n please use 03391mnltk parse corenlp corenlpparser0330m instead deprecationwarning stacklevel2 self stanfordjar findjar self jar pathtojar envvarsstanfordpostagger searchpath urlstanfordurl verboseverbose self encoding encoding self javaoptions javaoptions options if options is none else options self optionscmd joinfkeyval for key val in options items staticmethod def parsetokenizedoutputs return s splitlines def tokenizeself s cmd edu stanford nlp process ptbtokenizer return self parsetokenizedoutputself executecmd s def executeself cmd input verbosefalse encoding self encoding cmd extendcharset encoding optionscmd self optionscmd if optionscmd cmd extendoptions self optionscmd defaultoptions joinjavaoptions configure java configjavaoptionsself javaoptions verboseverbose windows is incompatible with namedtemporaryfile without passing in deletefalse with tempfile namedtemporaryfilemodewb deletefalse as inputfile write the actual sentences to the temporary input file if isinstanceinput str and encoding input input encodeencoding inputfile writeinput inputfile flush cmd appendinputfile name run the tagger and get the output stdout stderr java cmd classpathself stanfordjar stdoutpipe stderrpipe stdout stdout decodeencoding os unlinkinputfile name return java configurations to their default values configjavaoptionsdefaultoptions verbosefalse return stdout natural language toolkit interface to the stanford tokenizer c 2001 2023 nltk project steven xu xxu student unimelb edu au url https www nltk org for license information see license txt interface to the stanford tokenizer from nltk tokenize stanford import stanfordtokenizer s good muffins cost 3 88 nin new york please buy me ntwo of them nthanks stanfordtokenizer tokenize s doctest skip good muffins cost 3 88 in new york please buy me two of them thanks s the colour of the wall is blue stanfordtokenizer options americanize true tokenize s doctest skip the color of the wall is blue raise deprecation warning use stanford tokenizer s ptbtokenizer to tokenize multiple sentences configure java windows is incompatible with namedtemporaryfile without passing in delete false write the actual sentences to the temporary input file run the tagger and get the output return java configurations to their default values
import json
import os
import tempfile
import warnings
from subprocess import PIPE

from nltk.internals import _java_options, config_java, find_jar, java
from nltk.parse.corenlp import CoreNLPParser
from nltk.tokenize.api import TokenizerI

_stanford_url = "https://nlp.stanford.edu/software/tokenizer.shtml"


class StanfordTokenizer(TokenizerI):
    r

    _JAR = "stanford-postagger.jar"

    def __init__(
        self,
        path_to_jar=None,
        encoding="utf8",
        options=None,
        verbose=False,
        java_options="-mx1000m",
    ):
        
        warnings.warn(
            str(
                "\nThe StanfordTokenizer will "
                "be deprecated in version 3.2.5.\n"
                "Please use \033[91mnltk.parse.corenlp.CoreNLPParser\033[0m instead.'"
            ),
            DeprecationWarning,
            stacklevel=2,
        )

        self._stanford_jar = find_jar(
            self._JAR,
            path_to_jar,
            env_vars=("STANFORD_POSTAGGER",),
            searchpath=(),
            url=_stanford_url,
            verbose=verbose,
        )

        self._encoding = encoding
        self.java_options = java_options

        options = {} if options is None else options
        self._options_cmd = ",".join(f"{key}={val}" for key, val in options.items())

    @staticmethod
    def _parse_tokenized_output(s):
        return s.splitlines()

    def tokenize(self, s):
        
        cmd = ["edu.stanford.nlp.process.PTBTokenizer"]
        return self._parse_tokenized_output(self._execute(cmd, s))

    def _execute(self, cmd, input_, verbose=False):
        encoding = self._encoding
        cmd.extend(["-charset", encoding])
        _options_cmd = self._options_cmd
        if _options_cmd:
            cmd.extend(["-options", self._options_cmd])

        default_options = " ".join(_java_options)

        
        config_java(options=self.java_options, verbose=verbose)

        
        with tempfile.NamedTemporaryFile(mode="wb", delete=False) as input_file:
            
            if isinstance(input_, str) and encoding:
                input_ = input_.encode(encoding)
            input_file.write(input_)
            input_file.flush()

            cmd.append(input_file.name)

            
            stdout, stderr = java(
                cmd, classpath=self._stanford_jar, stdout=PIPE, stderr=PIPE
            )
            stdout = stdout.decode(encoding)

        os.unlink(input_file.name)

        
        config_java(options=default_options, verbose=False)

        return stdout