natural language toolkit interface to the stanford parser c 20012023 nltk project steven xu xxustudent unimelb edu au url https www nltk org for license information see license txt interface to the stanford parser modeljarpattern rstanfordparserd dmodels jar jar rstanfordparser jar mainclass edu stanford nlp parser lexparser lexicalizedparser usestdin false doublespacedoutput false def init self pathtojarnone pathtomodelsjarnone modelpathedustanfordnlpmodelslexparserenglishpcfg ser gz encodingutf8 verbosefalse javaoptionsmx4g corenlpoptions find the most recent code and model jar stanfordjar max findjariter self jar pathtojar envvarsstanfordparser stanfordcorenlp searchpath urlstanfordurl verboseverbose isregextrue keylambda modelpath os path dirnamemodelpath modeljar max findjariter self modeljarpattern pathtomodelsjar envvarsstanfordmodels stanfordcorenlp searchpath urlstanfordurl verboseverbose isregextrue keylambda modelpath os path dirnamemodelpath self classpath stanfordjar modeljar adding logging jar files to classpath stanforddir os path splitstanfordjar0 self classpath tuplemodeljar findjarswithinpathstanforddir self modelpath modelpath self encoding encoding self corenlpoptions corenlpoptions self javaoptions javaoptions def parsetreesoutputself output res curlines curtrees blank false for line in output splitlinesfalse if line if blank res appenditercurtrees curtrees blank false elif self doublespacedoutput curtrees appendself maketreen joincurlines curlines blank true else res appenditerself maketreen joincurlines curlines else curlines appendline blank false return iterres def parsesentsself sentences verbosefalse cmd self mainclass model self modelpath sentences newline outputformat self outputformat tokenized escaper edu stanford nlp process ptbescapingprocessor return self parsetreesoutput self execute cmd n join joinsentence for sentence in sentences verbose def rawparseself sentence verbosefalse return nextself rawparsesentssentence verbose def rawparsesentsself sentences verbosefalse cmd self mainclass model self modelpath sentences newline outputformat self outputformat return self parsetreesoutput self executecmd n joinsentences verbose def taggedparseself sentence verbosefalse return nextself taggedparsesentssentence verbose def taggedparsesentsself sentences verbosefalse tagseparator cmd self mainclass model self modelpath sentences newline outputformat self outputformat tokenized tagseparator tagseparator tokenizerfactory edu stanford nlp process whitespacetokenizer tokenizermethod newcorelabeltokenizerfactory we don t need to escape slashes as splitting is done on the last instance of the character in the token return self parsetreesoutput self execute cmd n join jointagseparator jointagged for tagged in sentence for sentence in sentences verbose def executeself cmd input verbosefalse encoding self encoding cmd extendencoding encoding if self corenlpoptions cmd extendself corenlpoptions split defaultoptions joinjavaoptions configure java configjavaoptionsself javaoptions verboseverbose windows is incompatible with namedtemporaryfile without passing in deletefalse with tempfile namedtemporaryfilemodewb deletefalse as inputfile write the actual sentences to the temporary input file if isinstanceinput str and encoding input input encodeencoding inputfile writeinput inputfile flush run the tagger and get the output if self usestdin inputfile seek0 stdout stderr java cmd classpathself classpath stdininputfile stdoutpipe stderrpipe else cmd appendinputfile name stdout stderr java cmd classpathself classpath stdoutpipe stderrpipe stdout stdout replacebxc2xa0 b stdout stdout replacebx00xa0 b stdout stdout decodeencoding os unlinkinputfile name return java configurations to their default values configjavaoptionsdefaultoptions verbosefalse return stdout class stanfordparsergenericstanfordparser outputformat penn def initself args kwargs warnings warn the stanfordparser will be deprecatedn please use 03391mnltk parse corenlp corenlpparser0330m instead deprecationwarning stacklevel2 super initargs kwargs def maketreeself result return tree fromstringresult class stanforddependencyparsergenericstanfordparser outputformat conll2007 def initself args kwargs warnings warn the stanforddependencyparser will be deprecatedn please use 03391mnltk parse corenlp corenlpdependencyparser0330m instead deprecationwarning stacklevel2 super initargs kwargs def maketreeself result return dependencygraphresult toprelationlabelroot class stanfordneuraldependencyparsergenericstanfordparser outputformat conll mainclass edu stanford nlp pipeline stanfordcorenlp jar rstanfordcorenlpd d jar modeljarpattern rstanfordcorenlpd dmodels jar usestdin true doublespacedoutput true def initself args kwargs warnings warn the stanfordneuraldependencyparser will be deprecatedn please use 03391mnltk parse corenlp corenlpdependencyparser0330m instead deprecationwarning stacklevel2 super initargs kwargs self corenlpoptions annotators tokenize ssplit pos depparse def taggedparsesentsself sentences verbosefalse raise notimplementederror taggedparsesents is not supported by stanfordneuraldependencyparser use parsesents or rawparsesents instead def maketreeself result return dependencygraphresult toprelationlabelroot natural language toolkit interface to the stanford parser c 2001 2023 nltk project steven xu xxu student unimelb edu au url https www nltk org for license information see license txt interface to the stanford parser find the most recent code and model jar self _classpath stanford_jar model_jar adding logging jar files to classpath use stanfordparser to parse multiple sentences takes multiple sentences as a list where each sentence is a list of words each sentence will be automatically tagged with this stanfordparser instance s tagger if whitespaces exists inside a token then the token will be treated as separate tokens param sentences input sentences to parse type sentences list list str rtype iter iter tree use stanfordparser to parse a sentence takes a sentence as a string before parsing it will be automatically tokenized and tagged by the stanford parser param sentence input sentence to parse type sentence str rtype iter tree use stanfordparser to parse multiple sentences takes multiple sentences as a list of strings each sentence will be automatically tokenized and tagged by the stanford parser param sentences input sentences to parse type sentences list str rtype iter iter tree use stanfordparser to parse a sentence takes a sentence as a list of word tag tuples the sentence must have already been tokenized and tagged param sentence input sentence to parse type sentence list tuple str str rtype iter tree use stanfordparser to parse multiple sentences takes multiple sentences where each sentence is a list of word tag tuples the sentences must have already been tokenized and tagged param sentences input sentences to parse type sentences list list tuple str str rtype iter iter tree we don t need to escape slashes as splitting is done on the last instance of the character in the token configure java windows is incompatible with namedtemporaryfile without passing in delete false write the actual sentences to the temporary input file run the tagger and get the output return java configurations to their default values parser stanfordparser model_path edu stanford nlp models lexparser englishpcfg ser gz doctest skip list parser raw_parse the quick brown fox jumps over the lazy dog doctest normalize_whitespace skip tree root tree np tree np tree dt the tree jj quick tree jj brown tree nn fox tree np tree np tree nns jumps tree pp tree in over tree np tree dt the tree jj lazy tree nn dog sum list dep_graphs for dep_graphs in parser raw_parse_sents the quick brown fox jumps over the lazy dog the quick grey wolf jumps over the lazy fox doctest normalize_whitespace skip tree root tree np tree np tree dt the tree jj quick tree jj brown tree nn fox tree np tree np tree nns jumps tree pp tree in over tree np tree dt the tree jj lazy tree nn dog tree root tree np tree np tree dt the tree jj quick tree jj grey tree nn wolf tree np tree np tree nns jumps tree pp tree in over tree np tree dt the tree jj lazy tree nn fox sum list dep_graphs for dep_graphs in parser parse_sents i m a dog split this is my friends cat the tabby split doctest normalize_whitespace skip tree root tree s tree np tree prp i tree vp tree vbp m tree np tree dt a tree nn dog tree root tree s tree np tree dt this tree vp tree vbz is tree np tree np tree np tree prp my tree nns friends tree pos tree nn cat tree prn tree lrb tree tree np tree dt the tree nn tabby tree rrb sum list dep_graphs for dep_graphs in parser tagged_parse_sents the dt quick jj brown jj fox nn jumped vbd over in the dt lazy jj dog nn doctest normalize_whitespace skip tree root tree s tree np tree dt the tree jj quick tree jj brown tree nn fox tree vp tree vbd jumped tree pp tree in over tree np tree dt the tree jj lazy tree nn dog tree dep_parser stanforddependencyparser model_path edu stanford nlp models lexparser englishpcfg ser gz doctest skip parse tree for parse in dep_parser raw_parse the quick brown fox jumps over the lazy dog doctest normalize_whitespace skip tree jumps tree fox the quick brown tree dog over the lazy list parse triples for parse in dep_parser raw_parse the quick brown fox jumps over the lazy dog doctest normalize_whitespace skip u jumps u vbz u nsubj u fox u nn u fox u nn u det u the u dt u fox u nn u amod u quick u jj u fox u nn u amod u brown u jj u jumps u vbz u nmod u dog u nn u dog u nn u case u over u in u dog u nn u det u the u dt u dog u nn u amod u lazy u jj sum parse tree for parse in dep_graphs for dep_graphs in dep_parser raw_parse_sents the quick brown fox jumps over the lazy dog the quick grey wolf jumps over the lazy fox doctest normalize_whitespace skip tree jumps tree fox the quick brown tree dog over the lazy tree jumps tree wolf the quick grey tree fox over the lazy sum parse tree for parse in dep_graphs for dep_graphs in dep_parser parse_sents i m a dog split this is my friends cat the tabby split doctest normalize_whitespace skip tree dog i m a tree cat this is tree friends my tree tabby the sum list parse triples for parse in dep_graphs for dep_graphs in dep_parser tagged_parse_sents the dt quick jj brown jj fox nn jumped vbd over in the dt lazy jj dog nn doctest normalize_whitespace skip u jumped u vbd u nsubj u fox u nn u fox u nn u det u the u dt u fox u nn u amod u quick u jj u fox u nn u amod u brown u jj u jumped u vbd u nmod u dog u nn u dog u nn u case u over u in u dog u nn u det u the u dt u dog u nn u amod u lazy u jj from nltk parse stanford import stanfordneuraldependencyparser doctest skip dep_parser stanfordneuraldependencyparser java_options mx4g doctest skip parse tree for parse in dep_parser raw_parse the quick brown fox jumps over the lazy dog doctest normalize_whitespace skip tree jumps tree fox the quick brown tree dog over the lazy list parse triples for parse in dep_parser raw_parse the quick brown fox jumps over the lazy dog doctest normalize_whitespace skip u jumps u vbz u nsubj u fox u nn u fox u nn u det u the u dt u fox u nn u amod u quick u jj u fox u nn u amod u brown u jj u jumps u vbz u nmod u dog u nn u dog u nn u case u over u in u dog u nn u det u the u dt u dog u nn u amod u lazy u jj u jumps u vbz u punct u u sum parse tree for parse in dep_graphs for dep_graphs in dep_parser raw_parse_sents the quick brown fox jumps over the lazy dog the quick grey wolf jumps over the lazy fox doctest normalize_whitespace skip tree jumps tree fox the quick brown tree dog over the lazy tree jumps tree wolf the quick grey tree fox over the lazy sum parse tree for parse in dep_graphs for dep_graphs in dep_parser parse_sents i m a dog split this is my friends cat the tabby split doctest normalize_whitespace skip tree dog i m a tree cat this is tree friends my tree tabby lrb the rrb currently unimplemented because the neural dependency parser and the stanfordcorenlp pipeline class doesn t support passing in pre tagged tokens
import os
import tempfile
import warnings
from subprocess import PIPE

from nltk.internals import (
    _java_options,
    config_java,
    find_jar_iter,
    find_jars_within_path,
    java,
)
from nltk.parse.api import ParserI
from nltk.parse.dependencygraph import DependencyGraph
from nltk.tree import Tree

_stanford_url = "https://nlp.stanford.edu/software/lex-parser.shtml"


class GenericStanfordParser(ParserI):
    

    _MODEL_JAR_PATTERN = r"stanford-parser-(\d+)(\.(\d+))+-models\.jar"
    _JAR = r"stanford-parser\.jar"
    _MAIN_CLASS = "edu.stanford.nlp.parser.lexparser.LexicalizedParser"

    _USE_STDIN = False
    _DOUBLE_SPACED_OUTPUT = False

    def __init__(
        self,
        path_to_jar=None,
        path_to_models_jar=None,
        model_path="edu/stanford/nlp/models/lexparser/englishPCFG.ser.gz",
        encoding="utf8",
        verbose=False,
        java_options="-mx4g",
        corenlp_options="",
    ):

        
        stanford_jar = max(
            find_jar_iter(
                self._JAR,
                path_to_jar,
                env_vars=("STANFORD_PARSER", "STANFORD_CORENLP"),
                searchpath=(),
                url=_stanford_url,
                verbose=verbose,
                is_regex=True,
            ),
            key=lambda model_path: os.path.dirname(model_path),
        )

        model_jar = max(
            find_jar_iter(
                self._MODEL_JAR_PATTERN,
                path_to_models_jar,
                env_vars=("STANFORD_MODELS", "STANFORD_CORENLP"),
                searchpath=(),
                url=_stanford_url,
                verbose=verbose,
                is_regex=True,
            ),
            key=lambda model_path: os.path.dirname(model_path),
        )

        

        
        stanford_dir = os.path.split(stanford_jar)[0]
        self._classpath = tuple([model_jar] + find_jars_within_path(stanford_dir))

        self.model_path = model_path
        self._encoding = encoding
        self.corenlp_options = corenlp_options
        self.java_options = java_options

    def _parse_trees_output(self, output_):
        res = []
        cur_lines = []
        cur_trees = []
        blank = False
        for line in output_.splitlines(False):
            if line == "":
                if blank:
                    res.append(iter(cur_trees))
                    cur_trees = []
                    blank = False
                elif self._DOUBLE_SPACED_OUTPUT:
                    cur_trees.append(self._make_tree("\n".join(cur_lines)))
                    cur_lines = []
                    blank = True
                else:
                    res.append(iter([self._make_tree("\n".join(cur_lines))]))
                    cur_lines = []
            else:
                cur_lines.append(line)
                blank = False
        return iter(res)

    def parse_sents(self, sentences, verbose=False):
        
        cmd = [
            self._MAIN_CLASS,
            "-model",
            self.model_path,
            "-sentences",
            "newline",
            "-outputFormat",
            self._OUTPUT_FORMAT,
            "-tokenized",
            "-escaper",
            "edu.stanford.nlp.process.PTBEscapingProcessor",
        ]
        return self._parse_trees_output(
            self._execute(
                cmd, "\n".join(" ".join(sentence) for sentence in sentences), verbose
            )
        )

    def raw_parse(self, sentence, verbose=False):
        
        return next(self.raw_parse_sents([sentence], verbose))

    def raw_parse_sents(self, sentences, verbose=False):
        
        cmd = [
            self._MAIN_CLASS,
            "-model",
            self.model_path,
            "-sentences",
            "newline",
            "-outputFormat",
            self._OUTPUT_FORMAT,
        ]
        return self._parse_trees_output(
            self._execute(cmd, "\n".join(sentences), verbose)
        )

    def tagged_parse(self, sentence, verbose=False):
        
        return next(self.tagged_parse_sents([sentence], verbose))

    def tagged_parse_sents(self, sentences, verbose=False):
        
        tag_separator = "/"
        cmd = [
            self._MAIN_CLASS,
            "-model",
            self.model_path,
            "-sentences",
            "newline",
            "-outputFormat",
            self._OUTPUT_FORMAT,
            "-tokenized",
            "-tagSeparator",
            tag_separator,
            "-tokenizerFactory",
            "edu.stanford.nlp.process.WhitespaceTokenizer",
            "-tokenizerMethod",
            "newCoreLabelTokenizerFactory",
        ]
        
        return self._parse_trees_output(
            self._execute(
                cmd,
                "\n".join(
                    " ".join(tag_separator.join(tagged) for tagged in sentence)
                    for sentence in sentences
                ),
                verbose,
            )
        )

    def _execute(self, cmd, input_, verbose=False):
        encoding = self._encoding
        cmd.extend(["-encoding", encoding])
        if self.corenlp_options:
            cmd.extend(self.corenlp_options.split())

        default_options = " ".join(_java_options)

        
        config_java(options=self.java_options, verbose=verbose)

        
        with tempfile.NamedTemporaryFile(mode="wb", delete=False) as input_file:
            
            if isinstance(input_, str) and encoding:
                input_ = input_.encode(encoding)
            input_file.write(input_)
            input_file.flush()

            
            if self._USE_STDIN:
                input_file.seek(0)
                stdout, stderr = java(
                    cmd,
                    classpath=self._classpath,
                    stdin=input_file,
                    stdout=PIPE,
                    stderr=PIPE,
                )
            else:
                cmd.append(input_file.name)
                stdout, stderr = java(
                    cmd, classpath=self._classpath, stdout=PIPE, stderr=PIPE
                )

            stdout = stdout.replace(b"\xc2\xa0", b" ")
            stdout = stdout.replace(b"\x00\xa0", b" ")
            stdout = stdout.decode(encoding)

        os.unlink(input_file.name)

        
        config_java(options=default_options, verbose=False)

        return stdout


class StanfordParser(GenericStanfordParser):
    

    _OUTPUT_FORMAT = "penn"

    def __init__(self, *args, **kwargs):
        warnings.warn(
            "The StanfordParser will be deprecated\n"
            "Please use \033[91mnltk.parse.corenlp.CoreNLPParser\033[0m instead.",
            DeprecationWarning,
            stacklevel=2,
        )

        super().__init__(*args, **kwargs)

    def _make_tree(self, result):
        return Tree.fromstring(result)


class StanfordDependencyParser(GenericStanfordParser):

    

    _OUTPUT_FORMAT = "conll2007"

    def __init__(self, *args, **kwargs):
        warnings.warn(
            "The StanfordDependencyParser will be deprecated\n"
            "Please use \033[91mnltk.parse.corenlp.CoreNLPDependencyParser\033[0m instead.",
            DeprecationWarning,
            stacklevel=2,
        )

        super().__init__(*args, **kwargs)

    def _make_tree(self, result):
        return DependencyGraph(result, top_relation_label="root")


class StanfordNeuralDependencyParser(GenericStanfordParser):
    

    _OUTPUT_FORMAT = "conll"
    _MAIN_CLASS = "edu.stanford.nlp.pipeline.StanfordCoreNLP"
    _JAR = r"stanford-corenlp-(\d+)(\.(\d+))+\.jar"
    _MODEL_JAR_PATTERN = r"stanford-corenlp-(\d+)(\.(\d+))+-models\.jar"
    _USE_STDIN = True
    _DOUBLE_SPACED_OUTPUT = True

    def __init__(self, *args, **kwargs):
        warnings.warn(
            "The StanfordNeuralDependencyParser will be deprecated\n"
            "Please use \033[91mnltk.parse.corenlp.CoreNLPDependencyParser\033[0m instead.",
            DeprecationWarning,
            stacklevel=2,
        )

        super().__init__(*args, **kwargs)
        self.corenlp_options += "-annotators tokenize,ssplit,pos,depparse"

    def tagged_parse_sents(self, sentences, verbose=False):
        
        raise NotImplementedError(
            "tagged_parse[_sents] is not supported by "
            "StanfordNeuralDependencyParser; use "
            "parse[_sents] or raw_parse[_sents] instead."
        )

    def _make_tree(self, result):
        return DependencyGraph(result, top_relation_label="ROOT")