usrbinenv python natural language toolkit interface to the stanford segmenter for chinese and arabic c 20012023 nltk project 52nlp 52nlpcngmail com casper lehmannstrm casperlehmanngmail com alex constantin alexkeyworder ch url https www nltk org for license information see license txt interface to the stanford segmenter if stanfordsegmenter version is older than 20161031 then pathtoslf4j should be provieded for example seg stanfordsegmenterpathtoslf4j yourpathslf4japi jar from nltk tokenize stanfordsegmenter import stanfordsegmenter seg stanfordsegmenter doctest skip seg defaultconfig zh doctest skip sent u printseg segmentsent doctest skip u8fd9 u662f u65afu5766u798f u4e2du6587 u5206u8bcdu5668 u6d4bu8bd5 blankline seg defaultconfig ar doctest skip sent u printseg segmentsent split doctest skip u0647u0630u0627 u0647u0648 u062au0635u0646u064au0641 u0633u062au0627u0646u0641u0648u0631u062f u0627u0644u0639u0631u0628u064a u0644 u0627u0644u0643u0644u0645u0627u062a blankline raise deprecation warning this is passed to java as the cp option the old version of segmenter needs slf4j the new version of stanfordsegmenter20161031 doesn t need slf4j attempt to initialize stanford word segmenter for the specified language using the stanfordsegmenter and stanfordmodels environment variables init for chinesespecific files cmd self javaclass loadclassifier self model keepallwhitespaces self keepwhitespaces textfile inputfilepath if self sihancorporadict is not none cmd extend serdictionary self dict sighancorporadict self sihancorporadict sighanpostprocessing self sihanpostprocessing stdout self executecmd return stdout def segmentself tokens return self segmentsentstokens def segmentsentsself sentences create a temporary input file write the actural sentences to the temporary input file delete the temporary file configure java return java configurations to their default values usr bin env python natural language toolkit interface to the stanford segmenter for chinese and arabic c 2001 2023 nltk project 52nlp 52nlpcn gmail com casper lehmann strøm casperlehmann gmail com alex constantin alex keyworder ch url https www nltk org for license information see license txt interface to the stanford segmenter if stanford segmenter version is older than 2016 10 31 then path_to_slf4j should be provieded for example seg stanfordsegmenter path_to_slf4j your_path slf4j api jar from nltk tokenize stanford_segmenter import stanfordsegmenter seg stanfordsegmenter doctest skip seg default_config zh doctest skip sent u 这是斯坦福中文分词器测试 print seg segment sent doctest skip u8fd9 u662f u65af u5766 u798f u4e2d u6587 u5206 u8bcd u5668 u6d4b u8bd5 blankline seg default_config ar doctest skip sent u هذا هو تصنيف ستانفورد العربي للكلمات print seg segment sent split doctest skip u0647 u0630 u0627 u0647 u0648 u062a u0635 u0646 u064a u0641 u0633 u062a u0627 u0646 u0641 u0648 u0631 u062f u0627 u0644 u0639 u0631 u0628 u064a u0644 u0627 u0644 u0643 u0644 u0645 u0627 u062a blankline raise deprecation warning this is passed to java as the cp option the old version of segmenter needs slf4j the new version of stanford segmenter 2016 10 31 doesn t need slf4j attempt to initialize stanford word segmenter for the specified language using the stanford_segmenter and stanford_models environment variables init for chinese specific files create a temporary input file write the actural sentences to the temporary input file delete the temporary file configure java return java configurations to their default values
import json
import os
import tempfile
import warnings
from subprocess import PIPE

from nltk.internals import (
    _java_options,
    config_java,
    find_dir,
    find_file,
    find_jar,
    java,
)
from nltk.tokenize.api import TokenizerI

_stanford_url = "https://nlp.stanford.edu/software"


class StanfordSegmenter(TokenizerI):
    

    _JAR = "stanford-segmenter.jar"

    def __init__(
        self,
        path_to_jar=None,
        path_to_slf4j=None,
        java_class=None,
        path_to_model=None,
        path_to_dict=None,
        path_to_sihan_corpora_dict=None,
        sihan_post_processing="false",
        keep_whitespaces="false",
        encoding="UTF-8",
        options=None,
        verbose=False,
        java_options="-mx2g",
    ):
        
        warnings.simplefilter("always", DeprecationWarning)
        warnings.warn(
            str(
                "\nThe StanfordTokenizer will "
                "be deprecated in version 3.2.5.\n"
                "Please use \033[91mnltk.parse.corenlp.CoreNLPTokenizer\033[0m instead.'"
            ),
            DeprecationWarning,
            stacklevel=2,
        )
        warnings.simplefilter("ignore", DeprecationWarning)

        stanford_segmenter = find_jar(
            self._JAR,
            path_to_jar,
            env_vars=("STANFORD_SEGMENTER",),
            searchpath=(),
            url=_stanford_url,
            verbose=verbose,
        )
        if path_to_slf4j is not None:
            slf4j = find_jar(
                "slf4j-api.jar",
                path_to_slf4j,
                env_vars=("SLF4J", "STANFORD_SEGMENTER"),
                searchpath=(),
                url=_stanford_url,
                verbose=verbose,
            )
        else:
            slf4j = None

        
        
        self._stanford_jar = os.pathsep.join(
            _ for _ in [stanford_segmenter, slf4j] if _ is not None
        )

        self._java_class = java_class
        self._model = path_to_model
        self._sihan_corpora_dict = path_to_sihan_corpora_dict
        self._sihan_post_processing = sihan_post_processing
        self._keep_whitespaces = keep_whitespaces
        self._dict = path_to_dict

        self._encoding = encoding
        self.java_options = java_options
        options = {} if options is None else options
        self._options_cmd = ",".join(
            f"{key}={json.dumps(val)}" for key, val in options.items()
        )

    def default_config(self, lang):
        

        search_path = ()
        if os.environ.get("STANFORD_SEGMENTER"):
            search_path = {os.path.join(os.environ.get("STANFORD_SEGMENTER"), "data")}

        
        self._dict = None
        self._sihan_corpora_dict = None
        self._sihan_post_processing = "false"

        if lang == "ar":
            self._java_class = (
                "edu.stanford.nlp.international.arabic.process.ArabicSegmenter"
            )
            model = "arabic-segmenter-atb+bn+arztrain.ser.gz"

        elif lang == "zh":
            self._java_class = "edu.stanford.nlp.ie.crf.CRFClassifier"
            model = "pku.gz"
            self._sihan_post_processing = "true"

            path_to_dict = "dict-chris6.ser.gz"
            try:
                self._dict = find_file(
                    path_to_dict,
                    searchpath=search_path,
                    url=_stanford_url,
                    verbose=False,
                    env_vars=("STANFORD_MODELS",),
                )
            except LookupError as e:
                raise LookupError(
                    "Could not find '%s' (tried using env. "
                    "variables STANFORD_MODELS and <STANFORD_SEGMENTER>/data/)"
                    % path_to_dict
                ) from e

            sihan_dir = "./data/"
            try:
                path_to_sihan_dir = find_dir(
                    sihan_dir,
                    url=_stanford_url,
                    verbose=False,
                    env_vars=("STANFORD_SEGMENTER",),
                )
                self._sihan_corpora_dict = os.path.join(path_to_sihan_dir, sihan_dir)
            except LookupError as e:
                raise LookupError(
                    "Could not find '%s' (tried using the "
                    "STANFORD_SEGMENTER environment variable)" % sihan_dir
                ) from e
        else:
            raise LookupError(f"Unsupported language {lang}")

        try:
            self._model = find_file(
                model,
                searchpath=search_path,
                url=_stanford_url,
                verbose=False,
                env_vars=("STANFORD_MODELS", "STANFORD_SEGMENTER"),
            )
        except LookupError as e:
            raise LookupError(
                "Could not find '%s' (tried using env. "
                "variables STANFORD_MODELS and <STANFORD_SEGMENTER>/data/)" % model
            ) from e

    def tokenize(self, s):
        super().tokenize(s)

    def segment_file(self, input_file_path):
        
        cmd = [
            self._java_class,
            "-loadClassifier",
            self._model,
            "-keepAllWhitespaces",
            self._keep_whitespaces,
            "-textFile",
            input_file_path,
        ]
        if self._sihan_corpora_dict is not None:
            cmd.extend(
                [
                    "-serDictionary",
                    self._dict,
                    "-sighanCorporaDict",
                    self._sihan_corpora_dict,
                    "-sighanPostProcessing",
                    self._sihan_post_processing,
                ]
            )

        stdout = self._execute(cmd)

        return stdout

    def segment(self, tokens):
        return self.segment_sents([tokens])

    def segment_sents(self, sentences):
        
        encoding = self._encoding
        
        _input_fh, self._input_file_path = tempfile.mkstemp(text=True)

        
        _input_fh = os.fdopen(_input_fh, "wb")
        _input = "\n".join(" ".join(x) for x in sentences)
        if isinstance(_input, str) and encoding:
            _input = _input.encode(encoding)
        _input_fh.write(_input)
        _input_fh.close()

        cmd = [
            self._java_class,
            "-loadClassifier",
            self._model,
            "-keepAllWhitespaces",
            self._keep_whitespaces,
            "-textFile",
            self._input_file_path,
        ]
        if self._sihan_corpora_dict is not None:
            cmd.extend(
                [
                    "-serDictionary",
                    self._dict,
                    "-sighanCorporaDict",
                    self._sihan_corpora_dict,
                    "-sighanPostProcessing",
                    self._sihan_post_processing,
                ]
            )

        stdout = self._execute(cmd)

        
        os.unlink(self._input_file_path)

        return stdout

    def _execute(self, cmd, verbose=False):
        encoding = self._encoding
        cmd.extend(["-inputEncoding", encoding])
        _options_cmd = self._options_cmd
        if _options_cmd:
            cmd.extend(["-options", self._options_cmd])

        default_options = " ".join(_java_options)

        
        config_java(options=self.java_options, verbose=verbose)

        stdout, _stderr = java(
            cmd, classpath=self._stanford_jar, stdout=PIPE, stderr=PIPE
        )
        stdout = stdout.decode(encoding)

        
        config_java(options=default_options, verbose=False)

        return stdout