natural language toolkit internal utility functions c 20012023 nltk project steven bird stevenbird1gmail com edward loper edlopergmail com nitin madnani nmadnaniets org url https www nltk org for license information see license txt java via commandline xx add classpath option to configjava configure nltk s java interface by letting nltk know where it can find the java binary and what extra options if any should be passed to java when it is run param bin the full path to the java binary if not specified then nltk will search the system for a java binary and if one is not found it will raise a lookuperror exception type bin str param options a list of options that should be passed to the java binary when it is called a common value is xmx512m which tells java binary to increase the maximum heap size to 512 megabytes if no options are specified then do not modify the options list type options liststr execute the given java command by opening a subprocess that calls java if java has not yet been configured it will be configured by calling configjava with no arguments param cmd the java command that should be called formatted as a list of strings typically the first string will be the name of the java class and the remaining strings will be arguments for that java class type cmd liststr param classpath a separated list of directories jar archives and zip archives to search for class files type classpath str param stdin specify the executed program s standard input file handles respectively valid values are subprocess pipe an existing file descriptor a positive integer an existing file object pipe stdout devnull and none subprocess pipe indicates that a new pipe to the child should be created with none no redirection will occur the child s file handles will be inherited from the parent additionally stderr can be subprocess stdout which indicates that the stderr data from the applications should be captured into the same file handle as for stdout param stdout specify the executed program s standard output file handle see stdin for valid values param stderr specify the executed program s standard error file handle see stdin for valid values param blocking if false then return immediately after spawning the subprocess in this case the return value is the popen object and not a stdout stderr tuple return if blockingtrue then return a tuple stdout stderr containing the stdout and stderr outputs generated by the java command if the stdout and stderr parameters were set to subprocess pipe or none otherwise if blockingfalse then return a subprocess popen object raise oserror if the java command returns a nonzero return code make sure we know where a java binary is set up the classpath construct the full command string call java via a subprocess check the return code parsing exception raised by read functions when they fail param position the index in the input string where an error occurred param expected what was expected when an error occurred if a python string literal begins at the specified position in the given string then return a tuple val endposition containing the value of the string literal and the position where it ends otherwise raise a readerror param s a string that will be checked to see if within which a python string literal exists type s str param startposition the specified beginning position of the string s to begin regex matching type startposition int return a tuple containing the matched string literal evaluated as a string and the end position of the string literal rtype tuplestr int raise readerror if the stringstartre regex doesn t return a match in s at startposition i e open quote if the stringendre regex doesn t return a match in s at the end of the first match i e close quote raise valueerror if an invalid string i e contains an invalid escape sequence is passed into the eval example from nltk internals import readstr readstr hello world 0 hello 7 read the open quote and any modifiers find the close quote process it using eval strings with invalid escape sequences might raise valueerror if an integer begins at the specified position in the given string then return a tuple val endposition containing the value of the integer and the position where it ends otherwise raise a readerror param s a string that will be checked to see if within which a python integer exists type s str param startposition the specified beginning position of the string s to begin regex matching type startposition int return a tuple containing the matched integer casted to an int and the end position of the int in s rtype tupleint int raise readerror if the readintre regex doesn t return a match in s at startposition example from nltk internals import readint readint 42 is the answer 0 42 2 if an integer or float begins at the specified position in the given string then return a tuple val endposition containing the value of the number and the position where it ends otherwise raise a readerror param s a string that will be checked to see if within which a python number exists type s str param startposition the specified beginning position of the string s to begin regex matching type startposition int return a tuple containing the matched number casted to a float and the end position of the number in s rtype tuplefloat int raise readerror if the readnumbervalue regex doesn t return a match in s at startposition example from nltk internals import readnumber readnumber pi is 3 14159 6 3 14159 13 check if a method has been overridden return true if method overrides some method with the same name in a base class this is typically used when defining abstract base classes or interfaces to allow subclasses to define either of two related methods class eateri subclass must define eat or batcheat def eatself food if overriddenself batcheat return self batcheatfood0 else raise notimplementederror def batcheatself foods return self eatfood for food in foods type method instance method return the method resolution order for cls i e a list containing cls and all its base classes in the order in which they would be checked by getattr for newstyle classes this is just cls mro for classic classes this can be obtained by a depthfirst lefttoright traversal of bases deprecation decorator base class xx dedent msg first if it comes from a docstring add an epytext field to a given object s docstring indent if we already have a docstring then add a blank line to separate it from the new field and check its indentation if obj doc obj doc obj doc rstrip nn indents re findallr n s obj doc expandtabs if indents indent minindents if we don t have a docstring add an empty one else obj doc obj doc textwrap fill ffield message initialindentindent subsequentindentindent def deprecatedmessage def decoratorfunc msg ffunction func name has been deprecated message msg n textwrap fillmsg initialindent subsequentindent def newfuncargs kwargs warnings warnmsg categorydeprecationwarning stacklevel2 return funcargs kwargs copy the old function s name docstring dict newfunc dict updatefunc dict newfunc name func name newfunc doc func doc newfunc deprecated true add a deprecated field to the docstring addepytextfieldnewfunc deprecated message return newfunc return decorator class deprecated def newcls args kwargs figure out which class is the deprecated one depcls none for base in mrocls if deprecated in base bases depcls base break assert depcls unable to determine which base is deprecated construct an appropriate warning doc depcls doc or strip if there s a deprecated field strip off the field marker doc re subrasdeprecated r doc strip off any indentation doc re subr ms doc construct a name string name class s depcls name if cls depcls name base class for s cls name put it all together msg fname has been deprecated doc wrap it msg n textwrap fillmsg initialindent subsequentindent warnings warnmsg categorydeprecationwarning stacklevel2 do the actual work of new return object newcls counter for unique naming class counter def initself initialvalue0 self value initialvalue def getself self value 1 return self value search for filesbinaries def findfileiter filename envvars searchpath filenamesnone urlnone verbosefalse findingdirfalse filenames filename filenames or assert isinstancefilename str assert not isinstancefilenames str assert not isinstancesearchpath str if isinstanceenvvars str envvars envvars split yielded false file exists no magic for alternative in filenames pathtofile os path joinfilename alternative if os path isfilepathtofile if verbose printffound filename pathtofile yielded true yield pathtofile check the bare alternatives if os path isfilealternative if verbose printffound filename alternative yielded true yield alternative check if the alternative is inside a file directory pathtofile os path joinfilename file alternative if os path isfilepathtofile if verbose printffound filename pathtofile yielded true yield pathtofile check environment variables for envvar in envvars if envvar in os environ if findingdir this is to file a directory instead of file yielded true yield os environenvvar for envdir in os environenvvar splitos pathsep check if the environment variable contains a direct path to the bin if os path isfileenvdir if verbose printffound filename envdir yielded true yield envdir check if the possible bin names exist inside the environment variable directories for alternative in filenames pathtofile os path joinenvdir alternative if os path isfilepathtofile if verbose printffound filename pathtofile yielded true yield pathtofile check if the alternative is inside a file directory pathtofile os path joinenvdir file alternative check if the alternative is inside a bin directory pathtofile os path joinenvdir bin alternative if os path isfilepathtofile if verbose printffound filename pathtofile yielded true yield pathtofile check the path list for directory in searchpath for alternative in filenames pathtofile os path joindirectory alternative if os path isfilepathtofile yielded true yield pathtofile if we re on a posix system then try using the which command to find the file if os name posix for alternative in filenames try p subprocess popen which alternative stdoutsubprocess pipe stderrsubprocess pipe stdout stderr p communicate path decodestdoutdatastdout strip if path endswithalternative and os path existspath if verbose printffound filename path yielded true yield path except keyboardinterrupt systemexit oserror raise finally pass if not yielded msg nltk was unable to find the s file nuse software specific configuration parameters filename if envvars msg or set the s environment variable envvars0 msg if searchpath msg nn searched in msg joinn s d for d in searchpath if url msg fnn for more information on filename see n url div 75 raise lookuperrorfnndivnmsgndiv def findfile filename envvars searchpath filenamesnone urlnone verbosefalse return next findfileiterfilename envvars searchpath filenames url verbose def finddir filename envvars searchpath filenamesnone urlnone verbosefalse return next findfileiter filename envvars searchpath filenames url verbose findingdirtrue def findbinaryiter name pathtobinnone envvars searchpath binarynamesnone urlnone verbosefalse yield from findfileiter pathtobin or name envvars searchpath binarynames url verbose def findbinary name pathtobinnone envvars searchpath binarynamesnone urlnone verbosefalse return next findbinaryiter name pathtobin envvars searchpath binarynames url verbose def findjariter namepattern pathtojarnone envvars searchpath urlnone verbosefalse isregexfalse assert isinstancenamepattern str assert not isinstancesearchpath str if isinstanceenvvars str envvars envvars split yielded false make sure we check the classpath first envvars classpath listenvvars if an explicit location was given then check it and yield it if it s present otherwise complain if pathtojar is not none if os path isfilepathtojar yielded true yield pathtojar else raise lookuperror fcould not find namepattern jar file at pathtojar check environment variables for envvar in envvars if envvar in os environ if envvar classpath classpath os environclasspath for cp in classpath splitos path pathsep cp os path expandusercp if os path isfilecp filename os path basenamecp if isregex and re matchnamepattern filename or not isregex and filename namepattern if verbose printffound namepattern cp yielded true yield cp the case where user put directory containing the jar file in the classpath if os path isdircp if not isregex if os path isfileos path joincp namepattern if verbose printffound namepattern cp yielded true yield os path joincp namepattern else look for file using regular expression for filename in os listdircp if re matchnamepattern filename if verbose print found s s namepattern os path joincp filename yielded true yield os path joincp filename else jarenv os path expanduseros environenvvar jariter os path joinjarenv pathtojar for pathtojar in os listdirjarenv if os path isdirjarenv else jarenv for pathtojar in jariter if os path isfilepathtojar filename os path basenamepathtojar if isregex and re matchnamepattern filename or not isregex and filename namepattern if verbose printffound namepattern pathtojar yielded true yield pathtojar check the path list for directory in searchpath if isregex for filename in os listdirdirectory pathtojar os path joindirectory filename if os path isfilepathtojar if re matchnamepattern filename if verbose printffound filename pathtojar yielded true yield pathtojar else pathtojar os path joindirectory namepattern if os path isfilepathtojar if verbose printffound namepattern pathtojar yielded true yield pathtojar if not yielded if nothing was found raise an error msg nltk was unable to find s namepattern if envvars msg set the s environment variable envvars0 msg textwrap fillmsg initialindent subsequentindent if searchpath msg nn searched in msg joinn s d for d in searchpath if url msg nn for more information on see n format namepattern url div 75 raise lookuperrorfnndivnmsgndiv def findjar namepattern pathtojarnone envvars searchpath urlnone verbosefalse isregexfalse return next findjariter namepattern pathtojar envvars searchpath url verbose isregex def findjarswithinpathpathtojars return os path joinroot filename for root dirnames filenames in os walkpathtojars for filename in fnmatch filterfilenames jar def decodestdoutdatastdoutdata import stdlib module when python is run from within the nltk directory tree the current directory is included at the beginning of the search path unfortunately that means that modules within nltk can sometimes shadow standard library modules as an example the stdlib inspect module will attempt to import the stdlib tokenize module but will instead end up importing nltk s tokenize module instead causing the import to fail wrapper for elementtree elements a wrapper around elementtree element objects whose main purpose is to provide nicer repr and str methods in addition any of the wrapped element s methods that return other element objects are overridden to wrap those values before returning them this makes elements more convenient to work with in interactive sessions and doctests at the expense of some efficiency prevent doublewrapping create and return a wrapper around a given element object if etree is an elementwrapper then etree is returned asis if isinstanceetree str etree elementtree fromstringetree self dictetree etree def unwrapself return self etree string representation def reprself s elementtree tostringself etree encodingutf8 decodeutf8 if lens 60 e s rfind if lens e 30 e 20 s fs 30 se return element r s def strself return elementtree tostringself etree encodingutf8 decodeutf8 rstrip element interface delegation passthrough def getattrself attrib return getattrself etree attrib def setattrself attr value return setattrself etree attr value def delattrself attr return delattrself etree attr def setitemself index element self etreeindex element def delitemself index del self etreeindex def setsliceself start stop elements self etreestart stop elements def delsliceself start stop del self etreestart stop def lenself return lenself etree element interface delegation wrap result def getitemself index return elementwrapperself etreeindex def getsliceself start stop return elementwrapperelt for elt in self etreestart stop def getchildrenself return elementwrapperelt for elt in self etree def getiteratorself tagnone return elementwrapperelt for elt in self etree getiteratortag def makeelementself tag attrib return elementwrapperself etree makeelementtag attrib def findself path elt self etree findpath if elt is none return elt else return elementwrapperelt def findallself path return elementwrapperelt for elt in self etree findallpath helper for handling slicing def sliceboundssequence sliceobj allowstepfalse start stop sliceobj start sliceobj stop if allowstep is true then include the step in our return value tuple if allowstep step sliceobj step if step is none step 1 use a recursive call without allowstep to find the slice bounds if step is negative then the roles of start and stop in terms of default values etc are swapped if step 0 start stop sliceboundssequence slicestop start else start stop sliceboundssequence slicestart stop return start stop step otherwise make sure that no nondefault step value is used elif sliceobj step not in none 1 raise valueerror slices with steps are not supported by s sequence class name supply default offsets if start is none start 0 if stop is none stop lensequence handle negative indices if start 0 start max0 lensequence start if stop 0 stop max0 lensequence stop make sure stop doesn t go past the end of the list note that we avoid calculating lensequence if possible because for lazy sequences calculating the length of a sequence can be expensive if stop 0 try sequencestop 1 except indexerror stop lensequence make sure start isn t past stop start minstart stop that s all folks return start stop permission checking def iswritablepath ensure that it exists if not os path existspath return false if we re on a posix system check its permissions if hasattros getuid statdata os statpath perm stat simodestatdata stmode is it worldwritable if perm 0o002 return true do we own it elif statdata stuid os getuid and perm 0o200 return true are we in a group that can write to it elif statdata stgid in os getgid os getgroups and perm 0o020 return true otherwise we can t write to it else return false otherwise we ll assume it s writable xx should we do other checks on other platforms return true nltk error reporting def raiseunorderabletypesordering a b raise typeerror unorderable types s s s typea name ordering typeb name natural language toolkit internal utility functions c 2001 2023 nltk project steven bird stevenbird1 gmail com edward loper edloper gmail com nitin madnani nmadnani ets org url https www nltk org for license information see license txt java via command line xx add classpath option to config_java configure nltk s java interface by letting nltk know where it can find the java binary and what extra options if any should be passed to java when it is run param bin the full path to the java binary if not specified then nltk will search the system for a java binary and if one is not found it will raise a lookuperror exception type bin str param options a list of options that should be passed to the java binary when it is called a common value is xmx512m which tells java binary to increase the maximum heap size to 512 megabytes if no options are specified then do not modify the options list type options list str execute the given java command by opening a subprocess that calls java if java has not yet been configured it will be configured by calling config_java with no arguments param cmd the java command that should be called formatted as a list of strings typically the first string will be the name of the java class and the remaining strings will be arguments for that java class type cmd list str param classpath a separated list of directories jar archives and zip archives to search for class files type classpath str param stdin specify the executed program s standard input file handles respectively valid values are subprocess pipe an existing file descriptor a positive integer an existing file object pipe stdout devnull and none subprocess pipe indicates that a new pipe to the child should be created with none no redirection will occur the child s file handles will be inherited from the parent additionally stderr can be subprocess stdout which indicates that the stderr data from the applications should be captured into the same file handle as for stdout param stdout specify the executed program s standard output file handle see stdin for valid values param stderr specify the executed program s standard error file handle see stdin for valid values param blocking if false then return immediately after spawning the subprocess in this case the return value is the popen object and not a stdout stderr tuple return if blocking true then return a tuple stdout stderr containing the stdout and stderr outputs generated by the java command if the stdout and stderr parameters were set to subprocess pipe or none otherwise if blocking false then return a subprocess popen object raise oserror if the java command returns a nonzero return code make sure we know where a java binary is set up the classpath construct the full command string call java via a subprocess check the return code parsing exception raised by read_ functions when they fail param position the index in the input string where an error occurred param expected what was expected when an error occurred if a python string literal begins at the specified position in the given string then return a tuple val end_position containing the value of the string literal and the position where it ends otherwise raise a readerror param s a string that will be checked to see if within which a python string literal exists type s str param start_position the specified beginning position of the string s to begin regex matching type start_position int return a tuple containing the matched string literal evaluated as a string and the end position of the string literal rtype tuple str int raise readerror if the _string_start_re regex doesn t return a match in s at start_position i e open quote if the _string_end_re regex doesn t return a match in s at the end of the first match i e close quote raise valueerror if an invalid string i e contains an invalid escape sequence is passed into the eval example from nltk internals import read_str read_str hello world 0 hello 7 read the open quote and any modifiers find the close quote process it using eval strings with invalid escape sequences might raise valueerror if an integer begins at the specified position in the given string then return a tuple val end_position containing the value of the integer and the position where it ends otherwise raise a readerror param s a string that will be checked to see if within which a python integer exists type s str param start_position the specified beginning position of the string s to begin regex matching type start_position int return a tuple containing the matched integer casted to an int and the end position of the int in s rtype tuple int int raise readerror if the _read_int_re regex doesn t return a match in s at start_position example from nltk internals import read_int read_int 42 is the answer 0 42 2 if an integer or float begins at the specified position in the given string then return a tuple val end_position containing the value of the number and the position where it ends otherwise raise a readerror param s a string that will be checked to see if within which a python number exists type s str param start_position the specified beginning position of the string s to begin regex matching type start_position int return a tuple containing the matched number casted to a float and the end position of the number in s rtype tuple float int raise readerror if the _read_number_value regex doesn t return a match in s at start_position example from nltk internals import read_number read_number pi is 3 14159 6 3 14159 13 check if a method has been overridden return true if method overrides some method with the same name in a base class this is typically used when defining abstract base classes or interfaces to allow subclasses to define either of two related methods class eateri subclass must define eat or batch_eat def eat self food if overridden self batch_eat return self batch_eat food 0 else raise notimplementederror def batch_eat self foods return self eat food for food in foods type method instance method return the method resolution order for cls i e a list containing cls and all its base classes in the order in which they would be checked by getattr for new style classes this is just cls __mro__ for classic classes this can be obtained by a depth first left to right traversal of __bases__ deprecation decorator base class xx dedent msg first if it comes from a docstring add an epytext field to a given object s docstring if we already have a docstring then add a blank line to separate it from the new field and check its indentation if we don t have a docstring add an empty one a decorator used to mark functions as deprecated this will cause a warning to be printed the when the function is used usage from nltk internals import deprecated deprecated use foo instead def bar x print x 10 copy the old function s name docstring dict add a deprecated field to the docstring a base class used to mark deprecated classes a typical usage is to alert users that the name of a class has changed from nltk internals import deprecated class newclassname pass all logic goes here class oldclassname deprecated newclassname use newclassname instead the docstring of the deprecated class will be used in the deprecation warning message figure out which class is the deprecated one construct an appropriate warning if there s a deprecated field strip off the field marker strip off any indentation construct a name string put it all together wrap it do the actual work of __new__ counter for unique naming a counter that auto increments each time its value is read search for files binaries search for a file to be used by nltk param filename the name or path of the file param env_vars a list of environment variable names to check param file_names a list of alternative file names to check param searchpath list of directories to search param url url presented to user for download help param verbose whether or not to print path when a file is found file exists no magic check the bare alternatives check if the alternative is inside a file directory check environment variables this is to file a directory instead of file check if the environment variable contains a direct path to the bin check if the possible bin names exist inside the environment variable directories check if the alternative is inside a file directory path_to_file os path join env_dir file alternative check if the alternative is inside a bin directory check the path list if we re on a posix system then try using the which command to find the file search for a file to be used by nltk param name the name or path of the file param path_to_bin the user supplied binary location deprecated param env_vars a list of environment variable names to check param file_names a list of alternative file names to check param searchpath list of directories to search param url url presented to user for download help param verbose whether or not to print path when a file is found search for a jar that is used by nltk param name_pattern the name of the jar file param path_to_jar the user supplied jar location or none param env_vars a list of environment variable names to check in addition to the classpath variable which is checked by default param searchpath list of directories to search param is_regex whether name is a regular expression make sure we check the classpath first if an explicit location was given then check it and yield it if it s present otherwise complain check environment variables the case where user put directory containing the jar file in the classpath look for file using regular expression check the path list if nothing was found raise an error convert data read from stdout stderr to unicode import stdlib module when python is run from within the nltk directory tree the current directory is included at the beginning of the search path unfortunately that means that modules within nltk can sometimes shadow standard library modules as an example the stdlib inspect module will attempt to import the stdlib tokenize module but will instead end up importing nltk s tokenize module instead causing the import to fail wrapper for elementtree elements a wrapper around elementtree element objects whose main purpose is to provide nicer __repr__ and __str__ methods in addition any of the wrapped element s methods that return other element objects are overridden to wrap those values before returning them this makes elements more convenient to work with in interactive sessions and doctests at the expense of some efficiency prevent double wrapping create and return a wrapper around a given element object if etree is an elementwrapper then etree is returned as is initialize a new element wrapper for etree if etree is a string then it will be converted to an element object using elementtree fromstring first elementwrapper test test element xml version 1 0 encoding utf8 n test return the element object wrapped by this wrapper string representation return the result of applying elementtree tostring to the wrapped element object element interface delegation pass through element interface delegation wrap result helper for handling slicing given a slice return the corresponding start stop bounds taking into account none indices and negative indices the following guarantees are made for the returned start and stop values 0 start len sequence 0 stop len sequence start stop raise valueerror if slice_obj step is not none param allow_step if true then the slice object may have a non none step if it does then return a tuple start stop step if allow_step is true then include the step in our return value tuple use a recursive call without allow_step to find the slice bounds if step is negative then the roles of start and stop in terms of default values etc are swapped otherwise make sure that no non default step value is used supply default offsets handle negative indices make sure stop doesn t go past the end of the list note that we avoid calculating len sequence if possible because for lazy sequences calculating the length of a sequence can be expensive make sure start isn t past stop that s all folks permission checking ensure that it exists if we re on a posix system check its permissions is it world writable do we own it are we in a group that can write to it otherwise we can t write to it otherwise we ll assume it s writable xx should we do other checks on other platforms nltk error reporting
import fnmatch
import locale
import os
import re
import stat
import subprocess
import sys
import textwrap
import types
import warnings
from xml.etree import ElementTree





_java_bin = None
_java_options = []

def config_java(bin=None, options=None, verbose=False):
    
    global _java_bin, _java_options
    _java_bin = find_binary(
        "java",
        bin,
        env_vars=["JAVAHOME", "JAVA_HOME"],
        verbose=verbose,
        binary_names=["java.exe"],
    )

    if options is not None:
        if isinstance(options, str):
            options = options.split()
        _java_options = list(options)


def java(cmd, classpath=None, stdin=None, stdout=None, stderr=None, blocking=True):
    

    subprocess_output_dict = {
        "pipe": subprocess.PIPE,
        "stdout": subprocess.STDOUT,
        "devnull": subprocess.DEVNULL,
    }

    stdin = subprocess_output_dict.get(stdin, stdin)
    stdout = subprocess_output_dict.get(stdout, stdout)
    stderr = subprocess_output_dict.get(stderr, stderr)

    if isinstance(cmd, str):
        raise TypeError("cmd should be a list of strings")

    
    if _java_bin is None:
        config_java()

    
    if isinstance(classpath, str):
        classpaths = [classpath]
    else:
        classpaths = list(classpath)
    classpath = os.path.pathsep.join(classpaths)

    
    cmd = list(cmd)
    cmd = ["-cp", classpath] + cmd
    cmd = [_java_bin] + _java_options + cmd

    
    p = subprocess.Popen(cmd, stdin=stdin, stdout=stdout, stderr=stderr)
    if not blocking:
        return p
    (stdout, stderr) = p.communicate()

    
    if p.returncode != 0:
        print(_decode_stdoutdata(stderr))
        raise OSError("Java command failed : " + str(cmd))

    return (stdout, stderr)







class ReadError(ValueError):
    

    def __init__(self, expected, position):
        ValueError.__init__(self, expected, position)
        self.expected = expected
        self.position = position

    def __str__(self):
        return f"Expected {self.expected} at {self.position}"


_STRING_START_RE = re.compile(r"[uU]?[rR]?(\"\"\"|\'\'\'|\"|\')")


def read_str(s, start_position):
    
    
    m = _STRING_START_RE.match(s, start_position)
    if not m:
        raise ReadError("open quote", start_position)
    quotemark = m.group(1)

    
    _STRING_END_RE = re.compile(r"\\|%s" % quotemark)
    position = m.end()
    while True:
        match = _STRING_END_RE.search(s, position)
        if not match:
            raise ReadError("close quote", position)
        if match.group(0) == "\\":
            position = match.end() + 1
        else:
            break

    
    
    try:
        return eval(s[start_position : match.end()]), match.end()
    except ValueError as e:
        raise ReadError("valid escape sequence", start_position) from e


_READ_INT_RE = re.compile(r"-?\d+")


def read_int(s, start_position):
    
    m = _READ_INT_RE.match(s, start_position)
    if not m:
        raise ReadError("integer", start_position)
    return int(m.group()), m.end()


_READ_NUMBER_VALUE = re.compile(r"-?(\d*)([.]?\d*)?")


def read_number(s, start_position):
    
    m = _READ_NUMBER_VALUE.match(s, start_position)
    if not m or not (m.group(1) or m.group(2)):
        raise ReadError("number", start_position)
    if m.group(2):
        return float(m.group()), m.end()
    else:
        return int(m.group()), m.end()







def overridden(method):
    
    if isinstance(method, types.MethodType) and method.__self__.__class__ is not None:
        name = method.__name__
        funcs = [
            cls.__dict__[name]
            for cls in _mro(method.__self__.__class__)
            if name in cls.__dict__
        ]
        return len(funcs) > 1
    else:
        raise TypeError("Expected an instance method.")


def _mro(cls):
    
    if isinstance(cls, type):
        return cls.__mro__
    else:
        mro = [cls]
        for base in cls.__bases__:
            mro.extend(_mro(base))
        return mro








def _add_epytext_field(obj, field, message):
    
    indent = ""
    
    
    if obj.__doc__:
        obj.__doc__ = obj.__doc__.rstrip() + "\n\n"
        indents = re.findall(r"(?<=\n)[ ]+(?!\s)", obj.__doc__.expandtabs())
        if indents:
            indent = min(indents)
    
    else:
        obj.__doc__ = ""

    obj.__doc__ += textwrap.fill(
        f"@{field}: {message}",
        initial_indent=indent,
        subsequent_indent=indent + "    ",
    )


def deprecated(message):
    

    def decorator(func):
        msg = f"Function {func.__name__}() has been deprecated.  {message}"
        msg = "\n" + textwrap.fill(msg, initial_indent="  ", subsequent_indent="  ")

        def newFunc(*args, **kwargs):
            warnings.warn(msg, category=DeprecationWarning, stacklevel=2)
            return func(*args, **kwargs)

        
        newFunc.__dict__.update(func.__dict__)
        newFunc.__name__ = func.__name__
        newFunc.__doc__ = func.__doc__
        newFunc.__deprecated__ = True
        
        _add_epytext_field(newFunc, "deprecated", message)
        return newFunc

    return decorator


class Deprecated:
    

    def __new__(cls, *args, **kwargs):
        
        dep_cls = None
        for base in _mro(cls):
            if Deprecated in base.__bases__:
                dep_cls = base
                break
        assert dep_cls, "Unable to determine which base is deprecated."

        
        doc = dep_cls.__doc__ or "".strip()
        
        doc = re.sub(r"\A\s*@deprecated:", r"", doc)
        
        doc = re.sub(r"(?m)^\s*", "", doc)
        
        name = "Class %s" % dep_cls.__name__
        if cls != dep_cls:
            name += " (base class for %s)" % cls.__name__
        
        msg = f"{name} has been deprecated.  {doc}"
        
        msg = "\n" + textwrap.fill(msg, initial_indent="    ", subsequent_indent="    ")
        warnings.warn(msg, category=DeprecationWarning, stacklevel=2)
        
        return object.__new__(cls)







class Counter:
    

    def __init__(self, initial_value=0):
        self._value = initial_value

    def get(self):
        self._value += 1
        return self._value







def find_file_iter(
    filename,
    env_vars=(),
    searchpath=(),
    file_names=None,
    url=None,
    verbose=False,
    finding_dir=False,
):
    
    file_names = [filename] + (file_names or [])
    assert isinstance(filename, str)
    assert not isinstance(file_names, str)
    assert not isinstance(searchpath, str)
    if isinstance(env_vars, str):
        env_vars = env_vars.split()
    yielded = False

    
    for alternative in file_names:
        path_to_file = os.path.join(filename, alternative)
        if os.path.isfile(path_to_file):
            if verbose:
                print(f"[Found {filename}: {path_to_file}]")
            yielded = True
            yield path_to_file
        
        if os.path.isfile(alternative):
            if verbose:
                print(f"[Found {filename}: {alternative}]")
            yielded = True
            yield alternative
        
        path_to_file = os.path.join(filename, "file", alternative)
        if os.path.isfile(path_to_file):
            if verbose:
                print(f"[Found {filename}: {path_to_file}]")
            yielded = True
            yield path_to_file

    
    for env_var in env_vars:
        if env_var in os.environ:
            if finding_dir:  
                yielded = True
                yield os.environ[env_var]

            for env_dir in os.environ[env_var].split(os.pathsep):
                
                if os.path.isfile(env_dir):
                    if verbose:
                        print(f"[Found {filename}: {env_dir}]")
                    yielded = True
                    yield env_dir
                
                for alternative in file_names:
                    path_to_file = os.path.join(env_dir, alternative)
                    if os.path.isfile(path_to_file):
                        if verbose:
                            print(f"[Found {filename}: {path_to_file}]")
                        yielded = True
                        yield path_to_file
                    
                    

                    
                    path_to_file = os.path.join(env_dir, "bin", alternative)

                    if os.path.isfile(path_to_file):
                        if verbose:
                            print(f"[Found {filename}: {path_to_file}]")
                        yielded = True
                        yield path_to_file

    
    for directory in searchpath:
        for alternative in file_names:
            path_to_file = os.path.join(directory, alternative)
            if os.path.isfile(path_to_file):
                yielded = True
                yield path_to_file

    
    
    if os.name == "posix":
        for alternative in file_names:
            try:
                p = subprocess.Popen(
                    ["which", alternative],
                    stdout=subprocess.PIPE,
                    stderr=subprocess.PIPE,
                )
                stdout, stderr = p.communicate()
                path = _decode_stdoutdata(stdout).strip()
                if path.endswith(alternative) and os.path.exists(path):
                    if verbose:
                        print(f"[Found {filename}: {path}]")
                    yielded = True
                    yield path
            except (KeyboardInterrupt, SystemExit, OSError):
                raise
            finally:
                pass

    if not yielded:
        msg = (
            "NLTK was unable to find the %s file!"
            "\nUse software specific "
            "configuration parameters" % filename
        )
        if env_vars:
            msg += " or set the %s environment variable" % env_vars[0]
        msg += "."
        if searchpath:
            msg += "\n\n  Searched in:"
            msg += "".join("\n    - %s" % d for d in searchpath)
        if url:
            msg += f"\n\n  For more information on {filename}, see:\n    <{url}>"
        div = "=" * 75
        raise LookupError(f"\n\n{div}\n{msg}\n{div}")


def find_file(
    filename, env_vars=(), searchpath=(), file_names=None, url=None, verbose=False
):
    return next(
        find_file_iter(filename, env_vars, searchpath, file_names, url, verbose)
    )


def find_dir(
    filename, env_vars=(), searchpath=(), file_names=None, url=None, verbose=False
):
    return next(
        find_file_iter(
            filename, env_vars, searchpath, file_names, url, verbose, finding_dir=True
        )
    )


def find_binary_iter(
    name,
    path_to_bin=None,
    env_vars=(),
    searchpath=(),
    binary_names=None,
    url=None,
    verbose=False,
):
    
    yield from find_file_iter(
        path_to_bin or name, env_vars, searchpath, binary_names, url, verbose
    )


def find_binary(
    name,
    path_to_bin=None,
    env_vars=(),
    searchpath=(),
    binary_names=None,
    url=None,
    verbose=False,
):
    return next(
        find_binary_iter(
            name, path_to_bin, env_vars, searchpath, binary_names, url, verbose
        )
    )


def find_jar_iter(
    name_pattern,
    path_to_jar=None,
    env_vars=(),
    searchpath=(),
    url=None,
    verbose=False,
    is_regex=False,
):
    

    assert isinstance(name_pattern, str)
    assert not isinstance(searchpath, str)
    if isinstance(env_vars, str):
        env_vars = env_vars.split()
    yielded = False

    
    env_vars = ["CLASSPATH"] + list(env_vars)

    
    
    if path_to_jar is not None:
        if os.path.isfile(path_to_jar):
            yielded = True
            yield path_to_jar
        else:
            raise LookupError(
                f"Could not find {name_pattern} jar file at {path_to_jar}"
            )

    
    for env_var in env_vars:
        if env_var in os.environ:
            if env_var == "CLASSPATH":
                classpath = os.environ["CLASSPATH"]
                for cp in classpath.split(os.path.pathsep):
                    cp = os.path.expanduser(cp)
                    if os.path.isfile(cp):
                        filename = os.path.basename(cp)
                        if (
                            is_regex
                            and re.match(name_pattern, filename)
                            or (not is_regex and filename == name_pattern)
                        ):
                            if verbose:
                                print(f"[Found {name_pattern}: {cp}]")
                            yielded = True
                            yield cp
                    
                    if os.path.isdir(cp):
                        if not is_regex:
                            if os.path.isfile(os.path.join(cp, name_pattern)):
                                if verbose:
                                    print(f"[Found {name_pattern}: {cp}]")
                                yielded = True
                                yield os.path.join(cp, name_pattern)
                        else:
                            
                            for file_name in os.listdir(cp):
                                if re.match(name_pattern, file_name):
                                    if verbose:
                                        print(
                                            "[Found %s: %s]"
                                            % (
                                                name_pattern,
                                                os.path.join(cp, file_name),
                                            )
                                        )
                                    yielded = True
                                    yield os.path.join(cp, file_name)

            else:
                jar_env = os.path.expanduser(os.environ[env_var])
                jar_iter = (
                    (
                        os.path.join(jar_env, path_to_jar)
                        for path_to_jar in os.listdir(jar_env)
                    )
                    if os.path.isdir(jar_env)
                    else (jar_env,)
                )
                for path_to_jar in jar_iter:
                    if os.path.isfile(path_to_jar):
                        filename = os.path.basename(path_to_jar)
                        if (
                            is_regex
                            and re.match(name_pattern, filename)
                            or (not is_regex and filename == name_pattern)
                        ):
                            if verbose:
                                print(f"[Found {name_pattern}: {path_to_jar}]")
                            yielded = True
                            yield path_to_jar

    
    for directory in searchpath:
        if is_regex:
            for filename in os.listdir(directory):
                path_to_jar = os.path.join(directory, filename)
                if os.path.isfile(path_to_jar):
                    if re.match(name_pattern, filename):
                        if verbose:
                            print(f"[Found {filename}: {path_to_jar}]")
                yielded = True
                yield path_to_jar
        else:
            path_to_jar = os.path.join(directory, name_pattern)
            if os.path.isfile(path_to_jar):
                if verbose:
                    print(f"[Found {name_pattern}: {path_to_jar}]")
                yielded = True
                yield path_to_jar

    if not yielded:
        
        msg = "NLTK was unable to find %s!" % name_pattern
        if env_vars:
            msg += " Set the %s environment variable" % env_vars[0]
        msg = textwrap.fill(msg + ".", initial_indent="  ", subsequent_indent="  ")
        if searchpath:
            msg += "\n\n  Searched in:"
            msg += "".join("\n    - %s" % d for d in searchpath)
        if url:
            msg += "\n\n  For more information, on {}, see:\n    <{}>".format(
                name_pattern,
                url,
            )
        div = "=" * 75
        raise LookupError(f"\n\n{div}\n{msg}\n{div}")


def find_jar(
    name_pattern,
    path_to_jar=None,
    env_vars=(),
    searchpath=(),
    url=None,
    verbose=False,
    is_regex=False,
):
    return next(
        find_jar_iter(
            name_pattern, path_to_jar, env_vars, searchpath, url, verbose, is_regex
        )
    )


def find_jars_within_path(path_to_jars):
    return [
        os.path.join(root, filename)
        for root, dirnames, filenames in os.walk(path_to_jars)
        for filename in fnmatch.filter(filenames, "*.jar")
    ]


def _decode_stdoutdata(stdoutdata):
    
    if not isinstance(stdoutdata, bytes):
        return stdoutdata

    encoding = getattr(sys.__stdout__, "encoding", locale.getpreferredencoding())
    if encoding is None:
        return stdoutdata.decode()
    return stdoutdata.decode(encoding)







def import_from_stdlib(module):
    
    old_path = sys.path
    sys.path = [d for d in sys.path if d not in ("", ".")]
    m = __import__(module)
    sys.path = old_path
    return m







class ElementWrapper:
    

    
    def __new__(cls, etree):
        
        if isinstance(etree, ElementWrapper):
            return etree
        else:
            return object.__new__(ElementWrapper)

    def __init__(self, etree):
        r
        if isinstance(etree, str):
            etree = ElementTree.fromstring(etree)
        self.__dict__["_etree"] = etree

    def unwrap(self):
        
        return self._etree

    
    
    

    def __repr__(self):
        s = ElementTree.tostring(self._etree, encoding="utf8").decode("utf8")
        if len(s) > 60:
            e = s.rfind("<")
            if (len(s) - e) > 30:
                e = -20
            s = f"{s[:30]}...{s[e:]}"
        return "<Element %r>" % s

    def __str__(self):
        
        return (
            ElementTree.tostring(self._etree, encoding="utf8").decode("utf8").rstrip()
        )

    
    
    

    def __getattr__(self, attrib):
        return getattr(self._etree, attrib)

    def __setattr__(self, attr, value):
        return setattr(self._etree, attr, value)

    def __delattr__(self, attr):
        return delattr(self._etree, attr)

    def __setitem__(self, index, element):
        self._etree[index] = element

    def __delitem__(self, index):
        del self._etree[index]

    def __setslice__(self, start, stop, elements):
        self._etree[start:stop] = elements

    def __delslice__(self, start, stop):
        del self._etree[start:stop]

    def __len__(self):
        return len(self._etree)

    
    
    

    def __getitem__(self, index):
        return ElementWrapper(self._etree[index])

    def __getslice__(self, start, stop):
        return [ElementWrapper(elt) for elt in self._etree[start:stop]]

    def getchildren(self):
        return [ElementWrapper(elt) for elt in self._etree]

    def getiterator(self, tag=None):
        return (ElementWrapper(elt) for elt in self._etree.getiterator(tag))

    def makeelement(self, tag, attrib):
        return ElementWrapper(self._etree.makeelement(tag, attrib))

    def find(self, path):
        elt = self._etree.find(path)
        if elt is None:
            return elt
        else:
            return ElementWrapper(elt)

    def findall(self, path):
        return [ElementWrapper(elt) for elt in self._etree.findall(path)]







def slice_bounds(sequence, slice_obj, allow_step=False):
    
    start, stop = (slice_obj.start, slice_obj.stop)

    
    
    if allow_step:
        step = slice_obj.step
        if step is None:
            step = 1
        
        
        
        if step < 0:
            start, stop = slice_bounds(sequence, slice(stop, start))
        else:
            start, stop = slice_bounds(sequence, slice(start, stop))
        return start, stop, step

    
    elif slice_obj.step not in (None, 1):
        raise ValueError(
            "slices with steps are not supported by %s" % sequence.__class__.__name__
        )

    
    if start is None:
        start = 0
    if stop is None:
        stop = len(sequence)

    
    if start < 0:
        start = max(0, len(sequence) + start)
    if stop < 0:
        stop = max(0, len(sequence) + stop)

    
    
    
    if stop > 0:
        try:
            sequence[stop - 1]
        except IndexError:
            stop = len(sequence)

    
    start = min(start, stop)

    
    return start, stop







def is_writable(path):
    
    if not os.path.exists(path):
        return False

    
    if hasattr(os, "getuid"):
        statdata = os.stat(path)
        perm = stat.S_IMODE(statdata.st_mode)
        
        if perm & 0o002:
            return True
        
        elif statdata.st_uid == os.getuid() and (perm & 0o200):
            return True
        
        elif (statdata.st_gid in [os.getgid()] + os.getgroups()) and (perm & 0o020):
            return True
        
        else:
            return False

    
    
    return True







def raise_unorderable_types(ordering, a, b):
    raise TypeError(
        "unorderable types: %s() %s %s()"
        % (type(a).__name__, ordering, type(b).__name__)
    )