natural language toolkit tokenizers c 20012023 nltk project yoav goldberg yoavgcs bgu ac il steven bird stevenbird1gmail com minor edits url https www nltk org for license information see license txt sexpression tokenizer sexprtokenizer is used to find parenthesized expressions in a string in particular it divides a string into a sequence of substrings that are either parenthesized expressions including any nested parenthesized expressions or other whitespaceseparated tokens from nltk tokenize import sexprtokenizer sexprtokenizer tokenize a b c d e f g a b c d e f g by default sexprtokenizer will raise a valueerror exception if used to tokenize an expression with nonmatching parentheses sexprtokenizer tokenize c d e f g traceback most recent call last valueerror unmatched close paren at char 1 the strict argument can be set to false to allow for nonmatching parentheses any unmatched close parentheses will be listed as their own sexpression and the last partial sexpr with unmatched open parentheses will be listed as its own sexpr sexprtokenizerstrictfalse tokenize c d e f g c d e f g the characters used for open and close parentheses may be customized using the parens argument to the sexprtokenizer constructor sexprtokenizerparens tokenize a b c d e f g a b c d e f g the sexpression tokenizer is also available as a function from nltk tokenize import sexprtokenize sexprtokenize a b c d e f g a b c d e f g a tokenizer that divides strings into sexpressions an sexpresion can be either a parenthesized expression including any nested parenthesized expressions or a sequence of nonwhitespace nonparenthesis characters for example the string a b c d e f consists of four sexpressions a b c d e and f by default the characters and are treated as open and close parentheses but alternative strings may be specified param parens a twoelement sequence specifying the open and close parentheses that should be used to find sexprs this will typically be either a twocharacter string or a list of two strings type parens str or list param strict if true then raise an exception when tokenizing an illformed sexpr return a list of sexpressions extracted from text for example sexprtokenizer tokenize a b c d e f g a b c d e f g all parentheses are assumed to mark sexpressions no special processing is done to exclude parentheses that occur inside strings or following backslash characters if the given expression contains nonmatching parentheses then the behavior of the tokenizer depends on the strict parameter to the constructor if strict is true then raise a valueerror if strict is false then any unmatched close parentheses will be listed as their own sexpression and the last partial sexpression with unmatched open parentheses will be listed as its own sexpression sexprtokenizerstrictfalse tokenize c d e f g c d e f g param text the string to be tokenized type text str or iterstr rtype iterstr natural language toolkit tokenizers c 2001 2023 nltk project yoav goldberg yoavg cs bgu ac il steven bird stevenbird1 gmail com minor edits url https www nltk org for license information see license txt s expression tokenizer sexprtokenizer is used to find parenthesized expressions in a string in particular it divides a string into a sequence of substrings that are either parenthesized expressions including any nested parenthesized expressions or other whitespace separated tokens from nltk tokenize import sexprtokenizer sexprtokenizer tokenize a b c d e f g a b c d e f g by default sexprtokenizer will raise a valueerror exception if used to tokenize an expression with non matching parentheses sexprtokenizer tokenize c d e f g traceback most recent call last valueerror un matched close paren at char 1 the strict argument can be set to false to allow for non matching parentheses any unmatched close parentheses will be listed as their own s expression and the last partial sexpr with unmatched open parentheses will be listed as its own sexpr sexprtokenizer strict false tokenize c d e f g c d e f g the characters used for open and close parentheses may be customized using the parens argument to the sexprtokenizer constructor sexprtokenizer parens tokenize a b c d e f g a b c d e f g the s expression tokenizer is also available as a function from nltk tokenize import sexpr_tokenize sexpr_tokenize a b c d e f g a b c d e f g a tokenizer that divides strings into s expressions an s expresion can be either a parenthesized expression including any nested parenthesized expressions or a sequence of non whitespace non parenthesis characters for example the string a b c d e f consists of four s expressions a b c d e and f by default the characters and are treated as open and close parentheses but alternative strings may be specified param parens a two element sequence specifying the open and close parentheses that should be used to find sexprs this will typically be either a two character string or a list of two strings type parens str or list param strict if true then raise an exception when tokenizing an ill formed sexpr return a list of s expressions extracted from text for example sexprtokenizer tokenize a b c d e f g a b c d e f g all parentheses are assumed to mark s expressions no special processing is done to exclude parentheses that occur inside strings or following backslash characters if the given expression contains non matching parentheses then the behavior of the tokenizer depends on the strict parameter to the constructor if strict is true then raise a valueerror if strict is false then any unmatched close parentheses will be listed as their own s expression and the last partial s expression with unmatched open parentheses will be listed as its own s expression sexprtokenizer strict false tokenize c d e f g c d e f g param text the string to be tokenized type text str or iter str rtype iter str
import re

from nltk.tokenize.api import TokenizerI


class SExprTokenizer(TokenizerI):
    

    def __init__(self, parens="()", strict=True):
        if len(parens) != 2:
            raise ValueError("parens must contain exactly two strings")
        self._strict = strict
        self._open_paren = parens[0]
        self._close_paren = parens[1]
        self._paren_regexp = re.compile(
            f"{re.escape(parens[0])}|{re.escape(parens[1])}"
        )

    def tokenize(self, text):
        
        result = []
        pos = 0
        depth = 0
        for m in self._paren_regexp.finditer(text):
            paren = m.group()
            if depth == 0:
                result += text[pos : m.start()].split()
                pos = m.start()
            if paren == self._open_paren:
                depth += 1
            if paren == self._close_paren:
                if self._strict and depth == 0:
                    raise ValueError("Un-matched close paren at char %d" % m.start())
                depth = max(0, depth - 1)
                if depth == 0:
                    result.append(text[pos : m.end()])
                    pos = m.end()
        if self._strict and depth > 0:
            raise ValueError("Un-matched open paren at char %d" % pos)
        if pos < len(text):
            result.append(text[pos:])
        return result


sexpr_tokenize = SExprTokenizer().tokenize