natural language toolkit simple tokenizers c 20012023 nltk project edward loper edlopergmail com steven bird stevenbird1gmail com url https www nltk org for license information see license txt from nltk tokenize api import stringtokenizer tokenizeri from nltk tokenize util import regexpspantokenize stringspantokenize class spacetokenizerstringtokenizer rtokenize a string using the space character as a delimiter which is the same as s split from nltk tokenize import spacetokenizer s good muffins cost 3 88nin new york please buy mentwo of them nnthanks spacetokenizer tokenizes doctest normalizewhitespace good muffins cost 3 88nin new york please buy mentwo of them nnthanks string t class chartokenizerstringtokenizer string none def tokenizeself s return lists def spantokenizeself s yield from enumeraterange1 lens 1 class linetokenizertokenizeri rtokenize a string into its lines optionally discarding blank lines this is similar to s split n from nltk tokenize import linetokenizer s good muffins cost 3 88nin new york please buy mentwo of them nnthanks linetokenizerblanklines keep tokenizes doctest normalizewhitespace good muffins cost 3 88 in new york please buy me two of them thanks same as l for l in s split n if l strip linetokenizerblanklines discard tokenizes doctest normalizewhitespace good muffins cost 3 88 in new york please buy me two of them thanks param blanklines indicates how blank lines should be handled valid values are discard strip blank lines out of the token list before returning it a line is considered blank if it contains only whitespace characters keep leave all blank lines in the token list discardeof if the string ends with a newline then do not generate a corresponding token after that newline if requested strip off blank lines discardeof not implemented tokenization functions xxx it is stated in module docs that there is no function versions natural language toolkit simple tokenizers c 2001 2023 nltk project edward loper edloper gmail com steven bird stevenbird1 gmail com url https www nltk org for license information see license txt simple tokenizers these tokenizers divide strings into substrings using the string split method when tokenizing using a particular delimiter string use the string split method directly as this is more efficient the simple tokenizers are not available as separate functions instead you should just use the string split method directly s good muffins cost 3 88 nin new york please buy me ntwo of them n nthanks s split doctest normalize_whitespace good muffins cost 3 88 in new york please buy me two of them thanks s split doctest normalize_whitespace good muffins cost 3 88 nin new york please buy me ntwo of them n nthanks s split n doctest normalize_whitespace good muffins cost 3 88 in new york please buy me two of them thanks the simple tokenizers are mainly useful because they follow the standard tokenizeri interface and so can be used with any code that expects a tokenizer for example these tokenizers can be used to specify the tokenization conventions when building a corpusreader tokenize a string using the space character as a delimiter which is the same as s split from nltk tokenize import spacetokenizer s good muffins cost 3 88 nin new york please buy me ntwo of them n nthanks spacetokenizer tokenize s doctest normalize_whitespace good muffins cost 3 88 nin new york please buy me ntwo of them n nthanks tokenize a string use the tab character as a delimiter the same as s split t from nltk tokenize import tabtokenizer tabtokenizer tokenize a tb c n t d a b c n d tokenize a string into individual characters if this functionality is ever required directly use for char in string tokenize a string into its lines optionally discarding blank lines this is similar to s split n from nltk tokenize import linetokenizer s good muffins cost 3 88 nin new york please buy me ntwo of them n nthanks linetokenizer blanklines keep tokenize s doctest normalize_whitespace good muffins cost 3 88 in new york please buy me two of them thanks same as l for l in s split n if l strip linetokenizer blanklines discard tokenize s doctest normalize_whitespace good muffins cost 3 88 in new york please buy me two of them thanks param blanklines indicates how blank lines should be handled valid values are discard strip blank lines out of the token list before returning it a line is considered blank if it contains only whitespace characters keep leave all blank lines in the token list discard eof if the string ends with a newline then do not generate a corresponding token after that newline if requested strip off blank lines discard eof not implemented tokenization functions xxx it is stated in module docs that there is no function versions
r

from nltk.tokenize.api import StringTokenizer, TokenizerI
from nltk.tokenize.util import regexp_span_tokenize, string_span_tokenize


class SpaceTokenizer(StringTokenizer):
    r

    _string = " "


class TabTokenizer(StringTokenizer):
    r

    _string = "\t"


class CharTokenizer(StringTokenizer):
    

    _string = None

    def tokenize(self, s):
        return list(s)

    def span_tokenize(self, s):
        yield from enumerate(range(1, len(s) + 1))


class LineTokenizer(TokenizerI):
    r

    def __init__(self, blanklines="discard"):
        valid_blanklines = ("discard", "keep", "discard-eof")
        if blanklines not in valid_blanklines:
            raise ValueError(
                "Blank lines must be one of: %s" % " ".join(valid_blanklines)
            )

        self._blanklines = blanklines

    def tokenize(self, s):
        lines = s.splitlines()
        
        if self._blanklines == "discard":
            lines = [l for l in lines if l.rstrip()]
        elif self._blanklines == "discard-eof":
            if lines and not lines[-1].strip():
                lines.pop()
        return lines

    
    def span_tokenize(self, s):
        if self._blanklines == "keep":
            yield from string_span_tokenize(s, r"\n")
        else:
            yield from regexp_span_tokenize(s, r"\n(\s+\n)*")








def line_tokenize(text, blanklines="discard"):
    return LineTokenizer(blanklines).tokenize(text)