natural language toolkit tokenizers c 20012023 nltk project edward loper edlopergmail com steven bird stevenbird1gmail com minor additions contributors matthewmc clouds56 url https www nltk org for license information see license txt import re from nltk data import load from nltk tokenize casual import tweettokenizer casualtokenize from nltk tokenize destructive import nltkwordtokenizer from nltk tokenize legalityprinciple import legalitysyllabletokenizer from nltk tokenize mwe import mwetokenizer from nltk tokenize punkt import punktsentencetokenizer from nltk tokenize regexp import blanklinetokenizer regexptokenizer whitespacetokenizer wordpuncttokenizer blanklinetokenize regexptokenize wordpuncttokenize from nltk tokenize repp import repptokenizer from nltk tokenize sexpr import sexprtokenizer sexprtokenize from nltk tokenize simple import linetokenizer spacetokenizer tabtokenizer linetokenize from nltk tokenize sonoritysequencing import syllabletokenizer from nltk tokenize stanfordsegmenter import stanfordsegmenter from nltk tokenize texttiling import texttilingtokenizer from nltk tokenize toktok import toktoktokenizer from nltk tokenize treebank import treebankworddetokenizer treebankwordtokenizer from nltk tokenize util import regexpspantokenize stringspantokenize standard sentence tokenizer def senttokenizetext languageenglish tokenizer loadftokenizerspunktlanguage pickle return tokenizer tokenizetext standard word tokenizer treebankwordtokenizer nltkwordtokenizer def wordtokenizetext languageenglish preservelinefalse sentences text if preserveline else senttokenizetext language return token for sent in sentences for token in treebankwordtokenizer tokenizesent natural language toolkit tokenizers c 2001 2023 nltk project edward loper edloper gmail com steven bird stevenbird1 gmail com minor additions contributors matthewmc clouds56 url https www nltk org for license information see license txt nltk tokenizer package tokenizers divide strings into lists of substrings for example tokenizers can be used to find the words and punctuation in a string from nltk tokenize import word_tokenize s good muffins cost 3 88 nin new york please buy me two of them n nthanks word_tokenize s doctest normalize_whitespace good muffins cost 3 88 in new york please buy me two of them thanks this particular tokenizer requires the punkt sentence tokenization models to be installed nltk also provides a simpler regular expression based tokenizer which splits text on whitespace and punctuation from nltk tokenize import wordpunct_tokenize wordpunct_tokenize s doctest normalize_whitespace good muffins cost 3 88 in new york please buy me two of them thanks we can also operate at the level of sentences using the sentence tokenizer directly as follows from nltk tokenize import sent_tokenize word_tokenize sent_tokenize s good muffins cost 3 88 nin new york please buy me ntwo of them thanks word_tokenize t for t in sent_tokenize s doctest normalize_whitespace good muffins cost 3 88 in new york please buy me two of them thanks caution when tokenizing a unicode string make sure you are not using an encoded version of the string it may be necessary to decode it first e g with s decode utf8 nltk tokenizers can produce token spans represented as tuples of integers having the same semantics as string slices to support efficient comparison of tokenizers these methods are implemented as generators from nltk tokenize import whitespacetokenizer list whitespacetokenizer span_tokenize s doctest normalize_whitespace 0 4 5 12 13 17 18 23 24 26 27 30 31 36 38 44 45 48 49 51 52 55 56 58 59 64 66 73 there are numerous ways to tokenize text if you need more control over tokenization see the other methods provided in this package for further information please see chapter 3 of the nltk book standard sentence tokenizer return a sentence tokenized copy of text using nltk s recommended sentence tokenizer currently class punktsentencetokenizer for the specified language param text text to split into sentences param language the model name in the punkt corpus standard word tokenizer return a tokenized copy of text using nltk s recommended word tokenizer currently an improved class treebankwordtokenizer along with class punktsentencetokenizer for the specified language param text text to split into words type text str param language the model name in the punkt corpus type language str param preserve_line a flag to decide whether to sentence tokenize the text or not type preserve_line bool
r

import re

from nltk.data import load
from nltk.tokenize.casual import TweetTokenizer, casual_tokenize
from nltk.tokenize.destructive import NLTKWordTokenizer
from nltk.tokenize.legality_principle import LegalitySyllableTokenizer
from nltk.tokenize.mwe import MWETokenizer
from nltk.tokenize.punkt import PunktSentenceTokenizer
from nltk.tokenize.regexp import (
    BlanklineTokenizer,
    RegexpTokenizer,
    WhitespaceTokenizer,
    WordPunctTokenizer,
    blankline_tokenize,
    regexp_tokenize,
    wordpunct_tokenize,
)
from nltk.tokenize.repp import ReppTokenizer
from nltk.tokenize.sexpr import SExprTokenizer, sexpr_tokenize
from nltk.tokenize.simple import (
    LineTokenizer,
    SpaceTokenizer,
    TabTokenizer,
    line_tokenize,
)
from nltk.tokenize.sonority_sequencing import SyllableTokenizer
from nltk.tokenize.stanford_segmenter import StanfordSegmenter
from nltk.tokenize.texttiling import TextTilingTokenizer
from nltk.tokenize.toktok import ToktokTokenizer
from nltk.tokenize.treebank import TreebankWordDetokenizer, TreebankWordTokenizer
from nltk.tokenize.util import regexp_span_tokenize, string_span_tokenize



def sent_tokenize(text, language="english"):
    
    tokenizer = load(f"tokenizers/punkt/{language}.pickle")
    return tokenizer.tokenize(text)



_treebank_word_tokenizer = NLTKWordTokenizer()


def word_tokenize(text, language="english", preserve_line=False):
    
    sentences = [text] if preserve_line else sent_tokenize(text, language)
    return [
        token for sent in sentences for token in _treebank_word_tokenizer.tokenize(sent)
    ]