natural language toolkit texts c 20012023 nltk project steven bird stevenbird1gmail com edward loper edlopergmail com url https www nltk org for license information see license txt this module brings together a variety of nltk functionality for text analysis and provides simple interactive interfaces functionality includes concordancing collocation discovery regular expression search over tokenized strings and distributional similarity a bidirectional index between words and their contexts in a text the context of a word is usually defined to be the words that occur in a fixed window around the word but other definitions may also be used by providing a custom context function one left token and one right token normalized to lowercase left tokensi 1 lower if i 0 else start right tokensi 1 lower if i lentokens 1 else end return left right def initself tokens contextfuncnone filternone keylambda x x self key key self tokens tokens if contextfunc self contextfunc contextfunc else self contextfunc self defaultcontext if filter tokens t for t in tokens if filtert self wordtocontexts cfd self keyw self contextfunctokens i for i w in enumeratetokens self contexttowords cfd self contextfunctokens i self keyw for i w in enumeratetokens def tokensself return self tokens def wordsimilaritydictself word word self keyword wordcontexts setself wordtocontextsword scores for w wcontexts in self wordtocontexts items scoresw fmeasurewordcontexts setwcontexts return scores def similarwordsself word n20 scores defaultdictint for c in self wordtocontextsself keyword for w in self contexttowordsc if w word scoresw self contexttowordscword self contexttowordscw return sortedscores keyscores get reversetrue n def commoncontextsself words failonunknownfalse words self keyw for w in words contexts setself wordtocontextsw for w in words empty wordsi for i in rangelenwords if not contextsi common reduceset intersection contexts if empty and failonunknown raise valueerrorthe following words were not found joinwords elif not common nothing in common just return an empty freqdist return freqdist else fd freqdist c for w in words for c in self wordtocontextsw if c in common return fd class concordanceindex def initself tokens keylambda x x self tokens tokens function mapping each token to an index key or none self offsets defaultdictlist initialize the index self offsets rtype liststr return the document that this concordance index was created from rtype listint return a list of the offset positions at which the given word occurs if a key function was specified for the index then given word s key will be looked up find all concordance lines given the query word provided with a list of words these will be found as a phrase find the instances of the word to create the concordanceline find the context of query word create the pretty lines with the queryword in the middle the wysiwyg line of the concordance create the concordanceline print concordance lines given the query word param word the target word or phrase a list of strings type word str or list param lines the number of lines to display default25 type lines int param width the width of each line in characters default80 type width int param save the option to save the concordance type save bool a class that makes it easier to use regular expressions to search over tokenized strings the tokenized string is converted to a string where tokens are marked with angle brackets e g thewindowisstillopen the regular expression passed to the findall method is modified to treat angle brackets as noncapturing parentheses in addition to matching the token boundaries and to have not match the angle brackets find instances of the regular expression in the text the text is a list of tokens and a regexp pattern to match a single token must be surrounded by angle brackets e g from nltk text import tokensearcher from nltk book import text1 text5 text9 text5 findall bro you rule bro telling you bro u twizted bro text1 findalla man monied nervous dangerous white white white pious queer good mature white cape great wise wise butterless white fiendish pale furious better certain complete dismasted younger brave brave brave brave text9 findallth 3 thread through those the thought that that the thing the thing that that that thing through these than through them that the through the thick them that they thought that the param regexp a regular expression type regexp str preprocess the regular expression perform the search sanity check postprocess the output a wrapper around a sequence of simple string tokens which is intended to support initial exploration of texts via the interactive console its methods perform a variety of analyses on the text s contexts e g counting concordancing collocation discovery and display the results if you wish to write a program which makes use of these analyses then you should bypass the text class and use the appropriate analysis function or class directly instead a text is typically initialized from a given document or corpus e g import nltk corpus from nltk text import text moby textnltk corpus gutenberg words melvillemobydick txt this defeats lazy loading but makes things faster this shouldn t be necessary because the corpus view should be doing intelligent caching but without this it s running slow look into whether the caching is working correctly create a text object param tokens the source text type tokens sequence of str support item slice access interactive console methods prints a concordance for word with the specified context window word matching is not casesensitive param word the target word or phrase a list of strings type word str or list param width the width of each line in characters default80 type width int param lines the number of lines to display default25 type lines int seealso concordanceindex generate a concordance for word with the specified context window word matching is not casesensitive param word the target word or phrase a list of strings type word str or list param width the width of each line in characters default80 type width int param lines the number of lines to display default25 type lines int seealso concordanceindex return collocations derived from the text ignoring stopwords from nltk book import text4 text4 collocationlist 2 united states fellow citizens param num the maximum number of collocations to return type num int param windowsize the number of tokens spanned by a collocation default2 type windowsize int rtype listtuplestr str printbuilding collocations list print collocations derived from the text ignoring stopwords from nltk book import text4 text4 collocations doctest normalizewhitespace united states fellow citizens years ago four years federal government general government american people vice president god bless chief justice one another fellow americans old world almighty god fellow citizens chief magistrate every citizen indian tribes public debt foreign nations param num the maximum number of collocations to print type num int param windowsize the number of tokens spanned by a collocation default2 type windowsize int count the number of times this word appears in the text find the index of the first occurrence of the word in the text code from nltkcontrib readability distributional similarity find other words which appear in the same contexts as the specified word list most similar words first param word the word used to seed the similarity search type word str param num the number of words to generate default20 type num int seealso contextindex similarwords print building wordcontext index words self wordcontextindex similarwordsword num find contexts where the specified words appear list most frequent common contexts first param words the words used to seed the similarity search type words str param num the number of words to generate default20 type num int seealso contextindex commoncontexts print building wordcontext index produce a plot showing the distribution of the words through the text requires pylab to be installed param words the words to be plotted type words liststr seealso nltk draw dispersionplot print random text generated using a trigram language model see also helpnltk lm param length the length of text to generate default100 type length int param textseed generation can be conditioned on preceding context type textseed liststr param randomseed a random seed or an instance of random random if provided makes the random sampling part of generation reproducible default42 type randomseed int create the model when using it the first time see documentation for freqdist plot seealso nltk prob freqdist plot seealso nltk prob freqdist printbuilding vocabulary index find instances of the regular expression in the text the text is a list of tokens and a regexp pattern to match a single token must be surrounded by angle brackets e g from nltk book import text1 text5 text9 text5 findall bro you rule bro telling you bro u twizted bro text1 findalla man monied nervous dangerous white white white pious queer good mature white cape great wise wise butterless white fiendish pale furious better certain complete dismasted younger brave brave brave brave text9 findallth 3 thread through those the thought that that the thing the thing that that that thing through these than through them that the through the thick them that they thought that the param regexp a regular expression type regexp str helper methods one left one right token both casenormalized skip over nonsentencefinal punctuation used by the contextindex that is created for similar and commoncontexts left context right context string display prototype only this approach will be slow to load a collection of texts which can be loaded with list of texts or with a corpus consisting of one or more texts and which supports counting concordancing collocation discovery etc initialize a textcollection as follows import nltk corpus from nltk text import textcollection from nltk book import text1 text2 text3 gutenberg textcollectionnltk corpus gutenberg mytexts textcollectiontext1 text2 text3 iterating over a textcollection produces all the tokens of all the texts in order the frequency of the term in text return text countterm lentext def idfself term idf values are cached for performance printautomatically generated text text generate print natural language toolkit texts c 2001 2023 nltk project steven bird stevenbird1 gmail com edward loper edloper gmail com url https www nltk org for license information see license txt this module brings together a variety of nltk functionality for text analysis and provides simple interactive interfaces functionality includes concordancing collocation discovery regular expression search over tokenized strings and distributional similarity a bidirectional index between words and their contexts in a text the context of a word is usually defined to be the words that occur in a fixed window around the word but other definitions may also be used by providing a custom context function one left token and one right token normalized to lowercase rtype list str return the document that this context index was created from return a dictionary mapping from words to similarity scores indicating how often these two words occur in the same context find contexts where the specified words can all appear and return a frequency distribution mapping each context to the number of times that context was used param words the words used to seed the similarity search type words str param fail_on_unknown if true then raise a value error if any of the given words do not occur at all in the index nothing in common just return an empty freqdist an index that can be used to look up the offset locations at which a given word occurs in a document construct a new concordance index param tokens the document list of tokens that this concordance index was created from this list can be used to access the context of a given word occurrence param key a function that maps each token to a normalized version that will be used as a key in the index e g if you use key lambda s s lower then the index will be case insensitive the document list of tokens that this concordance index was created from function mapping each token to an index key or none dictionary mapping words or keys to lists of offset indices initialize the index self _offsets rtype list str return the document that this concordance index was created from rtype list int return a list of the offset positions at which the given word occurs if a key function was specified for the index then given word s key will be looked up find all concordance lines given the query word provided with a list of words these will be found as a phrase approx number of words of context find the instances of the word to create the concordanceline find the context of query word create the pretty lines with the query_word in the middle the wysiwyg line of the concordance create the concordanceline print concordance lines given the query word param word the target word or phrase a list of strings type word str or list param lines the number of lines to display default 25 type lines int param width the width of each line in characters default 80 type width int param save the option to save the concordance type save bool a class that makes it easier to use regular expressions to search over tokenized strings the tokenized string is converted to a string where tokens are marked with angle brackets e g the window is still open the regular expression passed to the findall method is modified to treat angle brackets as non capturing parentheses in addition to matching the token boundaries and to have not match the angle brackets find instances of the regular expression in the text the text is a list of tokens and a regexp pattern to match a single token must be surrounded by angle brackets e g from nltk text import tokensearcher from nltk book import text1 text5 text9 text5 findall bro you rule bro telling you bro u twizted bro text1 findall a man monied nervous dangerous white white white pious queer good mature white cape great wise wise butterless white fiendish pale furious better certain complete dismasted younger brave brave brave brave text9 findall th 3 thread through those the thought that that the thing the thing that that that thing through these than through them that the through the thick them that they thought that the param regexp a regular expression type regexp str preprocess the regular expression perform the search sanity check postprocess the output a wrapper around a sequence of simple string tokens which is intended to support initial exploration of texts via the interactive console its methods perform a variety of analyses on the text s contexts e g counting concordancing collocation discovery and display the results if you wish to write a program which makes use of these analyses then you should bypass the text class and use the appropriate analysis function or class directly instead a text is typically initialized from a given document or corpus e g import nltk corpus from nltk text import text moby text nltk corpus gutenberg words melville moby_dick txt this defeats lazy loading but makes things faster this shouldn t be necessary because the corpus view should be doing intelligent caching but without this it s running slow look into whether the caching is working correctly create a text object param tokens the source text type tokens sequence of str support item slice access interactive console methods prints a concordance for word with the specified context window word matching is not case sensitive param word the target word or phrase a list of strings type word str or list param width the width of each line in characters default 80 type width int param lines the number of lines to display default 25 type lines int seealso concordanceindex generate a concordance for word with the specified context window word matching is not case sensitive param word the target word or phrase a list of strings type word str or list param width the width of each line in characters default 80 type width int param lines the number of lines to display default 25 type lines int seealso concordanceindex return collocations derived from the text ignoring stopwords from nltk book import text4 text4 collocation_list 2 united states fellow citizens param num the maximum number of collocations to return type num int param window_size the number of tokens spanned by a collocation default 2 type window_size int rtype list tuple str str print building collocations list print collocations derived from the text ignoring stopwords from nltk book import text4 text4 collocations doctest normalize_whitespace united states fellow citizens years ago four years federal government general government american people vice president god bless chief justice one another fellow americans old world almighty god fellow citizens chief magistrate every citizen indian tribes public debt foreign nations param num the maximum number of collocations to print type num int param window_size the number of tokens spanned by a collocation default 2 type window_size int count the number of times this word appears in the text find the index of the first occurrence of the word in the text code from nltk_contrib readability distributional similarity find other words which appear in the same contexts as the specified word list most similar words first param word the word used to seed the similarity search type word str param num the number of words to generate default 20 type num int seealso contextindex similar_words print building word context index words self _word_context_index similar_words word num find contexts where the specified words appear list most frequent common contexts first param words the words used to seed the similarity search type words str param num the number of words to generate default 20 type num int seealso contextindex common_contexts print building word context index produce a plot showing the distribution of the words through the text requires pylab to be installed param words the words to be plotted type words list str seealso nltk draw dispersion_plot print random text generated using a trigram language model see also help nltk lm param length the length of text to generate default 100 type length int param text_seed generation can be conditioned on preceding context type text_seed list str param random_seed a random seed or an instance of random random if provided makes the random sampling part of generation reproducible default 42 type random_seed int create the model when using it the first time see documentation for freqdist plot seealso nltk prob freqdist plot seealso nltk prob freqdist print building vocabulary index find instances of the regular expression in the text the text is a list of tokens and a regexp pattern to match a single token must be surrounded by angle brackets e g from nltk book import text1 text5 text9 text5 findall bro you rule bro telling you bro u twizted bro text1 findall a man monied nervous dangerous white white white pious queer good mature white cape great wise wise butterless white fiendish pale furious better certain complete dismasted younger brave brave brave brave text9 findall th 3 thread through those the thought that that the thing the thing that that that thing through these than through them that the through the thick them that they thought that the param regexp a regular expression type regexp str helper methods one left one right token both case normalized skip over non sentence final punctuation used by the contextindex that is created for similar and common_contexts left context right context string display prototype only this approach will be slow to load a collection of texts which can be loaded with list of texts or with a corpus consisting of one or more texts and which supports counting concordancing collocation discovery etc initialize a textcollection as follows import nltk corpus from nltk text import textcollection from nltk book import text1 text2 text3 gutenberg textcollection nltk corpus gutenberg mytexts textcollection text1 text2 text3 iterating over a textcollection produces all the tokens of all the texts in order bridge to the text corpus reader the frequency of the term in text the number of texts in the corpus divided by the number of texts that the term appears in if a term does not appear in the corpus 0 0 is returned idf values are cached for performance print automatically generated text text generate print
import re
import sys
from collections import Counter, defaultdict, namedtuple
from functools import reduce
from math import log

from nltk.collocations import BigramCollocationFinder
from nltk.lm import MLE
from nltk.lm.preprocessing import padded_everygram_pipeline
from nltk.metrics import BigramAssocMeasures, f_measure
from nltk.probability import ConditionalFreqDist as CFD
from nltk.probability import FreqDist
from nltk.tokenize import sent_tokenize
from nltk.util import LazyConcatenation, tokenwrap

ConcordanceLine = namedtuple(
    "ConcordanceLine",
    ["left", "query", "right", "offset", "left_print", "right_print", "line"],
)


class ContextIndex:
    

    @staticmethod
    def _default_context(tokens, i):
        
        left = tokens[i - 1].lower() if i != 0 else "*START*"
        right = tokens[i + 1].lower() if i != len(tokens) - 1 else "*END*"
        return (left, right)

    def __init__(self, tokens, context_func=None, filter=None, key=lambda x: x):
        self._key = key
        self._tokens = tokens
        if context_func:
            self._context_func = context_func
        else:
            self._context_func = self._default_context
        if filter:
            tokens = [t for t in tokens if filter(t)]
        self._word_to_contexts = CFD(
            (self._key(w), self._context_func(tokens, i)) for i, w in enumerate(tokens)
        )
        self._context_to_words = CFD(
            (self._context_func(tokens, i), self._key(w)) for i, w in enumerate(tokens)
        )

    def tokens(self):
        
        return self._tokens

    def word_similarity_dict(self, word):
        
        word = self._key(word)
        word_contexts = set(self._word_to_contexts[word])

        scores = {}
        for w, w_contexts in self._word_to_contexts.items():
            scores[w] = f_measure(word_contexts, set(w_contexts))

        return scores

    def similar_words(self, word, n=20):
        scores = defaultdict(int)
        for c in self._word_to_contexts[self._key(word)]:
            for w in self._context_to_words[c]:
                if w != word:
                    scores[w] += (
                        self._context_to_words[c][word] * self._context_to_words[c][w]
                    )
        return sorted(scores, key=scores.get, reverse=True)[:n]

    def common_contexts(self, words, fail_on_unknown=False):
        
        words = [self._key(w) for w in words]
        contexts = [set(self._word_to_contexts[w]) for w in words]
        empty = [words[i] for i in range(len(words)) if not contexts[i]]
        common = reduce(set.intersection, contexts)
        if empty and fail_on_unknown:
            raise ValueError("The following word(s) were not found:", " ".join(words))
        elif not common:
            
            return FreqDist()
        else:
            fd = FreqDist(
                c for w in words for c in self._word_to_contexts[w] if c in common
            )
            return fd


class ConcordanceIndex:
    

    def __init__(self, tokens, key=lambda x: x):
        
        self._tokens = tokens
        

        self._key = key
        

        self._offsets = defaultdict(list)
        
        
        for index, word in enumerate(tokens):
            word = self._key(word)
            self._offsets[word].append(index)

    def tokens(self):
        
        return self._tokens

    def offsets(self, word):
        
        word = self._key(word)
        return self._offsets[word]

    def __repr__(self):
        return "<ConcordanceIndex for %d tokens (%d types)>" % (
            len(self._tokens),
            len(self._offsets),
        )

    def find_concordance(self, word, width=80):
        
        if isinstance(word, list):
            phrase = word
        else:
            phrase = [word]

        half_width = (width - len(" ".join(phrase)) - 2) // 2
        context = width // 4  

        
        concordance_list = []
        offsets = self.offsets(phrase[0])
        for i, word in enumerate(phrase[1:]):
            word_offsets = {offset - i - 1 for offset in self.offsets(word)}
            offsets = sorted(word_offsets.intersection(offsets))
        if offsets:
            for i in offsets:
                query_word = " ".join(self._tokens[i : i + len(phrase)])
                
                left_context = self._tokens[max(0, i - context) : i]
                right_context = self._tokens[i + len(phrase) : i + context]
                
                left_print = " ".join(left_context)[-half_width:]
                right_print = " ".join(right_context)[:half_width]
                
                line_print = " ".join([left_print, query_word, right_print])
                
                concordance_line = ConcordanceLine(
                    left_context,
                    query_word,
                    right_context,
                    i,
                    left_print,
                    right_print,
                    line_print,
                )
                concordance_list.append(concordance_line)
        return concordance_list

    def print_concordance(self, word, width=80, lines=25):
        
        concordance_list = self.find_concordance(word, width=width)

        if not concordance_list:
            print("no matches")
        else:
            lines = min(lines, len(concordance_list))
            print(f"Displaying {lines} of {len(concordance_list)} matches:")
            for i, concordance_line in enumerate(concordance_list[:lines]):
                print(concordance_line.line)


class TokenSearcher:
    

    def __init__(self, tokens):
        self._raw = "".join("<" + w + ">" for w in tokens)

    def findall(self, regexp):
        
        
        regexp = re.sub(r"\s", "", regexp)
        regexp = re.sub(r"<", "(?:<(?:", regexp)
        regexp = re.sub(r">", ")>)", regexp)
        regexp = re.sub(r"(?<!\\)\.", "[^>]", regexp)

        
        hits = re.findall(regexp, self._raw)

        
        for h in hits:
            if not h.startswith("<") and h.endswith(">"):
                raise ValueError("Bad regexp for TokenSearcher.findall")

        
        hits = [h[1:-1].split("><") for h in hits]
        return hits


class Text:
    

    
    
    
    
    _COPY_TOKENS = True

    def __init__(self, tokens, name=None):
        
        if self._COPY_TOKENS:
            tokens = list(tokens)
        self.tokens = tokens

        if name:
            self.name = name
        elif "]" in tokens[:20]:
            end = tokens[:20].index("]")
            self.name = " ".join(str(tok) for tok in tokens[1:end])
        else:
            self.name = " ".join(str(tok) for tok in tokens[:8]) + "..."

    
    
    

    def __getitem__(self, i):
        return self.tokens[i]

    def __len__(self):
        return len(self.tokens)

    
    
    

    def concordance(self, word, width=79, lines=25):
        
        if "_concordance_index" not in self.__dict__:
            self._concordance_index = ConcordanceIndex(
                self.tokens, key=lambda s: s.lower()
            )

        return self._concordance_index.print_concordance(word, width, lines)

    def concordance_list(self, word, width=79, lines=25):
        
        if "_concordance_index" not in self.__dict__:
            self._concordance_index = ConcordanceIndex(
                self.tokens, key=lambda s: s.lower()
            )
        return self._concordance_index.find_concordance(word, width)[:lines]

    def collocation_list(self, num=20, window_size=2):
        
        if not (
            "_collocations" in self.__dict__
            and self._num == num
            and self._window_size == window_size
        ):
            self._num = num
            self._window_size = window_size

            
            from nltk.corpus import stopwords

            ignored_words = stopwords.words("english")
            finder = BigramCollocationFinder.from_words(self.tokens, window_size)
            finder.apply_freq_filter(2)
            finder.apply_word_filter(lambda w: len(w) < 3 or w.lower() in ignored_words)
            bigram_measures = BigramAssocMeasures()
            self._collocations = list(
                finder.nbest(bigram_measures.likelihood_ratio, num)
            )
        return self._collocations

    def collocations(self, num=20, window_size=2):
        

        collocation_strings = [
            w1 + " " + w2 for w1, w2 in self.collocation_list(num, window_size)
        ]
        print(tokenwrap(collocation_strings, separator="; "))

    def count(self, word):
        
        return self.tokens.count(word)

    def index(self, word):
        
        return self.tokens.index(word)

    def readability(self, method):
        
        raise NotImplementedError

    def similar(self, word, num=20):
        
        if "_word_context_index" not in self.__dict__:
            
            self._word_context_index = ContextIndex(
                self.tokens, filter=lambda x: x.isalpha(), key=lambda s: s.lower()
            )

        

        word = word.lower()
        wci = self._word_context_index._word_to_contexts
        if word in wci.conditions():
            contexts = set(wci[word])
            fd = Counter(
                w
                for w in wci.conditions()
                for c in wci[w]
                if c in contexts and not w == word
            )
            words = [w for w, _ in fd.most_common(num)]
            print(tokenwrap(words))
        else:
            print("No matches")

    def common_contexts(self, words, num=20):
        
        if "_word_context_index" not in self.__dict__:
            
            self._word_context_index = ContextIndex(
                self.tokens, key=lambda s: s.lower()
            )

        try:
            fd = self._word_context_index.common_contexts(words, True)
            if not fd:
                print("No common contexts were found")
            else:
                ranked_contexts = [w for w, _ in fd.most_common(num)]
                print(tokenwrap(w1 + "_" + w2 for w1, w2 in ranked_contexts))

        except ValueError as e:
            print(e)

    def dispersion_plot(self, words):
        
        from nltk.draw import dispersion_plot

        dispersion_plot(self, words)

    def _train_default_ngram_lm(self, tokenized_sents, n=3):
        train_data, padded_sents = padded_everygram_pipeline(n, tokenized_sents)
        model = MLE(order=n)
        model.fit(train_data, padded_sents)
        return model

    def generate(self, length=100, text_seed=None, random_seed=42):
        
        
        self._tokenized_sents = [
            sent.split(" ") for sent in sent_tokenize(" ".join(self.tokens))
        ]
        if not hasattr(self, "_trigram_model"):
            print("Building ngram index...", file=sys.stderr)
            self._trigram_model = self._train_default_ngram_lm(
                self._tokenized_sents, n=3
            )

        generated_tokens = []

        assert length > 0, "The `length` must be more than 0."
        while len(generated_tokens) < length:
            for idx, token in enumerate(
                self._trigram_model.generate(
                    length, text_seed=text_seed, random_seed=random_seed
                )
            ):
                if token == "<s>":
                    continue
                if token == "</s>":
                    break
                generated_tokens.append(token)
            random_seed += 1

        prefix = " ".join(text_seed) + " " if text_seed else ""
        output_str = prefix + tokenwrap(generated_tokens[:length])
        print(output_str)
        return output_str

    def plot(self, *args):
        
        return self.vocab().plot(*args)

    def vocab(self):
        
        if "_vocab" not in self.__dict__:
            
            self._vocab = FreqDist(self)
        return self._vocab

    def findall(self, regexp):
        

        if "_token_searcher" not in self.__dict__:
            self._token_searcher = TokenSearcher(self)

        hits = self._token_searcher.findall(regexp)
        hits = [" ".join(h) for h in hits]
        print(tokenwrap(hits, "; "))

    
    
    

    _CONTEXT_RE = re.compile(r"\w+|[\.\!\?]")

    def _context(self, tokens, i):
        
        
        j = i - 1
        while j >= 0 and not self._CONTEXT_RE.match(tokens[j]):
            j -= 1
        left = tokens[j] if j != 0 else "*START*"

        
        j = i + 1
        while j < len(tokens) and not self._CONTEXT_RE.match(tokens[j]):
            j += 1
        right = tokens[j] if j != len(tokens) else "*END*"

        return (left, right)

    
    
    

    def __str__(self):
        return "<Text: %s>" % self.name

    def __repr__(self):
        return "<Text: %s>" % self.name



class TextCollection(Text):
    

    def __init__(self, source):
        if hasattr(source, "words"):  
            source = [source.words(f) for f in source.fileids()]

        self._texts = source
        Text.__init__(self, LazyConcatenation(source))
        self._idf_cache = {}

    def tf(self, term, text):
        
        return text.count(term) / len(text)

    def idf(self, term):
        
        
        idf = self._idf_cache.get(term)
        if idf is None:
            matches = len([True for text in self._texts if term in text])
            if len(self._texts) == 0:
                raise ValueError("IDF undefined for empty document collection")
            idf = log(len(self._texts) / matches) if matches else 0.0
            self._idf_cache[term] = idf
        return idf

    def tf_idf(self, term, text):
        return self.tf(term, text) * self.idf(term)


def demo():
    from nltk.corpus import brown

    text = Text(brown.words(categories="news"))
    print(text)
    print()
    print("Concordance:")
    text.concordance("news")
    print()
    print("Distributionally similar words:")
    text.similar("news")
    print()
    print("Collocations:")
    text.collocations()
    print()
    
    
    
    print("Dispersion plot:")
    text.dispersion_plot(["news", "report", "said", "announced"])
    print()
    print("Vocabulary plot:")
    text.plot(50)
    print()
    print("Indexing:")
    print("text[3]:", text[3])
    print("text[3:5]:", text[3:5])
    print("text.vocab()['news']:", text.vocab()["news"])


if __name__ == "__main__":
    demo()

__all__ = [
    "ContextIndex",
    "ConcordanceIndex",
    "TokenSearcher",
    "Text",
    "TextCollection",
]