natural language toolkit language models c 20012023 nltk project s ilia kurenkov ilia kurenkovgmail com url https www nltk org for license information see license txt language model interface import random import warnings from abc import abcmeta abstractmethod from bisect import bisect from itertools import accumulate from nltk lm counter import ngramcounter from nltk lm util import logbase2 from nltk lm vocabulary import vocabulary class smoothingmetaclassabcmeta def initself vocabulary counter self vocab vocabulary self counts counter abstractmethod def unigramscoreself word raise notimplementederror abstractmethod def alphagammaself word context raise notimplementederror def meanitems like random choice but with weights heavily inspired by python 3 6 random choices abc for language models cannot be directly instantiated itself creates new languagemodel param vocabulary if provided this vocabulary will be used instead of creating a new one when training type vocabulary nltk lm vocabulary or none param counter if provided use this object to count ngrams type counter nltk lm ngramcounter or none param ngramsfn if given defines how sentences in training text are turned to ngram sequences type ngramsfn function or none param padfn if given defines how sentences in training text are padded type padfn function or none trains the model on a text param text training text as a sequence of sentences masks out of vocab oov words and computes their model score for modelspecific logic of calculating scores see the unmaskedscore method score a word given some optional context concrete models are expected to provide an implementation note that this method does not mask its arguments with the oov label use the score method for that param str word word for which we want the score param tuplestr context context the word is in if none compute unigram score param context tuplestr or none rtype float evaluate the log score of this word in this context the arguments are the same as for score and unmaskedscore helper method for retrieving counts for a given context assumes context has been checked and oov words in it masked type context tuplestr or none calculate crossentropy of model for given evaluation text param iterabletuplestr textngrams a sequence of ngram tuples rtype float calculates the perplexity of the given text this is simply 2 crossentropy for the text so the arguments are the same generate words from the model param int numwords how many words to generate by default 1 param textseed generation can be conditioned on preceding context param randomseed a random seed or an instance of random random if provided makes the random sampling part of generation reproducible return one str word or a list of words generated from model examples from nltk lm import mle lm mle2 lm fita b b c vocabularytext a b c lm fita b c lm generaterandomseed3 a lm generatetextseed a b this is the base recursion case sorting samples achieves two things reproducible randomness when sampling turns mapping into sequence which weightedchoice expects we build up text one word at a time using the preceding context natural language toolkit language models c 2001 2023 nltk project s ilia kurenkov ilia kurenkov gmail com url https www nltk org for license information see license txt language model interface ngram smoothing interface implements chen goodman 1995 s idea that all smoothing algorithms have certain features in common this should ideally allow smoothing algorithms to work both with backoff and interpolation param vocabulary the ngram vocabulary object type vocabulary nltk lm vocab vocabulary param counter the counts of the vocabulary items type counter nltk lm counter ngramcounter return average aka mean for sequence of items like random choice but with weights heavily inspired by python 3 6 random choices abc for language models cannot be directly instantiated itself creates new languagemodel param vocabulary if provided this vocabulary will be used instead of creating a new one when training type vocabulary nltk lm vocabulary or none param counter if provided use this object to count ngrams type counter nltk lm ngramcounter or none param ngrams_fn if given defines how sentences in training text are turned to ngram sequences type ngrams_fn function or none param pad_fn if given defines how sentences in training text are padded type pad_fn function or none trains the model on a text param text training text as a sequence of sentences masks out of vocab oov words and computes their model score for model specific logic of calculating scores see the unmasked_score method score a word given some optional context concrete models are expected to provide an implementation note that this method does not mask its arguments with the oov label use the score method for that param str word word for which we want the score param tuple str context context the word is in if none compute unigram score param context tuple str or none rtype float evaluate the log score of this word in this context the arguments are the same as for score and unmasked_score helper method for retrieving counts for a given context assumes context has been checked and oov words in it masked type context tuple str or none calculate cross entropy of model for given evaluation text param iterable tuple str text_ngrams a sequence of ngram tuples rtype float calculates the perplexity of the given text this is simply 2 cross entropy for the text so the arguments are the same generate words from the model param int num_words how many words to generate by default 1 param text_seed generation can be conditioned on preceding context param random_seed a random seed or an instance of random random if provided makes the random sampling part of generation reproducible return one str word or a list of words generated from model examples from nltk lm import mle lm mle 2 lm fit a b b c vocabulary_text a b c lm fit a b c lm generate random_seed 3 a lm generate text_seed a b this is the base recursion case sorting samples achieves two things reproducible randomness when sampling turns mapping into sequence which _weighted_choice expects we build up text one word at a time using the preceding context
import random
import warnings
from abc import ABCMeta, abstractmethod
from bisect import bisect
from itertools import accumulate

from nltk.lm.counter import NgramCounter
from nltk.lm.util import log_base2
from nltk.lm.vocabulary import Vocabulary


class Smoothing(metaclass=ABCMeta):
    

    def __init__(self, vocabulary, counter):
        
        self.vocab = vocabulary
        self.counts = counter

    @abstractmethod
    def unigram_score(self, word):
        raise NotImplementedError()

    @abstractmethod
    def alpha_gamma(self, word, context):
        raise NotImplementedError()


def _mean(items):
    
    return sum(items) / len(items)


def _random_generator(seed_or_generator):
    if isinstance(seed_or_generator, random.Random):
        return seed_or_generator
    return random.Random(seed_or_generator)


def _weighted_choice(population, weights, random_generator=None):
    
    if not population:
        raise ValueError("Can't choose from empty population")
    if len(population) != len(weights):
        raise ValueError("The number of weights does not match the population")
    cum_weights = list(accumulate(weights))
    total = cum_weights[-1]
    threshold = random_generator.random()
    return population[bisect(cum_weights, total * threshold)]


class LanguageModel(metaclass=ABCMeta):
    

    def __init__(self, order, vocabulary=None, counter=None):
        
        self.order = order
        if vocabulary and not isinstance(vocabulary, Vocabulary):
            warnings.warn(
                f"The `vocabulary` argument passed to {self.__class__.__name__!r} "
                "must be an instance of `nltk.lm.Vocabulary`.",
                stacklevel=3,
            )
        self.vocab = Vocabulary() if vocabulary is None else vocabulary
        self.counts = NgramCounter() if counter is None else counter

    def fit(self, text, vocabulary_text=None):
        
        if not self.vocab:
            if vocabulary_text is None:
                raise ValueError(
                    "Cannot fit without a vocabulary or text to create it from."
                )
            self.vocab.update(vocabulary_text)
        self.counts.update(self.vocab.lookup(sent) for sent in text)

    def score(self, word, context=None):
        
        return self.unmasked_score(
            self.vocab.lookup(word), self.vocab.lookup(context) if context else None
        )

    @abstractmethod
    def unmasked_score(self, word, context=None):
        
        raise NotImplementedError()

    def logscore(self, word, context=None):
        
        return log_base2(self.score(word, context))

    def context_counts(self, context):
        
        return (
            self.counts[len(context) + 1][context] if context else self.counts.unigrams
        )

    def entropy(self, text_ngrams):
        
        return -1 * _mean(
            [self.logscore(ngram[-1], ngram[:-1]) for ngram in text_ngrams]
        )

    def perplexity(self, text_ngrams):
        
        return pow(2.0, self.entropy(text_ngrams))

    def generate(self, num_words=1, text_seed=None, random_seed=None):
        
        text_seed = [] if text_seed is None else list(text_seed)
        random_generator = _random_generator(random_seed)
        
        if num_words == 1:
            context = (
                text_seed[-self.order + 1 :]
                if len(text_seed) >= self.order
                else text_seed
            )
            samples = self.context_counts(self.vocab.lookup(context))
            while context and not samples:
                context = context[1:] if len(context) > 1 else []
                samples = self.context_counts(self.vocab.lookup(context))
            
            
            
            samples = sorted(samples)
            return _weighted_choice(
                samples,
                tuple(self.score(w, context) for w in samples),
                random_generator,
            )
        
        generated = []
        for _ in range(num_words):
            generated.append(
                self.generate(
                    num_words=1,
                    text_seed=text_seed + generated,
                    random_seed=random_generator,
                )
            )
        return generated