natural language toolkit language models c 20012023 nltk project ilia kurenkov ilia kurenkovgmail com manu joseph manujosephvgmail com url https www nltk org for license information see license txt language models from nltk lm api import languagemodel smoothing from nltk lm smoothing import absolutediscounting kneserney wittenbell class mlelanguagemodel def unmaskedscoreself word contextnone return self contextcountscontext freqword class lidstonelanguagemodel def initself gamma args kwargs super initargs kwargs self gamma gamma def unmaskedscoreself word contextnone counts self contextcountscontext wordcount countsword normcount counts n return wordcount self gamma normcount lenself vocab self gamma class laplacelidstone def initself args kwargs super init1 args kwargs class stupidbackofflanguagemodel def initself alpha0 4 args kwargs super initargs kwargs self alpha alpha def unmaskedscoreself word contextnone if not context base recursion return self counts unigrams freqword counts self contextcountscontext wordcount countsword normcount counts n if wordcount 0 return wordcount normcount else return self alpha self unmaskedscoreword context1 class interpolatedlanguagemodellanguagemodel def initself smoothingcls order kwargs params kwargs popparams super initorder kwargs self estimator smoothingclsself vocab self counts params def unmaskedscoreself word contextnone if not context the base recursion case no context we only have a unigram return self estimator unigramscoreword if not self countscontext it can also happen that we have no data for this context in that case we defer to the lowerorder ngram this is the same as setting alpha to 0 and gamma to 1 alpha gamma 0 1 else alpha gamma self estimator alphagammaword context return alpha gamma self unmaskedscoreword context1 class wittenbellinterpolatedinterpolatedlanguagemodel interpolated version of smoothing with absolute discount def initself order discount0 75 kwargs super init absolutediscounting order paramsdiscount discount kwargs class kneserneyinterpolatedinterpolatedlanguagemodel natural language toolkit language models c 2001 2023 nltk project ilia kurenkov ilia kurenkov gmail com manu joseph manujosephv gmail com url https www nltk org for license information see license txt language models class for providing mle ngram model scores inherits initialization from basengrammodel returns the mle score for a word given a context args word is expected to be a string context is expected to be something reasonably convertible to a tuple provides lidstone smoothed scores in addition to initialization arguments from basengrammodel also requires a number by which to increase the counts gamma add one smoothing lidstone or laplace to see what kind look at gamma attribute on the class implements laplace add one smoothing initialization identical to basengrammodel because gamma is always 1 provides stupidbackoff scores in addition to initialization arguments from basengrammodel also requires a parameter alpha with which we scale the lower order probabilities note that this is not a true probability distribution as scores for ngrams of the same order do not sum up to unity base recursion logic common to all interpolated language models the idea to abstract this comes from chen goodman 1995 do not instantiate this class directly the base recursion case no context we only have a unigram it can also happen that we have no data for this context in that case we defer to the lower order ngram this is the same as setting alpha to 0 and gamma to 1 interpolated version of witten bell smoothing interpolated version of smoothing with absolute discount interpolated version of kneser ney smoothing
from nltk.lm.api import LanguageModel, Smoothing
from nltk.lm.smoothing import AbsoluteDiscounting, KneserNey, WittenBell


class MLE(LanguageModel):
    

    def unmasked_score(self, word, context=None):
        
        return self.context_counts(context).freq(word)


class Lidstone(LanguageModel):
    

    def __init__(self, gamma, *args, **kwargs):
        super().__init__(*args, **kwargs)
        self.gamma = gamma

    def unmasked_score(self, word, context=None):
        
        counts = self.context_counts(context)
        word_count = counts[word]
        norm_count = counts.N()
        return (word_count + self.gamma) / (norm_count + len(self.vocab) * self.gamma)


class Laplace(Lidstone):
    

    def __init__(self, *args, **kwargs):
        super().__init__(1, *args, **kwargs)


class StupidBackoff(LanguageModel):
    

    def __init__(self, alpha=0.4, *args, **kwargs):
        super().__init__(*args, **kwargs)
        self.alpha = alpha

    def unmasked_score(self, word, context=None):
        if not context:
            
            return self.counts.unigrams.freq(word)
        counts = self.context_counts(context)
        word_count = counts[word]
        norm_count = counts.N()
        if word_count > 0:
            return word_count / norm_count
        else:
            return self.alpha * self.unmasked_score(word, context[1:])


class InterpolatedLanguageModel(LanguageModel):
    

    def __init__(self, smoothing_cls, order, **kwargs):
        params = kwargs.pop("params", {})
        super().__init__(order, **kwargs)
        self.estimator = smoothing_cls(self.vocab, self.counts, **params)

    def unmasked_score(self, word, context=None):
        if not context:
            
            return self.estimator.unigram_score(word)
        if not self.counts[context]:
            
            
            
            alpha, gamma = 0, 1
        else:
            alpha, gamma = self.estimator.alpha_gamma(word, context)
        return alpha + gamma * self.unmasked_score(word, context[1:])


class WittenBellInterpolated(InterpolatedLanguageModel):
    

    def __init__(self, order, **kwargs):
        super().__init__(WittenBell, order, **kwargs)


class AbsoluteDiscountingInterpolated(InterpolatedLanguageModel):
    

    def __init__(self, order, discount=0.75, **kwargs):
        super().__init__(
            AbsoluteDiscounting, order, params={"discount": discount}, **kwargs
        )


class KneserNeyInterpolated(InterpolatedLanguageModel):
    

    def __init__(self, order, discount=0.1, **kwargs):
        if not (0 <= discount <= 1):
            raise ValueError(
                "Discount must be between 0 and 1 for probabilities to sum to unity."
            )
        super().__init__(
            KneserNey, order, params={"discount": discount, "order": order}, **kwargs
        )