natural language toolkit language model unit tests c 20012023 nltk project ilia kurenkov ilia kurenkovgmail com manu joseph manujosephvgmail com url https www nltk org for license information see license txt smoothing algorithms for language modeling according to chen goodman 1995 these should work with both backoff and interpolation count values that are greater than zero in a distribution assumes distribution is either a mapping with counts as values or an instance of nltk conditionalfreqdist we explicitly check that values are 0 to guard against negative counts wittenbell smoothing def initself vocabulary counter kwargs super initvocabulary counter kwargs def alphagammaself word context alpha self countscontext freqword gamma self gammacontext return 1 0 gamma alpha gamma def gammaself context nplus countvaluesgtzeroself countscontext return nplus nplus self countscontext n def unigramscoreself word return self counts unigrams freqword class absolutediscountingsmoothing kneserney smoothing this is an extension of smoothing with a discount resources https pages ucsd edurlevylign256winter2008kneserneyminiexample pdf https www youtube comwatch vody1ysutd7o https medium comdennycasimplenumericalexampleforkneserneysmoothingnlp4600addf38b8 https www cl uniheidelberg decoursesss15smtscribe6 pdf https wwwi6 informatik rwthaachen depublicationsdownload951knesericassp1995 pdf count continuations that end with context and word continuations track unique ngram types regardless of how many instances were observed for each type this is different than raw ngram counts which track number of instances natural language toolkit language model unit tests c 2001 2023 nltk project ilia kurenkov ilia kurenkov gmail com manu joseph manujosephv gmail com url https www nltk org for license information see license txt smoothing algorithms for language modeling according to chen goodman 1995 these should work with both backoff and interpolation count values that are greater than zero in a distribution assumes distribution is either a mapping with counts as values or an instance of nltk conditionalfreqdist we explicitly check that values are 0 to guard against negative counts witten bell smoothing smoothing with absolute discount kneser ney smoothing this is an extension of smoothing with a discount resources https pages ucsd edu rlevy lign256 winter2008 kneser_ney_mini_example pdf https www youtube com watch v ody1ysutd7o https medium com dennyc a simple numerical example for kneser ney smoothing nlp 4600addf38b8 https www cl uni heidelberg de courses ss15 smt scribe6 pdf https www i6 informatik rwth aachen de publications download 951 kneser icassp 1995 pdf count continuations that end with context and word continuations track unique ngram types regardless of how many instances were observed for each type this is different than raw ngram counts which track number of instances
from operator import methodcaller

from nltk.lm.api import Smoothing
from nltk.probability import ConditionalFreqDist


def _count_values_gt_zero(distribution):
    
    as_count = (
        methodcaller("N")
        if isinstance(distribution, ConditionalFreqDist)
        else lambda count: count
    )
    
    return sum(
        1 for dist_or_count in distribution.values() if as_count(dist_or_count) > 0
    )


class WittenBell(Smoothing):
    

    def __init__(self, vocabulary, counter, **kwargs):
        super().__init__(vocabulary, counter, **kwargs)

    def alpha_gamma(self, word, context):
        alpha = self.counts[context].freq(word)
        gamma = self._gamma(context)
        return (1.0 - gamma) * alpha, gamma

    def _gamma(self, context):
        n_plus = _count_values_gt_zero(self.counts[context])
        return n_plus / (n_plus + self.counts[context].N())

    def unigram_score(self, word):
        return self.counts.unigrams.freq(word)


class AbsoluteDiscounting(Smoothing):
    

    def __init__(self, vocabulary, counter, discount=0.75, **kwargs):
        super().__init__(vocabulary, counter, **kwargs)
        self.discount = discount

    def alpha_gamma(self, word, context):
        alpha = (
            max(self.counts[context][word] - self.discount, 0)
            / self.counts[context].N()
        )
        gamma = self._gamma(context)
        return alpha, gamma

    def _gamma(self, context):
        n_plus = _count_values_gt_zero(self.counts[context])
        return (self.discount * n_plus) / self.counts[context].N()

    def unigram_score(self, word):
        return self.counts.unigrams.freq(word)


class KneserNey(Smoothing):
    

    def __init__(self, vocabulary, counter, order, discount=0.1, **kwargs):
        super().__init__(vocabulary, counter, **kwargs)
        self.discount = discount
        self._order = order

    def unigram_score(self, word):
        word_continuation_count, total_count = self._continuation_counts(word)
        return word_continuation_count / total_count

    def alpha_gamma(self, word, context):
        prefix_counts = self.counts[context]
        word_continuation_count, total_count = (
            (prefix_counts[word], prefix_counts.N())
            if len(context) + 1 == self._order
            else self._continuation_counts(word, context)
        )
        alpha = max(word_continuation_count - self.discount, 0.0) / total_count
        gamma = self.discount * _count_values_gt_zero(prefix_counts) / total_count
        return alpha, gamma

    def _continuation_counts(self, word, context=tuple()):
        
        higher_order_ngrams_with_context = (
            counts
            for prefix_ngram, counts in self.counts[len(context) + 2].items()
            if prefix_ngram[1:] == context
        )
        higher_order_ngrams_with_word_count, total = 0, 0
        for counts in higher_order_ngrams_with_context:
            higher_order_ngrams_with_word_count += int(counts[word] > 0)
            total += _count_values_gt_zero(counts)
        return higher_order_ngrams_with_word_count, total