natural language toolkit language model unit tests c 20012023 nltk project ilia kurenkov ilia kurenkovgmail com url https www nltk org for license information see license txt tests for ngramcounter that only involve lookup no modification classmethod def setupclassself text listabcd listegdbe self trigramcounter ngramcounter everygramssent maxlen3 for sent in text self bigramcounter ngramcountereverygramssent maxlen2 for sent in text self case unittest testcase def testnself assert self bigramcounter n 16 assert self trigramcounter n 21 def testcounterlenchangeswithlookupself assert lenself bigramcounter 2 self bigramcounter50 assert lenself bigramcounter 3 def testngramorderaccessunigramsself assert self bigramcounter1 self bigramcounter unigrams def testngramconditionalfreqdistself case unittest testcase expectedtrigramcontexts a b b c e g g d d b expectedbigramcontexts a b d e c g bigrams self trigramcounter2 trigrams self trigramcounter3 self case assertcountequalexpectedbigramcontexts bigrams conditions self case assertcountequalexpectedtrigramcontexts trigrams conditions def testbigramcountsseenngramsself assert self bigramcounterab 1 assert self bigramcounterbc 1 def testbigramcountsunseenngramsself assert self bigramcounterbz 0 def testunigramcountsseenwordsself assert self bigramcounterb 2 def testunigramcountscompletelyunseenwordsself assert self bigramcounterz 0 class testngramcountertraining classmethod def setupclassself self counter ngramcounter self case unittest testcase pytest mark parametrizecase none def testemptyinputsself case test ngramcountercase assert 2 not in test assert test1 freqdist def testtrainonunigramsself words listabcd counter ngramcounterw for w in words assert not counter3 assert not counter2 self case assertcountequalwords counter1 keys def testtrainonillegalsentencesself strsent check this out listsent check this this out out with pytest raisestypeerror ngramcounterstrsent with pytest raisestypeerror ngramcounterlistsent def testtrainonbigramsself bigramsent a b c d counter ngramcounterbigramsent assert not boolcounter3 def testtrainonmixself mixedsent a b c d e f g h counter ngramcountermixedsent unigrams h bigramcontexts a c trigramcontexts e f self case assertcountequalunigrams counter1 keys self case assertcountequalbigramcontexts counter2 keys self case assertcountequaltrigramcontexts counter3 keys natural language toolkit language model unit tests c 2001 2023 nltk project ilia kurenkov ilia kurenkov gmail com url https www nltk org for license information see license txt tests for ngramcounter that only involve lookup no modification
import unittest

import pytest

from nltk import FreqDist
from nltk.lm import NgramCounter
from nltk.util import everygrams


class TestNgramCounter:
    

    @classmethod
    def setup_class(self):
        text = [list("abcd"), list("egdbe")]
        self.trigram_counter = NgramCounter(
            everygrams(sent, max_len=3) for sent in text
        )
        self.bigram_counter = NgramCounter(everygrams(sent, max_len=2) for sent in text)
        self.case = unittest.TestCase()

    def test_N(self):
        assert self.bigram_counter.N() == 16
        assert self.trigram_counter.N() == 21

    def test_counter_len_changes_with_lookup(self):
        assert len(self.bigram_counter) == 2
        self.bigram_counter[50]
        assert len(self.bigram_counter) == 3

    def test_ngram_order_access_unigrams(self):
        assert self.bigram_counter[1] == self.bigram_counter.unigrams

    def test_ngram_conditional_freqdist(self):
        case = unittest.TestCase()
        expected_trigram_contexts = [
            ("a", "b"),
            ("b", "c"),
            ("e", "g"),
            ("g", "d"),
            ("d", "b"),
        ]
        expected_bigram_contexts = [("a",), ("b",), ("d",), ("e",), ("c",), ("g",)]

        bigrams = self.trigram_counter[2]
        trigrams = self.trigram_counter[3]

        self.case.assertCountEqual(expected_bigram_contexts, bigrams.conditions())
        self.case.assertCountEqual(expected_trigram_contexts, trigrams.conditions())

    def test_bigram_counts_seen_ngrams(self):
        assert self.bigram_counter[["a"]]["b"] == 1
        assert self.bigram_counter[["b"]]["c"] == 1

    def test_bigram_counts_unseen_ngrams(self):
        assert self.bigram_counter[["b"]]["z"] == 0

    def test_unigram_counts_seen_words(self):
        assert self.bigram_counter["b"] == 2

    def test_unigram_counts_completely_unseen_words(self):
        assert self.bigram_counter["z"] == 0


class TestNgramCounterTraining:
    @classmethod
    def setup_class(self):
        self.counter = NgramCounter()
        self.case = unittest.TestCase()

    @pytest.mark.parametrize("case", ["", [], None])
    def test_empty_inputs(self, case):
        test = NgramCounter(case)
        assert 2 not in test
        assert test[1] == FreqDist()

    def test_train_on_unigrams(self):
        words = list("abcd")
        counter = NgramCounter([[(w,) for w in words]])

        assert not counter[3]
        assert not counter[2]
        self.case.assertCountEqual(words, counter[1].keys())

    def test_train_on_illegal_sentences(self):
        str_sent = ["Check", "this", "out", "!"]
        list_sent = [["Check", "this"], ["this", "out"], ["out", "!"]]

        with pytest.raises(TypeError):
            NgramCounter([str_sent])

        with pytest.raises(TypeError):
            NgramCounter([list_sent])

    def test_train_on_bigrams(self):
        bigram_sent = [("a", "b"), ("c", "d")]
        counter = NgramCounter([bigram_sent])
        assert not bool(counter[3])

    def test_train_on_mix(self):
        mixed_sent = [("a", "b"), ("c", "d"), ("e", "f", "g"), ("h",)]
        counter = NgramCounter([mixed_sent])
        unigrams = ["h"]
        bigram_contexts = [("a",), ("c",)]
        trigram_contexts = [("e", "f")]

        self.case.assertCountEqual(unigrams, counter[1].keys())
        self.case.assertCountEqual(bigram_contexts, counter[2].keys())
        self.case.assertCountEqual(trigram_contexts, counter[3].keys())