natural language toolkit c 20012023 nltk project ilia kurenkov ilia kurenkovgmail com url https www nltk org for license information see license txt language model vocabulary import sys from collections import counter from collections abc import iterable from functools import singledispatch from itertools import chain singledispatch def dispatchedlookupwords vocab raise typeerrorfunsupported type for looking up in vocabulary typewords dispatchedlookup registeriterable def words vocab return tupledispatchedlookupw vocab for w in words dispatchedlookup registerstr def stringlookupword vocab stores language model vocabulary satisfies two common language modeling requirements for a vocabulary when checking membership and calculating its size filters items by comparing their counts to a cutoff value adds a special unknown token which unseen words are mapped to words a c d c a b r a c d from nltk lm import vocabulary vocab vocabularywords unkcutoff2 tokens with counts greater than or equal to the cutoff value will be considered part of the vocabulary vocab c 3 c in vocab true vocab d 2 d in vocab true tokens with frequency counts less than the cutoff value will be considered not part of the vocabulary even though their entries in the count dictionary are preserved vocab b 1 b in vocab false vocab aliens 0 aliens in vocab false keeping the count entries for seen words allows us to change the cutoff value without having to recalculate the counts vocab2 vocabularyvocab counts unkcutoff1 b in vocab2 true the cutoff value influences not only membership checking but also the result of getting the size of the vocabulary using the builtin len note that while the number of keys in the vocabulary s counter stays the same the items in the vocabulary differ depending on the cutoff we use sorted to demonstrate because it keeps the order consistent sortedvocab2 counts a b c d r sortedvocab2 unk a b c d r sortedvocab counts a b c d r sortedvocab unk a c d in addition to items it gets populated with the vocabulary stores a special token that stands in for socalled unknown items by default it s unk unk in vocab true we can look up words in a vocabulary using its lookup method unseen words with counts less than cutoff are looked up as the unknown label if given one word a string as an input this method will return a string vocab lookupa a vocab lookupaliens unk if given a sequence it will return an tuple of the looked up words vocab lookupp a r d b c unk a unk d unk c it s possible to update the counts after the vocabulary has been created in general the interface is the same as that of collections counter vocab b 1 vocab updateb b c vocab b 3 create a new vocabulary param counts optional iterable or collections counter instance to preseed the vocabulary in case it is iterable counts are calculated param int unkcutoff words that occur less frequently than this value are not considered part of the vocabulary param unklabel label for marking words not part of vocabulary cutoff value items with count below this value are not considered part of vocabulary update vocabulary counts wraps collections counter update method look up one or more words in the vocabulary if passed one word as a string will return that word or self unklabel otherwise will assume it was passed a sequence of words will try to look each of them up and return an iterator over the looked up words param words words to look up type words iterablestr or str rtype generatorstr or str raises typeerror for types other than strings or iterables from nltk lm import vocabulary vocab vocabularya b c a b unkcutoff2 vocab lookupa a vocab lookupaliens unk vocab lookupa b c x b a b unk unk b only consider items with counts ge to cutoff as being in the vocabulary return selfitem self cutoff def iterself computing size of vocabulary reflects the cutoff return self len def eqself other return self unklabel other unklabel and self cutoff other cutoff and self counts other counts def strself return with cutoff unklabel and items format self class name self cutoff self unklabel lenself natural language toolkit c 2001 2023 nltk project ilia kurenkov ilia kurenkov gmail com url https www nltk org for license information see license txt language model vocabulary look up a sequence of words in the vocabulary returns an iterator over looked up words looks up one word in the vocabulary stores language model vocabulary satisfies two common language modeling requirements for a vocabulary when checking membership and calculating its size filters items by comparing their counts to a cutoff value adds a special unknown token which unseen words are mapped to words a c d c a b r a c d from nltk lm import vocabulary vocab vocabulary words unk_cutoff 2 tokens with counts greater than or equal to the cutoff value will be considered part of the vocabulary vocab c 3 c in vocab true vocab d 2 d in vocab true tokens with frequency counts less than the cutoff value will be considered not part of the vocabulary even though their entries in the count dictionary are preserved vocab b 1 b in vocab false vocab aliens 0 aliens in vocab false keeping the count entries for seen words allows us to change the cutoff value without having to recalculate the counts vocab2 vocabulary vocab counts unk_cutoff 1 b in vocab2 true the cutoff value influences not only membership checking but also the result of getting the size of the vocabulary using the built in len note that while the number of keys in the vocabulary s counter stays the same the items in the vocabulary differ depending on the cutoff we use sorted to demonstrate because it keeps the order consistent sorted vocab2 counts a b c d r sorted vocab2 unk a b c d r sorted vocab counts a b c d r sorted vocab unk a c d in addition to items it gets populated with the vocabulary stores a special token that stands in for so called unknown items by default it s unk unk in vocab true we can look up words in a vocabulary using its lookup method unseen words with counts less than cutoff are looked up as the unknown label if given one word a string as an input this method will return a string vocab lookup a a vocab lookup aliens unk if given a sequence it will return an tuple of the looked up words vocab lookup p a r d b c unk a unk d unk c it s possible to update the counts after the vocabulary has been created in general the interface is the same as that of collections counter vocab b 1 vocab update b b c vocab b 3 create a new vocabulary param counts optional iterable or collections counter instance to pre seed the vocabulary in case it is iterable counts are calculated param int unk_cutoff words that occur less frequently than this value are not considered part of the vocabulary param unk_label label for marking words not part of vocabulary cutoff value items with count below this value are not considered part of vocabulary update vocabulary counts wraps collections counter update method look up one or more words in the vocabulary if passed one word as a string will return that word or self unk_label otherwise will assume it was passed a sequence of words will try to look each of them up and return an iterator over the looked up words param words word s to look up type words iterable str or str rtype generator str or str raises typeerror for types other than strings or iterables from nltk lm import vocabulary vocab vocabulary a b c a b unk_cutoff 2 vocab lookup a a vocab lookup aliens unk vocab lookup a b c x b a b unk unk b only consider items with counts ge to cutoff as being in the vocabulary building on membership check define how to iterate over vocabulary computing size of vocabulary reflects the cutoff
import sys
from collections import Counter
from collections.abc import Iterable
from functools import singledispatch
from itertools import chain


@singledispatch
def _dispatched_lookup(words, vocab):
    raise TypeError(f"Unsupported type for looking up in vocabulary: {type(words)}")


@_dispatched_lookup.register(Iterable)
def _(words, vocab):
    
    return tuple(_dispatched_lookup(w, vocab) for w in words)


@_dispatched_lookup.register(str)
def _string_lookup(word, vocab):
    
    return word if word in vocab else vocab.unk_label


class Vocabulary:
    

    def __init__(self, counts=None, unk_cutoff=1, unk_label="<UNK>"):
        
        self.unk_label = unk_label
        if unk_cutoff < 1:
            raise ValueError(f"Cutoff value cannot be less than 1. Got: {unk_cutoff}")
        self._cutoff = unk_cutoff

        self.counts = Counter()
        self.update(counts if counts is not None else "")

    @property
    def cutoff(self):
        
        return self._cutoff

    def update(self, *counter_args, **counter_kwargs):
        
        self.counts.update(*counter_args, **counter_kwargs)
        self._len = sum(1 for _ in self)

    def lookup(self, words):
        
        return _dispatched_lookup(words, self)

    def __getitem__(self, item):
        return self._cutoff if item == self.unk_label else self.counts[item]

    def __contains__(self, item):
        
        return self[item] >= self.cutoff

    def __iter__(self):
        
        return chain(
            (item for item in self.counts if item in self),
            [self.unk_label] if self.counts else [],
        )

    def __len__(self):
        
        return self._len

    def __eq__(self, other):
        return (
            self.unk_label == other.unk_label
            and self.cutoff == other.cutoff
            and self.counts == other.counts
        )

    def __str__(self):
        return "<{} with cutoff={} unk_label='{}' and {} items>".format(
            self.__class__.__name__, self.cutoff, self.unk_label, len(self)
        )