natural language toolkit naive bayes classifiers c 20012023 nltk project edward loper edlopergmail com url https www nltk org for license information see license txt a classifier based on the naive bayes algorithm in order to find the probability for a label this algorithm first uses the bayes rule to express plabelfeatures in terms of plabel and pfeatureslabel plabel pfeatureslabel plabelfeatures pfeatures the algorithm then makes the naive assumption that all features are independent given the label plabel pf1label pfnlabel plabelfeatures pfeatures rather than computing pfeatures explicitly the algorithm just calculates the numerator for each label and normalizes them so they sum to one plabel pf1label pfnlabel plabelfeatures suml pl pf1l pfnl naive bayes classifier a naive bayes classifier naive bayes classifiers are paramaterized by two probability distributions plabel gives the probability that an input will receive each label given no information about the input s features pfnamefvallabel gives the probability that a given feature fname will receive a given value fval given that the label label if the classifier encounters an input with a feature that has never been seen with any label then rather than assigning a probability of 0 to all labels it will ignore that feature the feature value none is reserved for unseen feature values you generally should not use none as a feature value for one of your own features param labelprobdist plabel the probability distribution over labels it is expressed as a probdisti whose samples are labels i e plabel labelprobdist problabel param featureprobdist pfnamefvallabel the probability distribution for feature values given labels it is expressed as a dictionary whose keys are label fname pairs and whose values are probdisti objects over feature values i e pfnamefvallabel featureprobdistlabel fname probfval if a given label fname is not a key in featureprobdist then it is assumed that the corresponding pfnamefvallabel is 0 for all values of fval discard any feature names that we ve never seen before otherwise we ll just assign a probability of 0 to everything print ignoring unseen feature s fname find the log probability of each label given the features start with the log probability of the label itself then add in the log probability of features given labels nb this case will never come up if the classifier was naivebayesclassifier train determine the most relevant features and display them return a list of the most informative features used by this classifier for the purpose of this function the informativeness of a feature fname fval is equal to the highest value of pfnamefvallabel for any label divided by the lowest value of pfnamefvallabel for any label max pfnamefvallabel1 pfnamefvallabel2 the set of fname fval pairs used by this classifier the max min probability associated w each fname fval pair maps fname fval float convert features to a list sort it by how informative features are param labeledfeaturesets a list of classified featuresets i e a list of tuples featureset label count up how many times each feature value occurred given the label and featurename increment freqfvallabel fname record that fname can take the value fval keep a list of all feature names if a feature didn t have a value given for an instance then we assume that it gets the implicit value none this loop counts up the number of missing feature values for each label fname pair and increments the count of the fval none by that amount only add a none key when necessary i e if there are any samples with feature fname missing create the plabel distribution create the pfvallabel fname distribution demo natural language toolkit naive bayes classifiers c 2001 2023 nltk project edward loper edloper gmail com url https www nltk org for license information see license txt a classifier based on the naive bayes algorithm in order to find the probability for a label this algorithm first uses the bayes rule to express p label features in terms of p label and p features label p label p features label p label features p features the algorithm then makes the naive assumption that all features are independent given the label p label p f1 label p fn label p label features p features rather than computing p features explicitly the algorithm just calculates the numerator for each label and normalizes them so they sum to one p label p f1 label p fn label p label features sum l p l p f1 l p fn l naive bayes classifier a naive bayes classifier naive bayes classifiers are paramaterized by two probability distributions p label gives the probability that an input will receive each label given no information about the input s features p fname fval label gives the probability that a given feature fname will receive a given value fval given that the label label if the classifier encounters an input with a feature that has never been seen with any label then rather than assigning a probability of 0 to all labels it will ignore that feature the feature value none is reserved for unseen feature values you generally should not use none as a feature value for one of your own features param label_probdist p label the probability distribution over labels it is expressed as a probdisti whose samples are labels i e p label label_probdist prob label param feature_probdist p fname fval label the probability distribution for feature values given labels it is expressed as a dictionary whose keys are label fname pairs and whose values are probdisti objects over feature values i e p fname fval label feature_probdist label fname prob fval if a given label fname is not a key in feature_probdist then it is assumed that the corresponding p fname fval label is 0 for all values of fval discard any feature names that we ve never seen before otherwise we ll just assign a probability of 0 to everything print ignoring unseen feature s fname find the log probability of each label given the features start with the log probability of the label itself then add in the log probability of features given labels nb this case will never come up if the classifier was naivebayesclassifier train inf determine the most relevant features and display them return a list of the most informative features used by this classifier for the purpose of this function the informativeness of a feature fname fval is equal to the highest value of p fname fval label for any label divided by the lowest value of p fname fval label for any label max p fname fval label1 p fname fval label2 the set of fname fval pairs used by this classifier the max min probability associated w each fname fval pair maps fname fval float convert features to a list sort it by how informative features are param labeled_featuresets a list of classified featuresets i e a list of tuples featureset label count up how many times each feature value occurred given the label and featurename increment freq fval label fname record that fname can take the value fval keep a list of all feature names if a feature didn t have a value given for an instance then we assume that it gets the implicit value none this loop counts up the number of missing feature values for each label fname pair and increments the count of the fval none by that amount only add a none key when necessary i e if there are any samples with feature fname missing create the p label distribution create the p fval label fname distribution demo
from collections import defaultdict

from nltk.classify.api import ClassifierI
from nltk.probability import DictionaryProbDist, ELEProbDist, FreqDist, sum_logs






class NaiveBayesClassifier(ClassifierI):
    

    def __init__(self, label_probdist, feature_probdist):
        
        self._label_probdist = label_probdist
        self._feature_probdist = feature_probdist
        self._labels = list(label_probdist.samples())

    def labels(self):
        return self._labels

    def classify(self, featureset):
        return self.prob_classify(featureset).max()

    def prob_classify(self, featureset):
        
        
        
        featureset = featureset.copy()
        for fname in list(featureset.keys()):
            for label in self._labels:
                if (label, fname) in self._feature_probdist:
                    break
            else:
                
                del featureset[fname]

        
        
        logprob = {}
        for label in self._labels:
            logprob[label] = self._label_probdist.logprob(label)

        
        for label in self._labels:
            for (fname, fval) in featureset.items():
                if (label, fname) in self._feature_probdist:
                    feature_probs = self._feature_probdist[label, fname]
                    logprob[label] += feature_probs.logprob(fval)
                else:
                    
                    
                    
                    logprob[label] += sum_logs([])  

        return DictionaryProbDist(logprob, normalize=True, log=True)

    def show_most_informative_features(self, n=10):
        
        cpdist = self._feature_probdist
        print("Most Informative Features")

        for (fname, fval) in self.most_informative_features(n):

            def labelprob(l):
                return cpdist[l, fname].prob(fval)

            labels = sorted(
                (l for l in self._labels if fval in cpdist[l, fname].samples()),
                key=lambda element: (-labelprob(element), element),
                reverse=True,
            )
            if len(labels) == 1:
                continue
            l0 = labels[0]
            l1 = labels[-1]
            if cpdist[l0, fname].prob(fval) == 0:
                ratio = "INF"
            else:
                ratio = "%8.1f" % (
                    cpdist[l1, fname].prob(fval) / cpdist[l0, fname].prob(fval)
                )
            print(
                "%24s = %-14r %6s : %-6s = %s : 1.0"
                % (fname, fval, ("%s" % l1)[:6], ("%s" % l0)[:6], ratio)
            )

    def most_informative_features(self, n=100):
        
        if hasattr(self, "_most_informative_features"):
            return self._most_informative_features[:n]
        else:
            
            features = set()
            
            
            maxprob = defaultdict(lambda: 0.0)
            minprob = defaultdict(lambda: 1.0)

            for (label, fname), probdist in self._feature_probdist.items():
                for fval in probdist.samples():
                    feature = (fname, fval)
                    features.add(feature)
                    p = probdist.prob(fval)
                    maxprob[feature] = max(p, maxprob[feature])
                    minprob[feature] = min(p, minprob[feature])
                    if minprob[feature] == 0:
                        features.discard(feature)

            
            
            self._most_informative_features = sorted(
                features,
                key=lambda feature_: (
                    minprob[feature_] / maxprob[feature_],
                    feature_[0],
                    feature_[1] in [None, False, True],
                    str(feature_[1]).lower(),
                ),
            )
        return self._most_informative_features[:n]

    @classmethod
    def train(cls, labeled_featuresets, estimator=ELEProbDist):
        
        label_freqdist = FreqDist()
        feature_freqdist = defaultdict(FreqDist)
        feature_values = defaultdict(set)
        fnames = set()

        
        
        for featureset, label in labeled_featuresets:
            label_freqdist[label] += 1
            for fname, fval in featureset.items():
                
                feature_freqdist[label, fname][fval] += 1
                
                feature_values[fname].add(fval)
                
                fnames.add(fname)

        
        
        
        
        
        for label in label_freqdist:
            num_samples = label_freqdist[label]
            for fname in fnames:
                count = feature_freqdist[label, fname].N()
                
                
                if num_samples - count > 0:
                    feature_freqdist[label, fname][None] += num_samples - count
                    feature_values[fname].add(None)

        
        label_probdist = estimator(label_freqdist)

        
        feature_probdist = {}
        for ((label, fname), freqdist) in feature_freqdist.items():
            probdist = estimator(freqdist, bins=len(feature_values[fname]))
            feature_probdist[label, fname] = probdist

        return cls(label_probdist, feature_probdist)







def demo():
    from nltk.classify.util import names_demo

    classifier = names_demo(NaiveBayesClassifier.train)
    classifier.show_most_informative_features()


if __name__ == "__main__":
    demo()