natural language toolkit confusion matrices c 20012023 nltk project edward loper edlopergmail com steven bird stevenbird1gmail com tom aarsen url https www nltk org for license information see license txt the confusion matrix between a list of reference values and a corresponding list of test values entry r t of this matrix is a count of the number of times that the reference value r corresponds to the test value t e g from nltk metrics import confusionmatrix ref det nn vb det jj nn nn in det nn split test det vb vb det nn nn nn in det nn split cm confusionmatrixref test printcm nn nn 3 note that the diagonal entries ritj of this matrix corresponds to correct values and the offdiagonal entries correspond to incorrect values construct a new confusion matrix from a list of reference values and a corresponding list of test values type reference list param reference an ordered list of reference values type test list param test a list of values to compare against the corresponding reference values raise valueerror if reference and length do not have the same length get a list of all values construct a valueindex dictionary make a confusion matrix table a list of all values in reference or test a dictionary mapping values in self values to their indices the confusion matrix itself as a list of lists of counts the greatest count in self confusion used for printing the total number of values in the confusion matrix the number of correct ondiagonal values in the matrix return the number of times that value li was expected and value lj was given rtype int return a multiline string representation of this confusion matrix type truncate int param truncate if specified then only show the specified number of values any sorting e g sortbycount will be performed before truncation param sortbycount if true then sort by the count of each label in the reference data i e labels that occur more frequently in the reference label will be towards the left edge of the matrix and labels that occur less frequently will be towards the right edge todo add marginals construct a format string for row values construct a format string for matrix entries write the column values write a dividing line write the entries write a dividing line write a key given a value in the confusion matrix return the recall that corresponds to this value the recall is defined as r true positive true positive false positive and can loosely be considered the ratio of how often value was predicted correctly relative to how often value was the true result param value value used in the confusionmatrix return the recall corresponding to value rtype float number of times value was correct and also predicted number of times value was correct given a value in the confusion matrix return the precision that corresponds to this value the precision is defined as p true positive true positive false negative and can loosely be considered the ratio of how often value was predicted correctly relative to the number of predictions for value param value value used in the confusionmatrix return the precision corresponding to value rtype float number of times value was correct and also predicted number of times value was predicted given a value used in the confusion matrix return the fmeasure that corresponds to this value the fmeasure is the harmonic mean of the precision and recall weighted by alpha in particular given the precision p and recall r defined by p true positive true positive false negative r true positive true positive false positive the fmeasure is 1alphap 1alphar with alpha 0 5 this reduces to 2pr p r param value value used in the confusionmatrix param alpha ratio of the cost of false negative compared to false positives defaults to 0 5 where the costs are equal type alpha float return the fmeasure corresponding to value rtype float tabulate the recall precision and fmeasure for each value in this confusion matrix reference det nn vb det jj nn nn in det nn split test det vb vb det nn nn nn in det nn split cm confusionmatrixreference test printcm evaluate tag prec recall fmeasure det 1 0000 1 0000 1 0000 in 1 0000 1 0000 1 0000 jj 0 0000 0 0000 0 0000 nn 0 7500 0 7500 0 7500 vb 0 5000 1 0000 0 6667 blankline param alpha ratio of the cost of false negative compared to false positives as used in the fmeasure computation defaults to 0 5 where the costs are equal type alpha float param truncate if specified then only show the specified number of values any sorting e g sortbycount will be performed before truncation defaults to none type truncate int optional param sortbycount whether to sort the outputs on frequency in the reference label defaults to false type sortbycount bool optional return a tabulated recall precision and fmeasure string rtype str apply keyword parameters construct the header construct the body natural language toolkit confusion matrices c 2001 2023 nltk project edward loper edloper gmail com steven bird stevenbird1 gmail com tom aarsen url https www nltk org for license information see license txt the confusion matrix between a list of reference values and a corresponding list of test values entry r t of this matrix is a count of the number of times that the reference value r corresponds to the test value t e g from nltk metrics import confusionmatrix ref det nn vb det jj nn nn in det nn split test det vb vb det nn nn nn in det nn split cm confusionmatrix ref test print cm nn nn 3 note that the diagonal entries ri tj of this matrix corresponds to correct values and the off diagonal entries correspond to incorrect values construct a new confusion matrix from a list of reference values and a corresponding list of test values type reference list param reference an ordered list of reference values type test list param test a list of values to compare against the corresponding reference values raise valueerror if reference and length do not have the same length get a list of all values construct a value index dictionary make a confusion matrix table maximum confusion a list of all values in reference or test a dictionary mapping values in self _values to their indices the confusion matrix itself as a list of lists of counts the greatest count in self _confusion used for printing the total number of values in the confusion matrix the number of correct on diagonal values in the matrix return the number of times that value li was expected and value lj was given rtype int return a multi line string representation of this confusion matrix type truncate int param truncate if specified then only show the specified number of values any sorting e g sort_by_count will be performed before truncation param sort_by_count if true then sort by the count of each label in the reference data i e labels that occur more frequently in the reference label will be towards the left edge of the matrix and labels that occur less frequently will be towards the right edge todo add marginals construct a format string for row values construct a format string for matrix entries write the column values write a dividing line write the entries write a dividing line write a key given a value in the confusion matrix return the recall that corresponds to this value the recall is defined as r true positive true positive false positive and can loosely be considered the ratio of how often value was predicted correctly relative to how often value was the true result param value value used in the confusionmatrix return the recall corresponding to value rtype float number of times value was correct and also predicted number of times value was correct given a value in the confusion matrix return the precision that corresponds to this value the precision is defined as p true positive true positive false negative and can loosely be considered the ratio of how often value was predicted correctly relative to the number of predictions for value param value value used in the confusionmatrix return the precision corresponding to value rtype float number of times value was correct and also predicted number of times value was predicted given a value used in the confusion matrix return the f measure that corresponds to this value the f measure is the harmonic mean of the precision and recall weighted by alpha in particular given the precision p and recall r defined by p true positive true positive false negative r true positive true positive false positive the f measure is 1 alpha p 1 alpha r with alpha 0 5 this reduces to 2pr p r param value value used in the confusionmatrix param alpha ratio of the cost of false negative compared to false positives defaults to 0 5 where the costs are equal type alpha float return the f measure corresponding to value rtype float tabulate the recall precision and f measure for each value in this confusion matrix reference det nn vb det jj nn nn in det nn split test det vb vb det nn nn nn in det nn split cm confusionmatrix reference test print cm evaluate tag prec recall f measure det 1 0000 1 0000 1 0000 in 1 0000 1 0000 1 0000 jj 0 0000 0 0000 0 0000 nn 0 7500 0 7500 0 7500 vb 0 5000 1 0000 0 6667 blankline param alpha ratio of the cost of false negative compared to false positives as used in the f measure computation defaults to 0 5 where the costs are equal type alpha float param truncate if specified then only show the specified number of values any sorting e g sort_by_count will be performed before truncation defaults to none type truncate int optional param sort_by_count whether to sort the outputs on frequency in the reference label defaults to false type sort_by_count bool optional return a tabulated recall precision and f measure string rtype str apply keyword parameters construct the header construct the body
from nltk.probability import FreqDist


class ConfusionMatrix:
    

    def __init__(self, reference, test, sort_by_count=False):
        
        if len(reference) != len(test):
            raise ValueError("Lists must have the same length.")

        
        if sort_by_count:
            ref_fdist = FreqDist(reference)
            test_fdist = FreqDist(test)

            def key(v):
                return -(ref_fdist[v] + test_fdist[v])

            values = sorted(set(reference + test), key=key)
        else:
            values = sorted(set(reference + test))

        
        indices = {val: i for (i, val) in enumerate(values)}

        
        confusion = [[0 for _ in values] for _ in values]
        max_conf = 0  
        for w, g in zip(reference, test):
            confusion[indices[w]][indices[g]] += 1
            max_conf = max(max_conf, confusion[indices[w]][indices[g]])

        
        self._values = values
        
        self._indices = indices
        
        self._confusion = confusion
        
        self._max_conf = max_conf
        
        self._total = len(reference)
        
        self._correct = sum(confusion[i][i] for i in range(len(values)))

    def __getitem__(self, li_lj_tuple):
        
        (li, lj) = li_lj_tuple
        i = self._indices[li]
        j = self._indices[lj]
        return self._confusion[i][j]

    def __repr__(self):
        return f"<ConfusionMatrix: {self._correct}/{self._total} correct>"

    def __str__(self):
        return self.pretty_format()

    def pretty_format(
        self,
        show_percents=False,
        values_in_chart=True,
        truncate=None,
        sort_by_count=False,
    ):
        
        confusion = self._confusion

        values = self._values
        if sort_by_count:
            values = sorted(
                values, key=lambda v: -sum(self._confusion[self._indices[v]])
            )

        if truncate:
            values = values[:truncate]

        if values_in_chart:
            value_strings = ["%s" % val for val in values]
        else:
            value_strings = [str(n + 1) for n in range(len(values))]

        
        valuelen = max(len(val) for val in value_strings)
        value_format = "%" + repr(valuelen) + "s | "
        
        if show_percents:
            entrylen = 6
            entry_format = "%5.1f%%"
            zerostr = "     ."
        else:
            entrylen = len(repr(self._max_conf))
            entry_format = "%" + repr(entrylen) + "d"
            zerostr = " " * (entrylen - 1) + "."

        
        s = ""
        for i in range(valuelen):
            s += (" " * valuelen) + " |"
            for val in value_strings:
                if i >= valuelen - len(val):
                    s += val[i - valuelen + len(val)].rjust(entrylen + 1)
                else:
                    s += " " * (entrylen + 1)
            s += " |\n"

        
        s += "{}-+-{}+\n".format("-" * valuelen, "-" * ((entrylen + 1) * len(values)))

        
        for val, li in zip(value_strings, values):
            i = self._indices[li]
            s += value_format % val
            for lj in values:
                j = self._indices[lj]
                if confusion[i][j] == 0:
                    s += zerostr
                elif show_percents:
                    s += entry_format % (100.0 * confusion[i][j] / self._total)
                else:
                    s += entry_format % confusion[i][j]
                if i == j:
                    prevspace = s.rfind(" ")
                    s = s[:prevspace] + "<" + s[prevspace + 1 :] + ">"
                else:
                    s += " "
            s += "|\n"

        
        s += "{}-+-{}+\n".format("-" * valuelen, "-" * ((entrylen + 1) * len(values)))

        
        s += "(row = reference; col = test)\n"
        if not values_in_chart:
            s += "Value key:\n"
            for i, value in enumerate(values):
                s += "%6d: %s\n" % (i + 1, value)

        return s

    def key(self):
        values = self._values
        str = "Value key:\n"
        indexlen = len(repr(len(values) - 1))
        key_format = "  %" + repr(indexlen) + "d: %s\n"
        for i in range(len(values)):
            str += key_format % (i, values[i])

        return str

    def recall(self, value):
        
        
        TP = self[value, value]
        
        TP_FN = sum(self[value, pred_value] for pred_value in self._values)
        if TP_FN == 0:
            return 0.0
        return TP / TP_FN

    def precision(self, value):
        
        
        TP = self[value, value]
        
        TP_FP = sum(self[real_value, value] for real_value in self._values)
        if TP_FP == 0:
            return 0.0
        return TP / TP_FP

    def f_measure(self, value, alpha=0.5):
        
        p = self.precision(value)
        r = self.recall(value)
        if p == 0.0 or r == 0.0:
            return 0.0
        return 1.0 / (alpha / p + (1 - alpha) / r)

    def evaluate(self, alpha=0.5, truncate=None, sort_by_count=False):
        
        tags = self._values

        
        if sort_by_count:
            tags = sorted(tags, key=lambda v: -sum(self._confusion[self._indices[v]]))
        if truncate:
            tags = tags[:truncate]

        tag_column_len = max(max(len(tag) for tag in tags), 3)

        
        s = (
            f"{' ' * (tag_column_len - 3)}Tag | Prec.  | Recall | F-measure\n"
            f"{'-' * tag_column_len}-+--------+--------+-----------\n"
        )

        
        for tag in tags:
            s += (
                f"{tag:>{tag_column_len}} | "
                f"{self.precision(tag):<6.4f} | "
                f"{self.recall(tag):<6.4f} | "
                f"{self.f_measure(tag, alpha=alpha):.4f}\n"
            )

        return s


def demo():
    reference = "DET NN VB DET JJ NN NN IN DET NN".split()
    test = "DET VB VB DET NN NN NN IN DET NN".split()
    print("Reference =", reference)
    print("Test    =", test)
    print("Confusion matrix:")
    print(ConfusionMatrix(reference, test))
    print(ConfusionMatrix(reference, test).pretty_format(sort_by_count=True))

    print(ConfusionMatrix(reference, test).recall("VB"))


if __name__ == "__main__":
    demo()