<jupyter_start><jupyter_text>Precision Recall CurveA precision-recall curve is a plot of the precision (y-axis) and the recall (x-axis) for different thresholds, much like the ROC curve. When to use PR AUC? Ref- when you want to communicate precision/recall decisions to other stakeholders- when you want to choose the threshold that fits the business problem.- when your data is heavily imbalanced. As mentioned before, it was discussed extensively in this article by <NAME> and <NAME>. The intuition is the following: since PR AUC focuses mainly on the positive class (PPV and TPR) it cares less about the frequent negative class.- when you care more about positive than negative class. If you care more about the positive class and hence PPV and TPR you should go with the Precision-Recall curve and PR AUC (average precision). Area Under Curve - like the AUC, summarizes the integral or an approximation of the area under the precision-recall curve.- In terms of model selection, F-Measure summarizes model skill for a specific probability threshold (e.g. 0.5), whereas the area under curve summarizes the skill of a model across thresholds, like ROC AUC<jupyter_code>import numpy as np
import matplotlib.pyplot as plt
class PRAUCv2:
    def __init__(self, probabilities: np.array, labels: np.array, thresholds: np.array):
        assert(len(probabilities) == len(labels)), "lens of pred and labels doesn't match!"
        probabilities = np.array(probabilities)
        labels = np.array(labels)
        thresholds = np.array(thresholds)
        # sort the entries according to the predicted confidence
        idx = np.argsort(probabilities)[::-1]
        probabilities = probabilities[idx]
        labels = labels[idx]
#         print("index: {}, probabilities: {}".format(idx, probabilities))
        self.probabilities = probabilities
        self.labels = labels
        self.thresholds = thresholds[::-1]
    
    def calc_precision_recall(self, predictions):
        """
        calc the precision and recall score based on the given predictions
        """
        prediction_list = list(predictions)
        label_list = list(self.labels)
        
        tp, fp, fn = 0, 0, 0
        for pred, label in zip(prediction_list, label_list):
            if pred == label == 1:
                tp += 1
            if pred == 1 and label != pred:
                fp += 1
            if pred == 0 and label != pred:
                fn += 1
        
        try:
            precision = tp/(tp+fp)
        except:
            precision = 1
        
        try:
            recall = tp/(tp+fn)
        except:
            recall = 1
        
        return precision, recall
    
    def calc_precision_recall_curve(self):
        # Containers for true positive / false positive rates
        precision_scores = []
        recall_scores = []
        for thres in self.thresholds:
            predictions = []
            for prob  in self.probabilities:
                if prob > thres:
                    predictions.append(1)
                else:
                    predictions.append(0)
                    
            precision, recall = self.calc_precision_recall(predictions)
        
            precision_scores.append(precision)
            recall_scores.append(recall)
        
        return np.array(precision_scores), np.array(recall_scores)
    
    
    def calc_AUC(self, prec, rec):
        # add sentinel values to the beginning and end of the PR curve
        mpre = np.concatenate(([0.], prec, [0.]))
        mrec = np.concatenate(([0.], rec, [1.]))

        # To smooth out the zigzag pattern and to correct the first sentinel value, 
        # we will calculate the envelope of the precision. 
        for i in range(mpre.size - 1, 0, -1):
            mpre[i - 1] = np.maximum(mpre[i - 1], mpre[i])
        
        # In order to calculate the area and the precision-recall-curve, 
        # we will partition the graph using rectangles
        # need only those points who lie in the upper left corners of each rectangle    
        index = np.where(mrec[1:] != mrec[:-1])[0]
        pr_auc = np.sum((mrec[index + 1] - mrec[index]) * mpre[index + 1])
        return pr_auc
    
    def plot(self, prec, rec):
        prec_list = list(prec)
        rec_list = list(rec)
#         print("prec: {}".format(prec_list))
#         print("recall: {}".format(rec_list))
        plt.plot(rec_list, prec_list)
        plt.xlabel("Recall")
        plt.ylabel("Precision")
        plt.title("PR Curve - AUC")
        plt.show()
probabilities = [0.65,0.1,0.15,0.43,0.97,0.24,0.82,0.7,0.32,0.84]
labels = [0, 0, 1, 0, 1, 1, 0, 1, 1, 1]
probability_thresholds = np.linspace(0, 1, num=50)
pr_auc = PRAUCv2(probabilities, labels, probability_thresholds)
prec, rec = pr_auc.calc_precision_recall_curve()
pr_auc.plot(prec, rec)
print("auc: {}".format(pr_auc.calc_AUC(prec, rec)))
# Area Under Precision-Recall Curve (PR-AUC)
# https://towardsdatascience.com/how-to-efficiently-implement-area-under-precision-recall-curve-pr-auc-a85872fd7f14
#  Rank Prediction Label  TP(cumsum) Precision(TP/Rank) Recall(TP/#num)
#   1    0.97        1      1            1.0                0.167
#   2    0.84        1      2            1.0                0.333
#   3    0.82        0      2            0.667              0.333  
#   4    0.7         1      3            0.75               0.500
class PRAUC:
    def __init__(self, predictions: np.array, labels: np.array):
        assert(len(predictions) == len(labels)), "lens of pred and labels don't match"
        self.predictions = predictions
        self.labels = labels
        
    def calc_PR(self):
        # sort the entries according to the predicted confidence
        idx = np.argsort(self.predictions)[::-1]
        predictions = self.predictions[idx]
        # labels is either 0 or 1
        labels = self.labels[idx]

        # calculate cum sum of labels
        cumsum = np.cumsum(labels)
        # rank is the index starting from 1, and can be used to indicate how many estimations are generated
        rank = np.arange(len(predictions)) + 1
        # total number of ground truth
        num_gts = cumsum[-1]
        prec = cumsum / rank
        rec = cumsum / num_gts
        return prec, rec
    
    def calc_AUC(self, prec, rec):
        # add sentinel values to the beginning and end of the PR curve
        mpre = np.concatenate(([0.], prec, [0.]))
        mrec = np.concatenate(([0.], rec, [1.]))

        # To smooth out the zigzag pattern and to correct the first sentinel value, 
        # we will calculate the envelope of the precision. 
        for i in range(mpre.size - 1, 0, -1):
            mpre[i - 1] = np.maximum(mpre[i - 1], mpre[i])
        
        # In order to calculate the area and the precision-recall-curve, 
        # we will partition the graph using rectangles
        # need only those points who lie in the upper left corners of each rectangle    
        index = np.where(mrec[1:] != mrec[:-1])[0]
#         print("i: {}".format(i))
#         print("x: {}".format((mrec[i + 1] - mrec[i])))
#         print("y: {}".format(mpre[i + 1]))
        pr_auc = np.sum((mrec[index + 1] - mrec[index]) * mpre[index + 1])
        return pr_auc
        
        
    def plot(self, prec, rec):
        prec_list = list(prec)
        rec_list = list(rec)
        print("prec: {}".format(prec_list))
        print("recall: {}".format(rec_list))
        plt.plot(rec_list, prec_list)
        plt.xlabel("Recall")
        plt.ylabel("Precision")
        plt.title("PR Curve - AUC")
        plt.show()
probabilities = np.array([0.65,0.1,0.15,0.43,0.97,0.24,0.82,0.7,0.32,0.84])
labels = np.array([0, 0, 1, 0, 1, 1, 0, 1, 1, 1])
pr_auc = PRAUC(probabilities, labels)
prec, rec = pr_auc.calc_PR()
pr_auc.plot(prec, rec)
print("auc: {}".format(pr_auc.calc_AUC(prec, rec)))<jupyter_output>prec: [1.0, 1.0, 0.6666666666666666, 0.75, 0.6, 0.5, 0.5714285714285714, 0.625, 0.6666666666666666, 0.6]
recall: [0.16666666666666666, 0.3333333333333333, 0.3333333333333333, 0.5, 0.5, 0.5, 0.6666666666666666, 0.8333333333333334, 1.0, 1.0]
{'fraction_non_alphanumeric': 0.11007025761124122, 'fraction_numerical': 0.04400345125107852, 'mean_word_length': 2.233957752092467, 'pattern_counts': {'":': 0, '<': 6, '<?xml version=': 0, '>': 7, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 7, 'word_list_counts': {'cursed_substrings.json': 3, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '559223', 'license': 'MIT', 'n_tokens_mistral': 2751, 'n_tokens_neox': 2458, 'n_words': 781, 'path': 'PR_Curve_AUC.ipynb', 'repo_name': 'louis-xu-ustc/machine_learning_basics', 'size': '8128'}