calculates the number of the given confusion matrix condition args confusionmatrixcond one of metricsutils confusionmatrix conditions thresholds optional defaults to 0 5 a float value or a python list tuple of float threshold values in 0 1 a threshold is compared with prediction values to determine the truth value of predictions i e above the threshold is true below is false one metric value is generated for each threshold value name optional string name of the metric instance dtype optional data type of the metric result accumulates the metric statistics args ytrue the ground truth values ypred the predicted values sampleweight optional weighting of each example defaults to 1 can be a tensor whose rank is either 0 or the same rank as ytrue and must be broadcastable to ytrue calculates the number of false positives if sampleweight is given calculates the sum of the weights of false positives this metric creates one local variable accumulator that is used to keep track of the number of false positives if sampleweight is none weights default to 1 use sampleweight of 0 to mask values args thresholds optional defaults to 0 5 a float value or a python listtuple of float threshold values in 0 1 a threshold is compared with prediction values to determine the truth value of predictions i e above the threshold is true below is false if used with a loss function that sets fromlogitstrue i e no sigmoid applied to predictions thresholds should be set to 0 one metric value is generated for each threshold value name optional string name of the metric instance dtype optional data type of the metric result standalone usage m keras metrics falsepositives m updatestate0 1 0 0 0 0 1 1 m result 2 0 m resetstate m updatestate0 1 0 0 0 0 1 1 sampleweight0 0 1 0 m result 1 0 calculates the number of false negatives if sampleweight is given calculates the sum of the weights of false negatives this metric creates one local variable accumulator that is used to keep track of the number of false negatives if sampleweight is none weights default to 1 use sampleweight of 0 to mask values args thresholds optional defaults to 0 5 a float value or a python listtuple of float threshold values in 0 1 a threshold is compared with prediction values to determine the truth value of predictions i e above the threshold is true below is false if used with a loss function that sets fromlogitstrue i e no sigmoid applied to predictions thresholds should be set to 0 one metric value is generated for each threshold value name optional string name of the metric instance dtype optional data type of the metric result standalone usage m keras metrics falsenegatives m updatestate0 1 1 1 0 1 0 0 m result 2 0 m resetstate m updatestate0 1 1 1 0 1 0 0 sampleweight0 0 1 0 m result 1 0 calculates the number of true negatives if sampleweight is given calculates the sum of the weights of true negatives this metric creates one local variable accumulator that is used to keep track of the number of true negatives if sampleweight is none weights default to 1 use sampleweight of 0 to mask values args thresholds optional defaults to 0 5 a float value or a python listtuple of float threshold values in 0 1 a threshold is compared with prediction values to determine the truth value of predictions i e above the threshold is true below is false if used with a loss function that sets fromlogitstrue i e no sigmoid applied to predictions thresholds should be set to 0 one metric value is generated for each threshold value name optional string name of the metric instance dtype optional data type of the metric result standalone usage m keras metrics truenegatives m updatestate0 1 0 0 1 1 0 0 m result 2 0 m resetstate m updatestate0 1 0 0 1 1 0 0 sampleweight0 0 1 0 m result 1 0 calculates the number of true positives if sampleweight is given calculates the sum of the weights of true positives this metric creates one local variable truepositives that is used to keep track of the number of true positives if sampleweight is none weights default to 1 use sampleweight of 0 to mask values args thresholds optional defaults to 0 5 a float value or a python listtuple of float threshold values in 0 1 a threshold is compared with prediction values to determine the truth value of predictions i e above the threshold is true below is false if used with a loss function that sets fromlogitstrue i e no sigmoid applied to predictions thresholds should be set to 0 one metric value is generated for each threshold value name optional string name of the metric instance dtype optional data type of the metric result standalone usage m keras metrics truepositives m updatestate0 1 1 1 1 0 1 1 m result 2 0 m resetstate m updatestate0 1 1 1 1 0 1 1 sampleweight0 0 1 0 m result 1 0 computes the precision of the predictions with respect to the labels the metric creates two local variables truepositives and falsepositives that are used to compute the precision this value is ultimately returned as precision an idempotent operation that simply divides truepositives by the sum of truepositives and falsepositives if sampleweight is none weights default to 1 use sampleweight of 0 to mask values if topk is set we ll calculate precision as how often on average a class among the topk classes with the highest predicted values of a batch entry is correct and can be found in the label for that entry if classid is specified we calculate precision by considering only the entries in the batch for which classid is above the threshold andor in the topk highest predictions and computing the fraction of them for which classid is indeed a correct label args thresholds optional a float value or a python listtuple of float threshold values in 0 1 a threshold is compared with prediction values to determine the truth value of predictions i e above the threshold is true below is false if used with a loss function that sets fromlogitstrue i e no sigmoid applied to predictions thresholds should be set to 0 one metric value is generated for each threshold value if neither thresholds nor topk are set the default is to calculate precision with thresholds0 5 topk optional unset by default an int value specifying the topk predictions to consider when calculating precision classid optional integer class id for which we want binary metrics this must be in the halfopen interval 0 numclasses where numclasses is the last dimension of predictions name optional string name of the metric instance dtype optional data type of the metric result standalone usage m keras metrics precision m updatestate0 1 1 1 1 0 1 1 m result 0 6666667 m resetstate m updatestate0 1 1 1 1 0 1 1 sampleweight0 0 1 0 m result 1 0 with topk2 it will calculate precision over ytrue 2 and ypred 2 m keras metrics precisiontopk2 m updatestate0 0 1 1 1 1 1 1 m result 0 0 with topk4 it will calculate precision over ytrue 4 and ypred 4 m keras metrics precisiontopk4 m updatestate0 0 1 1 1 1 1 1 m result 0 5 usage with compile api python model compileoptimizer sgd loss mse metricskeras metrics precision usage with a loss with fromlogitstrue python model compileoptimizer adam losskeras losses binarycrossentropyfromlogitstrue metricskeras metrics precisionthresholds0 metric should be maximized during optimization accumulates true positive and false positive statistics args ytrue the ground truth values with the same dimensions as ypred will be cast to bool ypred the predicted values each element must be in the range 0 1 sampleweight optional weighting of each example defaults to 1 can be a tensor whose rank is either 0 or the same rank as ytrue and must be broadcastable to ytrue computes the recall of the predictions with respect to the labels this metric creates two local variables truepositives and falsenegatives that are used to compute the recall this value is ultimately returned as recall an idempotent operation that simply divides truepositives by the sum of truepositives and falsenegatives if sampleweight is none weights default to 1 use sampleweight of 0 to mask values if topk is set recall will be computed as how often on average a class among the labels of a batch entry is in the topk predictions if classid is specified we calculate recall by considering only the entries in the batch for which classid is in the label and computing the fraction of them for which classid is above the threshold andor in the topk predictions args thresholds optional a float value or a python listtuple of float threshold values in 0 1 a threshold is compared with prediction values to determine the truth value of predictions i e above the threshold is true below is false if used with a loss function that sets fromlogitstrue i e no sigmoid applied to predictions thresholds should be set to 0 one metric value is generated for each threshold value if neither thresholds nor topk are set the default is to calculate recall with thresholds0 5 topk optional unset by default an int value specifying the topk predictions to consider when calculating recall classid optional integer class id for which we want binary metrics this must be in the halfopen interval 0 numclasses where numclasses is the last dimension of predictions name optional string name of the metric instance dtype optional data type of the metric result standalone usage m keras metrics recall m updatestate0 1 1 1 1 0 1 1 m result 0 6666667 m resetstate m updatestate0 1 1 1 1 0 1 1 sampleweight0 0 1 0 m result 1 0 usage with compile api python model compileoptimizer sgd loss mse metricskeras metrics recall usage with a loss with fromlogitstrue python model compileoptimizer adam losskeras losses binarycrossentropyfromlogitstrue metricskeras metrics recallthresholds0 metric should be maximized during optimization accumulates true positive and false negative statistics args ytrue the ground truth values with the same dimensions as ypred will be cast to bool ypred the predicted values each element must be in the range 0 1 sampleweight optional weighting of each example defaults to 1 can be a tensor whose rank is either 0 or the same rank as ytrue and must be broadcastable to ytrue abstract base class for computing sensitivity and specificity for additional information about specificity and sensitivity see the followinghttps en wikipedia orgwikisensitivityandspecificity metric should be maximized during optimization compute numthresholds thresholds in 0 1 accumulates confusion matrix statistics args ytrue the ground truth values ypred the predicted values sampleweight optional weighting of each example defaults to 1 can be a tensor whose rank is either 0 or the same rank as ytrue and must be broadcastable to ytrue returns the maximum of dependentstatistic that satisfies the constraint args constrained over these values the constraint is specified a rank1 tensor dependent from these values the maximum that satiesfies the constraint is selected values in this tensor and in constrained are linked by having the same threshold at each position hence this tensor must have the same shape predicate a binary boolean functor to be applied to arguments constrained and self value e g ops greater returns maximal dependent value if no value satisfies the constraint 0 0 computes best sensitivity where specificity is specified value sensitivity measures the proportion of actual positives that are correctly identified as such tp tp fn specificity measures the proportion of actual negatives that are correctly identified as such tn tn fp this metric creates four local variables truepositives truenegatives falsepositives and falsenegatives that are used to compute the sensitivity at the given specificity the threshold for the given specificity value is computed and used to evaluate the corresponding sensitivity if sampleweight is none weights default to 1 use sampleweight of 0 to mask values if classid is specified we calculate precision by considering only the entries in the batch for which classid is above the threshold predictions and computing the fraction of them for which classid is indeed a correct label for additional information about specificity and sensitivity see the followinghttps en wikipedia orgwikisensitivityandspecificity args specificity a scalar value in range 0 1 numthresholds optional defaults to 200 the number of thresholds to use for matching the given specificity classid optional integer class id for which we want binary metrics this must be in the halfopen interval 0 numclasses where numclasses is the last dimension of predictions name optional string name of the metric instance dtype optional data type of the metric result standalone usage m keras metrics sensitivityatspecificity0 5 m updatestate0 0 0 1 1 0 0 3 0 8 0 3 0 8 m result 0 5 m resetstate m updatestate0 0 0 1 1 0 0 3 0 8 0 3 0 8 sampleweight1 1 2 2 1 m result 0 333333 usage with compile api python model compile optimizer sgd loss mse metricskeras metrics sensitivityatspecificity computes best specificity where sensitivity is specified value sensitivity measures the proportion of actual positives that are correctly identified as such tp tp fn specificity measures the proportion of actual negatives that are correctly identified as such tn tn fp this metric creates four local variables truepositives truenegatives falsepositives and falsenegatives that are used to compute the specificity at the given sensitivity the threshold for the given sensitivity value is computed and used to evaluate the corresponding specificity if sampleweight is none weights default to 1 use sampleweight of 0 to mask values if classid is specified we calculate precision by considering only the entries in the batch for which classid is above the threshold predictions and computing the fraction of them for which classid is indeed a correct label for additional information about specificity and sensitivity see the followinghttps en wikipedia orgwikisensitivityandspecificity args sensitivity a scalar value in range 0 1 numthresholds optional defaults to 200 the number of thresholds to use for matching the given sensitivity classid optional integer class id for which we want binary metrics this must be in the halfopen interval 0 numclasses where numclasses is the last dimension of predictions name optional string name of the metric instance dtype optional data type of the metric result standalone usage m keras metrics specificityatsensitivity0 5 m updatestate0 0 0 1 1 0 0 3 0 8 0 3 0 8 m result 0 66666667 m resetstate m updatestate0 0 0 1 1 0 0 3 0 8 0 3 0 8 sampleweight1 1 2 2 2 m result 0 5 usage with compile api python model compile optimizer sgd loss mse metricskeras metrics specificityatsensitivity computes best precision where recall is specified value this metric creates four local variables truepositives truenegatives falsepositives and falsenegatives that are used to compute the precision at the given recall the threshold for the given recall value is computed and used to evaluate the corresponding precision if sampleweight is none weights default to 1 use sampleweight of 0 to mask values if classid is specified we calculate precision by considering only the entries in the batch for which classid is above the threshold predictions and computing the fraction of them for which classid is indeed a correct label args recall a scalar value in range 0 1 numthresholds optional defaults to 200 the number of thresholds to use for matching the given recall classid optional integer class id for which we want binary metrics this must be in the halfopen interval 0 numclasses where numclasses is the last dimension of predictions name optional string name of the metric instance dtype optional data type of the metric result standalone usage m keras metrics precisionatrecall0 5 m updatestate0 0 0 1 1 0 0 3 0 8 0 3 0 8 m result 0 5 m resetstate m updatestate0 0 0 1 1 0 0 3 0 8 0 3 0 8 sampleweight2 2 2 1 1 m result 0 33333333 usage with compile api python model compile optimizer sgd loss mse metricskeras metrics precisionatrecallrecall0 8 computes best recall where precision is specified value for a given scorelabeldistribution the required precision might not be achievable in this case 0 0 is returned as recall this metric creates four local variables truepositives truenegatives falsepositives and falsenegatives that are used to compute the recall at the given precision the threshold for the given precision value is computed and used to evaluate the corresponding recall if sampleweight is none weights default to 1 use sampleweight of 0 to mask values if classid is specified we calculate precision by considering only the entries in the batch for which classid is above the threshold predictions and computing the fraction of them for which classid is indeed a correct label args precision a scalar value in range 0 1 numthresholds optional defaults to 200 the number of thresholds to use for matching the given precision classid optional integer class id for which we want binary metrics this must be in the halfopen interval 0 numclasses where numclasses is the last dimension of predictions name optional string name of the metric instance dtype optional data type of the metric result standalone usage m keras metrics recallatprecision0 8 m updatestate0 0 1 1 0 0 5 0 3 0 9 m result 0 5 m resetstate m updatestate0 0 1 1 0 0 5 0 3 0 9 sampleweight1 0 0 1 m result 1 0 usage with compile api python model compile optimizer sgd loss mse metricskeras metrics recallatprecisionprecision0 8 approximates the auc area under the curve of the roc or pr curves the auc area under the curve of the roc receiver operating characteristic default or pr precision recall curves are quality measures of binary classifiers unlike the accuracy and like crossentropy losses rocauc and prauc evaluate all the operational points of a model this class approximates aucs using a riemann sum during the metric accumulation phrase predictions are accumulated within predefined buckets by value the auc is then computed by interpolating perbucket averages these buckets define the evaluated operational points this metric creates four local variables truepositives truenegatives falsepositives and falsenegatives that are used to compute the auc to discretize the auc curve a linearly spaced set of thresholds is used to compute pairs of recall and precision values the area under the roccurve is therefore computed using the height of the recall values by the false positive rate while the area under the prcurve is the computed using the height of the precision values by the recall this value is ultimately returned as auc an idempotent operation that computes the area under a discretized curve of precision versus recall values computed using the aforementioned variables the numthresholds variable controls the degree of discretization with larger numbers of thresholds more closely approximating the true auc the quality of the approximation may vary dramatically depending on numthresholds the thresholds parameter can be used to manually specify thresholds which split the predictions more evenly for a best approximation of the real auc predictions should be distributed approximately uniformly in the range 0 1 if fromlogitsfalse the quality of the auc approximation may be poor if this is not the case setting summationmethod to minoring or majoring can help quantify the error in the approximation by providing lower or upper bound estimate of the auc if sampleweight is none weights default to 1 use sampleweight of 0 to mask values args numthresholds optional the number of thresholds to use when discretizing the roc curve values must be 1 defaults to 200 curve optional specifies the name of the curve to be computed roc default or pr for the precisionrecallcurve summationmethod optional specifies the riemann summation method https en wikipedia orgwikiriemannsum used interpolation default applies midpoint summation scheme for roc for prauc interpolates truefalse positives but not the ratio that is precision see davis goadrich 2006 for details minoring applies left summation for increasing intervals and right summation for decreasing intervals majoring does the opposite name optional string name of the metric instance dtype optional data type of the metric result thresholds optional a list of floating point values to use as the thresholds for discretizing the curve if set the numthresholds parameter is ignored values should be in 0 1 endpoint thresholds equal to epsilon 1epsilon for a small positive epsilon value will be automatically included with these to correctly handle predictions equal to exactly 0 or 1 multilabel boolean indicating whether multilabel data should be treated as such wherein auc is computed separately for each label and then averaged across labels or when false if the data should be flattened into a single label before auc computation in the latter case when multilabel data is passed to auc each labelprediction pair is treated as an individual data point should be set to false for multiclass data numlabels optional the number of labels used when multilabel is true if numlabels is not specified then state variables get created on the first call to updatestate labelweights optional list array or tensor of nonnegative weights used to compute aucs for multilabel data when multilabel is true the weights are applied to the individual label aucs when they are averaged to produce the multilabel auc when it s false they are used to weight the individual label predictions in computing the confusion matrix on the flattened data note that this is unlike classweights in that classweights weights the example depending on the value of its label whereas labelweights depends only on the index of that label before flattening therefore labelweights should not be used for multiclass data fromlogits boolean indicating whether the predictions ypred in updatestate are probabilities or sigmoid logits as a rule of thumb when using a keras loss the fromlogits constructor argument of the loss should match the auc fromlogits constructor argument standalone usage m keras metrics aucnumthresholds3 m updatestate0 0 1 1 0 0 5 0 3 0 9 threshold values are 0 1e7 0 5 1 1e7 tp 2 1 0 fp 2 0 0 fn 0 1 2 tn 0 2 2 tprate recall 1 0 5 0 fprate 1 0 0 auc 1 0 5 2 1 0 0 5 0 2 0 0 0 75 m result 0 75 m resetstate m updatestate0 0 1 1 0 0 5 0 3 0 9 sampleweight1 0 0 1 m result 1 0 usage with compile api python reports the auc of a model outputting a probability model compileoptimizer sgd losskeras losses binarycrossentropy metricskeras metrics auc reports the auc of a model outputting a logit model compileoptimizer sgd losskeras losses binarycrossentropyfromlogitstrue metricskeras metrics aucfromlogitstrue metric should be maximized during optimization validate configurations update properties if specified use the supplied thresholds otherwise linearly interpolate numthresholds 2 thresholds in 0 1 add an endpoint threshold below zero and above one for either threshold method to account for floating point imprecisions handle multilabel arguments the thresholds used for evaluating auc return listself thresholds def buildself shape create metric variables accumulates confusion matrix statistics args ytrue the ground truth values ypred the predicted values sampleweight optional weighting of each example can be a tensor whose rank is either 0 or the same rank as ytrue and must be broadcastable to ytrue defaults to 1 ytrue should have shape number of examples number of labels tp tn fp and fn should all have shape number of thresholds number of labels labelweights should be of length equal to the number of labels only forward labelweights to updateconfusionmatrixvariables when multilabel is false otherwise the averaging of individual label aucs is handled in auc result interpolation formula inspired by section 4 of davis goadrich 2006 https www biostat wisc edupagerocpr pdf note here we derive use a closed formula not present in the paper as follows precision tp tp fp tp p modeling all of tp true positive fp false positive and their sum p tp fp predicted positive as varying linearly within each interval a b between successive thresholds we get precision slope dtp dp tpb tpa pb pa tp tpa p pa precision tpa slope p pa p the area within the interval is slope totalposweight times intabprecision dp intabtpa slope p pa dp p intabprecision dp intabslope dp intercept dp p where intercept tpa slope pa tpb slope pb resulting in intabprecision dp tpb tpa intercept logpb pa bringing back the factor slope totalposweight we d put aside we get slope dtp intercept logpb pa totalposweight where dtp tpb tpa note that when pa 0 the above calculation simplifies into intabprecision dtp intabslope dtp slope tpb tpa which is really equivalent to imputing constant precision throughout the first bucket having 0 true positives returns prauc an approximation of the area under the pr curve evenly weighted average of the label aucs weighted average of the label aucs this use case is different and is handled separately set x and y values for the curves based on curve config find the rectangle heights based on summationmethod note the case pr interpolation has been handled above self summationmethod metricsutils aucsummationmethod majoring sum up the areas of all the rectangles unweighted average of the label aucs weighted average of the label aucs optimization to avoid serializing a large number of generated thresholds we remove the endpoint thresholds as an inverse of how the thresholds were initialized this ensures that a metric initialized from this config has the same thresholds calculates the number of the given confusion matrix condition args confusion_matrix_cond one of metrics_utils confusionmatrix conditions thresholds optional defaults to 0 5 a float value or a python list tuple of float threshold values in 0 1 a threshold is compared with prediction values to determine the truth value of predictions i e above the threshold is true below is false one metric value is generated for each threshold value name optional string name of the metric instance dtype optional data type of the metric result accumulates the metric statistics args y_true the ground truth values y_pred the predicted values sample_weight optional weighting of each example defaults to 1 can be a tensor whose rank is either 0 or the same rank as y_true and must be broadcastable to y_true calculates the number of false positives if sample_weight is given calculates the sum of the weights of false positives this metric creates one local variable accumulator that is used to keep track of the number of false positives if sample_weight is none weights default to 1 use sample_weight of 0 to mask values args thresholds optional defaults to 0 5 a float value or a python list tuple of float threshold values in 0 1 a threshold is compared with prediction values to determine the truth value of predictions i e above the threshold is true below is false if used with a loss function that sets from_logits true i e no sigmoid applied to predictions thresholds should be set to 0 one metric value is generated for each threshold value name optional string name of the metric instance dtype optional data type of the metric result standalone usage m keras metrics falsepositives m update_state 0 1 0 0 0 0 1 1 m result 2 0 m reset_state m update_state 0 1 0 0 0 0 1 1 sample_weight 0 0 1 0 m result 1 0 calculates the number of false negatives if sample_weight is given calculates the sum of the weights of false negatives this metric creates one local variable accumulator that is used to keep track of the number of false negatives if sample_weight is none weights default to 1 use sample_weight of 0 to mask values args thresholds optional defaults to 0 5 a float value or a python list tuple of float threshold values in 0 1 a threshold is compared with prediction values to determine the truth value of predictions i e above the threshold is true below is false if used with a loss function that sets from_logits true i e no sigmoid applied to predictions thresholds should be set to 0 one metric value is generated for each threshold value name optional string name of the metric instance dtype optional data type of the metric result standalone usage m keras metrics falsenegatives m update_state 0 1 1 1 0 1 0 0 m result 2 0 m reset_state m update_state 0 1 1 1 0 1 0 0 sample_weight 0 0 1 0 m result 1 0 calculates the number of true negatives if sample_weight is given calculates the sum of the weights of true negatives this metric creates one local variable accumulator that is used to keep track of the number of true negatives if sample_weight is none weights default to 1 use sample_weight of 0 to mask values args thresholds optional defaults to 0 5 a float value or a python list tuple of float threshold values in 0 1 a threshold is compared with prediction values to determine the truth value of predictions i e above the threshold is true below is false if used with a loss function that sets from_logits true i e no sigmoid applied to predictions thresholds should be set to 0 one metric value is generated for each threshold value name optional string name of the metric instance dtype optional data type of the metric result standalone usage m keras metrics truenegatives m update_state 0 1 0 0 1 1 0 0 m result 2 0 m reset_state m update_state 0 1 0 0 1 1 0 0 sample_weight 0 0 1 0 m result 1 0 calculates the number of true positives if sample_weight is given calculates the sum of the weights of true positives this metric creates one local variable true_positives that is used to keep track of the number of true positives if sample_weight is none weights default to 1 use sample_weight of 0 to mask values args thresholds optional defaults to 0 5 a float value or a python list tuple of float threshold values in 0 1 a threshold is compared with prediction values to determine the truth value of predictions i e above the threshold is true below is false if used with a loss function that sets from_logits true i e no sigmoid applied to predictions thresholds should be set to 0 one metric value is generated for each threshold value name optional string name of the metric instance dtype optional data type of the metric result standalone usage m keras metrics truepositives m update_state 0 1 1 1 1 0 1 1 m result 2 0 m reset_state m update_state 0 1 1 1 1 0 1 1 sample_weight 0 0 1 0 m result 1 0 computes the precision of the predictions with respect to the labels the metric creates two local variables true_positives and false_positives that are used to compute the precision this value is ultimately returned as precision an idempotent operation that simply divides true_positives by the sum of true_positives and false_positives if sample_weight is none weights default to 1 use sample_weight of 0 to mask values if top_k is set we ll calculate precision as how often on average a class among the top k classes with the highest predicted values of a batch entry is correct and can be found in the label for that entry if class_id is specified we calculate precision by considering only the entries in the batch for which class_id is above the threshold and or in the top k highest predictions and computing the fraction of them for which class_id is indeed a correct label args thresholds optional a float value or a python list tuple of float threshold values in 0 1 a threshold is compared with prediction values to determine the truth value of predictions i e above the threshold is true below is false if used with a loss function that sets from_logits true i e no sigmoid applied to predictions thresholds should be set to 0 one metric value is generated for each threshold value if neither thresholds nor top_k are set the default is to calculate precision with thresholds 0 5 top_k optional unset by default an int value specifying the top k predictions to consider when calculating precision class_id optional integer class id for which we want binary metrics this must be in the half open interval 0 num_classes where num_classes is the last dimension of predictions name optional string name of the metric instance dtype optional data type of the metric result standalone usage m keras metrics precision m update_state 0 1 1 1 1 0 1 1 m result 0 6666667 m reset_state m update_state 0 1 1 1 1 0 1 1 sample_weight 0 0 1 0 m result 1 0 with top_k 2 it will calculate precision over y_true 2 and y_pred 2 m keras metrics precision top_k 2 m update_state 0 0 1 1 1 1 1 1 m result 0 0 with top_k 4 it will calculate precision over y_true 4 and y_pred 4 m keras metrics precision top_k 4 m update_state 0 0 1 1 1 1 1 1 m result 0 5 usage with compile api python model compile optimizer sgd loss mse metrics keras metrics precision usage with a loss with from_logits true python model compile optimizer adam loss keras losses binarycrossentropy from_logits true metrics keras metrics precision thresholds 0 metric should be maximized during optimization accumulates true positive and false positive statistics args y_true the ground truth values with the same dimensions as y_pred will be cast to bool y_pred the predicted values each element must be in the range 0 1 sample_weight optional weighting of each example defaults to 1 can be a tensor whose rank is either 0 or the same rank as y_true and must be broadcastable to y_true noqa e501 noqa e501 computes the recall of the predictions with respect to the labels this metric creates two local variables true_positives and false_negatives that are used to compute the recall this value is ultimately returned as recall an idempotent operation that simply divides true_positives by the sum of true_positives and false_negatives if sample_weight is none weights default to 1 use sample_weight of 0 to mask values if top_k is set recall will be computed as how often on average a class among the labels of a batch entry is in the top k predictions if class_id is specified we calculate recall by considering only the entries in the batch for which class_id is in the label and computing the fraction of them for which class_id is above the threshold and or in the top k predictions args thresholds optional a float value or a python list tuple of float threshold values in 0 1 a threshold is compared with prediction values to determine the truth value of predictions i e above the threshold is true below is false if used with a loss function that sets from_logits true i e no sigmoid applied to predictions thresholds should be set to 0 one metric value is generated for each threshold value if neither thresholds nor top_k are set the default is to calculate recall with thresholds 0 5 top_k optional unset by default an int value specifying the top k predictions to consider when calculating recall class_id optional integer class id for which we want binary metrics this must be in the half open interval 0 num_classes where num_classes is the last dimension of predictions name optional string name of the metric instance dtype optional data type of the metric result standalone usage m keras metrics recall m update_state 0 1 1 1 1 0 1 1 m result 0 6666667 m reset_state m update_state 0 1 1 1 1 0 1 1 sample_weight 0 0 1 0 m result 1 0 usage with compile api python model compile optimizer sgd loss mse metrics keras metrics recall usage with a loss with from_logits true python model compile optimizer adam loss keras losses binarycrossentropy from_logits true metrics keras metrics recall thresholds 0 metric should be maximized during optimization accumulates true positive and false negative statistics args y_true the ground truth values with the same dimensions as y_pred will be cast to bool y_pred the predicted values each element must be in the range 0 1 sample_weight optional weighting of each example defaults to 1 can be a tensor whose rank is either 0 or the same rank as y_true and must be broadcastable to y_true noqa e501 noqa e501 abstract base class for computing sensitivity and specificity for additional information about specificity and sensitivity see the following https en wikipedia org wiki sensitivity_and_specificity metric should be maximized during optimization compute num_thresholds thresholds in 0 1 accumulates confusion matrix statistics args y_true the ground truth values y_pred the predicted values sample_weight optional weighting of each example defaults to 1 can be a tensor whose rank is either 0 or the same rank as y_true and must be broadcastable to y_true noqa e501 noqa e501 noqa e501 noqa e501 returns the maximum of dependent_statistic that satisfies the constraint args constrained over these values the constraint is specified a rank 1 tensor dependent from these values the maximum that satiesfies the constraint is selected values in this tensor and in constrained are linked by having the same threshold at each position hence this tensor must have the same shape predicate a binary boolean functor to be applied to arguments constrained and self value e g ops greater returns maximal dependent value if no value satisfies the constraint 0 0 computes best sensitivity where specificity is specified value sensitivity measures the proportion of actual positives that are correctly identified as such tp tp fn specificity measures the proportion of actual negatives that are correctly identified as such tn tn fp this metric creates four local variables true_positives true_negatives false_positives and false_negatives that are used to compute the sensitivity at the given specificity the threshold for the given specificity value is computed and used to evaluate the corresponding sensitivity if sample_weight is none weights default to 1 use sample_weight of 0 to mask values if class_id is specified we calculate precision by considering only the entries in the batch for which class_id is above the threshold predictions and computing the fraction of them for which class_id is indeed a correct label for additional information about specificity and sensitivity see the following https en wikipedia org wiki sensitivity_and_specificity args specificity a scalar value in range 0 1 num_thresholds optional defaults to 200 the number of thresholds to use for matching the given specificity class_id optional integer class id for which we want binary metrics this must be in the half open interval 0 num_classes where num_classes is the last dimension of predictions name optional string name of the metric instance dtype optional data type of the metric result standalone usage m keras metrics sensitivityatspecificity 0 5 m update_state 0 0 0 1 1 0 0 3 0 8 0 3 0 8 m result 0 5 m reset_state m update_state 0 0 0 1 1 0 0 3 0 8 0 3 0 8 sample_weight 1 1 2 2 1 m result 0 333333 usage with compile api python model compile optimizer sgd loss mse metrics keras metrics sensitivityatspecificity computes best specificity where sensitivity is specified value sensitivity measures the proportion of actual positives that are correctly identified as such tp tp fn specificity measures the proportion of actual negatives that are correctly identified as such tn tn fp this metric creates four local variables true_positives true_negatives false_positives and false_negatives that are used to compute the specificity at the given sensitivity the threshold for the given sensitivity value is computed and used to evaluate the corresponding specificity if sample_weight is none weights default to 1 use sample_weight of 0 to mask values if class_id is specified we calculate precision by considering only the entries in the batch for which class_id is above the threshold predictions and computing the fraction of them for which class_id is indeed a correct label for additional information about specificity and sensitivity see the following https en wikipedia org wiki sensitivity_and_specificity args sensitivity a scalar value in range 0 1 num_thresholds optional defaults to 200 the number of thresholds to use for matching the given sensitivity class_id optional integer class id for which we want binary metrics this must be in the half open interval 0 num_classes where num_classes is the last dimension of predictions name optional string name of the metric instance dtype optional data type of the metric result standalone usage m keras metrics specificityatsensitivity 0 5 m update_state 0 0 0 1 1 0 0 3 0 8 0 3 0 8 m result 0 66666667 m reset_state m update_state 0 0 0 1 1 0 0 3 0 8 0 3 0 8 sample_weight 1 1 2 2 2 m result 0 5 usage with compile api python model compile optimizer sgd loss mse metrics keras metrics specificityatsensitivity computes best precision where recall is specified value this metric creates four local variables true_positives true_negatives false_positives and false_negatives that are used to compute the precision at the given recall the threshold for the given recall value is computed and used to evaluate the corresponding precision if sample_weight is none weights default to 1 use sample_weight of 0 to mask values if class_id is specified we calculate precision by considering only the entries in the batch for which class_id is above the threshold predictions and computing the fraction of them for which class_id is indeed a correct label args recall a scalar value in range 0 1 num_thresholds optional defaults to 200 the number of thresholds to use for matching the given recall class_id optional integer class id for which we want binary metrics this must be in the half open interval 0 num_classes where num_classes is the last dimension of predictions name optional string name of the metric instance dtype optional data type of the metric result standalone usage m keras metrics precisionatrecall 0 5 m update_state 0 0 0 1 1 0 0 3 0 8 0 3 0 8 m result 0 5 m reset_state m update_state 0 0 0 1 1 0 0 3 0 8 0 3 0 8 sample_weight 2 2 2 1 1 m result 0 33333333 usage with compile api python model compile optimizer sgd loss mse metrics keras metrics precisionatrecall recall 0 8 computes best recall where precision is specified value for a given score label distribution the required precision might not be achievable in this case 0 0 is returned as recall this metric creates four local variables true_positives true_negatives false_positives and false_negatives that are used to compute the recall at the given precision the threshold for the given precision value is computed and used to evaluate the corresponding recall if sample_weight is none weights default to 1 use sample_weight of 0 to mask values if class_id is specified we calculate precision by considering only the entries in the batch for which class_id is above the threshold predictions and computing the fraction of them for which class_id is indeed a correct label args precision a scalar value in range 0 1 num_thresholds optional defaults to 200 the number of thresholds to use for matching the given precision class_id optional integer class id for which we want binary metrics this must be in the half open interval 0 num_classes where num_classes is the last dimension of predictions name optional string name of the metric instance dtype optional data type of the metric result standalone usage m keras metrics recallatprecision 0 8 m update_state 0 0 1 1 0 0 5 0 3 0 9 m result 0 5 m reset_state m update_state 0 0 1 1 0 0 5 0 3 0 9 sample_weight 1 0 0 1 m result 1 0 usage with compile api python model compile optimizer sgd loss mse metrics keras metrics recallatprecision precision 0 8 approximates the auc area under the curve of the roc or pr curves the auc area under the curve of the roc receiver operating characteristic default or pr precision recall curves are quality measures of binary classifiers unlike the accuracy and like cross entropy losses roc auc and pr auc evaluate all the operational points of a model this class approximates aucs using a riemann sum during the metric accumulation phrase predictions are accumulated within predefined buckets by value the auc is then computed by interpolating per bucket averages these buckets define the evaluated operational points this metric creates four local variables true_positives true_negatives false_positives and false_negatives that are used to compute the auc to discretize the auc curve a linearly spaced set of thresholds is used to compute pairs of recall and precision values the area under the roc curve is therefore computed using the height of the recall values by the false positive rate while the area under the pr curve is the computed using the height of the precision values by the recall this value is ultimately returned as auc an idempotent operation that computes the area under a discretized curve of precision versus recall values computed using the aforementioned variables the num_thresholds variable controls the degree of discretization with larger numbers of thresholds more closely approximating the true auc the quality of the approximation may vary dramatically depending on num_thresholds the thresholds parameter can be used to manually specify thresholds which split the predictions more evenly for a best approximation of the real auc predictions should be distributed approximately uniformly in the range 0 1 if from_logits false the quality of the auc approximation may be poor if this is not the case setting summation_method to minoring or majoring can help quantify the error in the approximation by providing lower or upper bound estimate of the auc if sample_weight is none weights default to 1 use sample_weight of 0 to mask values args num_thresholds optional the number of thresholds to use when discretizing the roc curve values must be 1 defaults to 200 curve optional specifies the name of the curve to be computed roc default or pr for the precision recall curve summation_method optional specifies the riemann summation method https en wikipedia org wiki riemann_sum used interpolation default applies mid point summation scheme for roc for pr auc interpolates true false positives but not the ratio that is precision see davis goadrich 2006 for details minoring applies left summation for increasing intervals and right summation for decreasing intervals majoring does the opposite name optional string name of the metric instance dtype optional data type of the metric result thresholds optional a list of floating point values to use as the thresholds for discretizing the curve if set the num_thresholds parameter is ignored values should be in 0 1 endpoint thresholds equal to epsilon 1 epsilon for a small positive epsilon value will be automatically included with these to correctly handle predictions equal to exactly 0 or 1 multi_label boolean indicating whether multilabel data should be treated as such wherein auc is computed separately for each label and then averaged across labels or when false if the data should be flattened into a single label before auc computation in the latter case when multilabel data is passed to auc each label prediction pair is treated as an individual data point should be set to false for multi class data num_labels optional the number of labels used when multi_label is true if num_labels is not specified then state variables get created on the first call to update_state label_weights optional list array or tensor of non negative weights used to compute aucs for multilabel data when multi_label is true the weights are applied to the individual label aucs when they are averaged to produce the multi label auc when it s false they are used to weight the individual label predictions in computing the confusion matrix on the flattened data note that this is unlike class_weights in that class_weights weights the example depending on the value of its label whereas label_weights depends only on the index of that label before flattening therefore label_weights should not be used for multi class data from_logits boolean indicating whether the predictions y_pred in update_state are probabilities or sigmoid logits as a rule of thumb when using a keras loss the from_logits constructor argument of the loss should match the auc from_logits constructor argument standalone usage m keras metrics auc num_thresholds 3 m update_state 0 0 1 1 0 0 5 0 3 0 9 threshold values are 0 1e 7 0 5 1 1e 7 tp 2 1 0 fp 2 0 0 fn 0 1 2 tn 0 2 2 tp_rate recall 1 0 5 0 fp_rate 1 0 0 auc 1 0 5 2 1 0 0 5 0 2 0 0 0 75 m result 0 75 m reset_state m update_state 0 0 1 1 0 0 5 0 3 0 9 sample_weight 1 0 0 1 m result 1 0 usage with compile api python reports the auc of a model outputting a probability model compile optimizer sgd loss keras losses binarycrossentropy metrics keras metrics auc reports the auc of a model outputting a logit model compile optimizer sgd loss keras losses binarycrossentropy from_logits true metrics keras metrics auc from_logits true metric should be maximized during optimization validate configurations update properties if specified use the supplied thresholds otherwise linearly interpolate num_thresholds 2 thresholds in 0 1 add an endpoint threshold below zero and above one for either threshold method to account for floating point imprecisions handle multilabel arguments the thresholds used for evaluating auc initialize tp fp tn and fn tensors given the shape of the data create metric variables accumulates confusion matrix statistics args y_true the ground truth values y_pred the predicted values sample_weight optional weighting of each example can be a tensor whose rank is either 0 or the same rank as y_true and must be broadcastable to y_true defaults to 1 y_true should have shape number of examples number of labels tp tn fp and fn should all have shape number of thresholds number of labels label_weights should be of length equal to the number of labels only forward label_weights to update_confusion_matrix_variables when multi_label is false otherwise the averaging of individual label aucs is handled in auc result noqa e501 noqa e501 noqa e501 noqa e501 interpolation formula inspired by section 4 of davis goadrich 2006 https www biostat wisc edu page rocpr pdf note here we derive use a closed formula not present in the paper as follows precision tp tp fp tp p modeling all of tp true positive fp false positive and their sum p tp fp predicted positive as varying linearly within each interval a b between successive thresholds we get precision slope dtp dp tp_b tp_a p_b p_a tp tp_a p p_a precision tp_a slope p p_a p the area within the interval is slope total_pos_weight times int_a b precision dp int_a b tp_a slope p p_a dp p int_a b precision dp int_a b slope dp intercept dp p where intercept tp_a slope p_a tp_b slope p_b resulting in int_a b precision dp tp_b tp_a intercept log p_b p_a bringing back the factor slope total_pos_weight we d put aside we get slope dtp intercept log p_b p_a total_pos_weight where dtp tp_b tp_a note that when p_a 0 the above calculation simplifies into int_a b precision dtp int_a b slope dtp slope tp_b tp_a which is really equivalent to imputing constant precision throughout the first bucket having 0 true positives returns pr_auc an approximation of the area under the p r curve evenly weighted average of the label aucs weighted average of the label aucs this use case is different and is handled separately set x and y values for the curves based on curve config curve pr find the rectangle heights based on summation_method note the case pr interpolation has been handled above self summation_method metrics_utils aucsummationmethod majoring sum up the areas of all the rectangles unweighted average of the label aucs weighted average of the label aucs optimization to avoid serializing a large number of generated thresholds we remove the endpoint thresholds as an inverse of how the thresholds were initialized this ensures that a metric initialized from this config has the same thresholds
import numpy as np

from keras import activations
from keras import backend
from keras import initializers
from keras import ops
from keras.api_export import keras_export
from keras.metrics import metrics_utils
from keras.metrics.metric import Metric
from keras.utils.python_utils import to_list


class _ConfusionMatrixConditionCount(Metric):
    

    def __init__(
        self, confusion_matrix_cond, thresholds=None, name=None, dtype=None
    ):
        super().__init__(name=name, dtype=dtype)
        self._confusion_matrix_cond = confusion_matrix_cond
        self.init_thresholds = thresholds
        self.thresholds = metrics_utils.parse_init_thresholds(
            thresholds, default_threshold=0.5
        )
        self._thresholds_distributed_evenly = (
            metrics_utils.is_evenly_distributed_thresholds(self.thresholds)
        )
        self.accumulator = self.add_variable(
            shape=(len(self.thresholds),),
            initializer=initializers.Zeros(),
            name="accumulator",
        )

    def update_state(self, y_true, y_pred, sample_weight=None):
        
        return metrics_utils.update_confusion_matrix_variables(
            {self._confusion_matrix_cond: self.accumulator},
            y_true,
            y_pred,
            thresholds=self.thresholds,
            thresholds_distributed_evenly=self._thresholds_distributed_evenly,
            sample_weight=sample_weight,
        )

    def result(self):
        if len(self.thresholds) == 1:
            result = self.accumulator[0]
        else:
            result = self.accumulator
        return backend.convert_to_tensor(result)

    def get_config(self):
        config = {"thresholds": self.init_thresholds}
        base_config = super().get_config()
        return {**base_config, **config}


@keras_export("keras.metrics.FalsePositives")
class FalsePositives(_ConfusionMatrixConditionCount):
    

    def __init__(self, thresholds=None, name=None, dtype=None):
        super().__init__(
            confusion_matrix_cond=metrics_utils.ConfusionMatrix.FALSE_POSITIVES,
            thresholds=thresholds,
            name=name,
            dtype=dtype,
        )


@keras_export("keras.metrics.FalseNegatives")
class FalseNegatives(_ConfusionMatrixConditionCount):
    

    def __init__(self, thresholds=None, name=None, dtype=None):
        super().__init__(
            confusion_matrix_cond=metrics_utils.ConfusionMatrix.FALSE_NEGATIVES,
            thresholds=thresholds,
            name=name,
            dtype=dtype,
        )


@keras_export("keras.metrics.TrueNegatives")
class TrueNegatives(_ConfusionMatrixConditionCount):
    

    def __init__(self, thresholds=None, name=None, dtype=None):
        super().__init__(
            confusion_matrix_cond=metrics_utils.ConfusionMatrix.TRUE_NEGATIVES,
            thresholds=thresholds,
            name=name,
            dtype=dtype,
        )


@keras_export("keras.metrics.TruePositives")
class TruePositives(_ConfusionMatrixConditionCount):
    

    def __init__(self, thresholds=None, name=None, dtype=None):
        super().__init__(
            confusion_matrix_cond=metrics_utils.ConfusionMatrix.TRUE_POSITIVES,
            thresholds=thresholds,
            name=name,
            dtype=dtype,
        )


@keras_export("keras.metrics.Precision")
class Precision(Metric):
    

    def __init__(
        self, thresholds=None, top_k=None, class_id=None, name=None, dtype=None
    ):
        super().__init__(name=name, dtype=dtype)
        
        self._direction = "up"

        self.init_thresholds = thresholds
        self.top_k = top_k
        self.class_id = class_id

        default_threshold = 0.5 if top_k is None else metrics_utils.NEG_INF
        self.thresholds = metrics_utils.parse_init_thresholds(
            thresholds, default_threshold=default_threshold
        )
        self._thresholds_distributed_evenly = (
            metrics_utils.is_evenly_distributed_thresholds(self.thresholds)
        )
        self.true_positives = self.add_variable(
            shape=(len(self.thresholds),),
            initializer=initializers.Zeros(),
            name="true_positives",
        )
        self.false_positives = self.add_variable(
            shape=(len(self.thresholds),),
            initializer=initializers.Zeros(),
            name="false_positives",
        )

    def update_state(self, y_true, y_pred, sample_weight=None):
        
        metrics_utils.update_confusion_matrix_variables(
            {
                metrics_utils.ConfusionMatrix.TRUE_POSITIVES: self.true_positives,  
                metrics_utils.ConfusionMatrix.FALSE_POSITIVES: self.false_positives,  
            },
            y_true,
            y_pred,
            thresholds=self.thresholds,
            thresholds_distributed_evenly=self._thresholds_distributed_evenly,
            top_k=self.top_k,
            class_id=self.class_id,
            sample_weight=sample_weight,
        )

    def result(self):
        result = ops.divide(
            self.true_positives,
            self.true_positives + self.false_positives + backend.epsilon(),
        )
        return result[0] if len(self.thresholds) == 1 else result

    def reset_state(self):
        num_thresholds = len(to_list(self.thresholds))
        self.true_positives.assign(ops.zeros((num_thresholds,)))
        self.false_positives.assign(ops.zeros((num_thresholds,)))

    def get_config(self):
        config = {
            "thresholds": self.init_thresholds,
            "top_k": self.top_k,
            "class_id": self.class_id,
        }
        base_config = super().get_config()
        return {**base_config, **config}


@keras_export("keras.metrics.Recall")
class Recall(Metric):
    

    def __init__(
        self, thresholds=None, top_k=None, class_id=None, name=None, dtype=None
    ):
        super().__init__(name=name, dtype=dtype)
        
        self._direction = "up"

        self.init_thresholds = thresholds
        self.top_k = top_k
        self.class_id = class_id

        default_threshold = 0.5 if top_k is None else metrics_utils.NEG_INF
        self.thresholds = metrics_utils.parse_init_thresholds(
            thresholds, default_threshold=default_threshold
        )
        self._thresholds_distributed_evenly = (
            metrics_utils.is_evenly_distributed_thresholds(self.thresholds)
        )
        self.true_positives = self.add_variable(
            shape=(len(self.thresholds),),
            initializer=initializers.Zeros(),
            name="true_positives",
        )
        self.false_negatives = self.add_variable(
            shape=(len(self.thresholds),),
            initializer=initializers.Zeros(),
            name="false_negatives",
        )

    def update_state(self, y_true, y_pred, sample_weight=None):
        
        metrics_utils.update_confusion_matrix_variables(
            {
                metrics_utils.ConfusionMatrix.TRUE_POSITIVES: self.true_positives,  
                metrics_utils.ConfusionMatrix.FALSE_NEGATIVES: self.false_negatives,  
            },
            y_true,
            y_pred,
            thresholds=self.thresholds,
            thresholds_distributed_evenly=self._thresholds_distributed_evenly,
            top_k=self.top_k,
            class_id=self.class_id,
            sample_weight=sample_weight,
        )

    def result(self):
        result = ops.divide(
            self.true_positives,
            self.true_positives + self.false_negatives + backend.epsilon(),
        )
        return result[0] if len(self.thresholds) == 1 else result

    def reset_state(self):
        num_thresholds = len(to_list(self.thresholds))
        self.true_positives.assign(ops.zeros((num_thresholds,)))
        self.false_negatives.assign(ops.zeros((num_thresholds,)))

    def get_config(self):
        config = {
            "thresholds": self.init_thresholds,
            "top_k": self.top_k,
            "class_id": self.class_id,
        }
        base_config = super().get_config()
        return {**base_config, **config}


class SensitivitySpecificityBase(Metric):
    

    def __init__(
        self, value, num_thresholds=200, class_id=None, name=None, dtype=None
    ):
        super().__init__(name=name, dtype=dtype)
        
        self._direction = "up"

        if num_thresholds <= 0:
            raise ValueError(
                "Argument `num_thresholds` must be an integer > 0. "
                f"Received: num_thresholds={num_thresholds}"
            )
        self.value = value
        self.class_id = class_id

        
        if num_thresholds == 1:
            self.thresholds = [0.5]
            self._thresholds_distributed_evenly = False
        else:
            thresholds = [
                (i + 1) * 1.0 / (num_thresholds - 1)
                for i in range(num_thresholds - 2)
            ]
            self.thresholds = [0.0] + thresholds + [1.0]
            self._thresholds_distributed_evenly = True

        self.true_positives = self.add_variable(
            shape=(len(self.thresholds),),
            initializer=initializers.Zeros(),
            name="true_positives",
        )
        self.false_positives = self.add_variable(
            shape=(len(self.thresholds),),
            initializer=initializers.Zeros(),
            name="false_positives",
        )
        self.true_negatives = self.add_variable(
            shape=(len(self.thresholds),),
            initializer=initializers.Zeros(),
            name="true_negatives",
        )
        self.false_negatives = self.add_variable(
            shape=(len(self.thresholds),),
            initializer=initializers.Zeros(),
            name="false_negatives",
        )

    def update_state(self, y_true, y_pred, sample_weight=None):
        
        metrics_utils.update_confusion_matrix_variables(
            {
                metrics_utils.ConfusionMatrix.TRUE_POSITIVES: self.true_positives,  
                metrics_utils.ConfusionMatrix.TRUE_NEGATIVES: self.true_negatives,  
                metrics_utils.ConfusionMatrix.FALSE_POSITIVES: self.false_positives,  
                metrics_utils.ConfusionMatrix.FALSE_NEGATIVES: self.false_negatives,  
            },
            y_true,
            y_pred,
            thresholds=self.thresholds,
            thresholds_distributed_evenly=self._thresholds_distributed_evenly,
            class_id=self.class_id,
            sample_weight=sample_weight,
        )

    def reset_state(self):
        num_thresholds = len(self.thresholds)
        self.true_positives.assign(ops.zeros((num_thresholds,)))
        self.false_positives.assign(ops.zeros((num_thresholds,)))
        self.true_negatives.assign(ops.zeros((num_thresholds,)))
        self.false_negatives.assign(ops.zeros((num_thresholds,)))

    def get_config(self):
        config = {"class_id": self.class_id}
        base_config = super().get_config()
        return {**base_config, **config}

    def _find_max_under_constraint(self, constrained, dependent, predicate):
        
        feasible = backend.convert_to_numpy(
            ops.nonzero(predicate(constrained, self.value))
        )

        feasible_exists = ops.greater(ops.size(feasible), 0)
        max_dependent = ops.max(ops.take(dependent, feasible), initial=0)

        return ops.where(feasible_exists, max_dependent, 0.0)


@keras_export("keras.metrics.SensitivityAtSpecificity")
class SensitivityAtSpecificity(SensitivitySpecificityBase):
    

    def __init__(
        self,
        specificity,
        num_thresholds=200,
        class_id=None,
        name=None,
        dtype=None,
    ):
        if specificity < 0 or specificity > 1:
            raise ValueError(
                "Argument `specificity` must be in the range [0, 1]. "
                f"Received: specificity={specificity}"
            )
        self.specificity = specificity
        self.num_thresholds = num_thresholds
        super().__init__(
            specificity,
            num_thresholds=num_thresholds,
            class_id=class_id,
            name=name,
            dtype=dtype,
        )

    def result(self):
        sensitivities = ops.divide(
            self.true_positives,
            self.true_positives + self.false_negatives + backend.epsilon(),
        )
        specificities = ops.divide(
            self.true_negatives,
            self.true_negatives + self.false_positives + backend.epsilon(),
        )
        return self._find_max_under_constraint(
            specificities, sensitivities, ops.greater_equal
        )

    def get_config(self):
        config = {
            "num_thresholds": self.num_thresholds,
            "specificity": self.specificity,
        }
        base_config = super().get_config()
        return {**base_config, **config}


@keras_export("keras.metrics.SpecificityAtSensitivity")
class SpecificityAtSensitivity(SensitivitySpecificityBase):
    

    def __init__(
        self,
        sensitivity,
        num_thresholds=200,
        class_id=None,
        name=None,
        dtype=None,
    ):
        if sensitivity < 0 or sensitivity > 1:
            raise ValueError(
                "Argument `sensitivity` must be in the range [0, 1]. "
                f"Received: sensitivity={sensitivity}"
            )
        self.sensitivity = sensitivity
        self.num_thresholds = num_thresholds
        super().__init__(
            sensitivity,
            num_thresholds=num_thresholds,
            class_id=class_id,
            name=name,
            dtype=dtype,
        )

    def result(self):
        sensitivities = ops.divide(
            self.true_positives,
            self.true_positives + self.false_negatives + backend.epsilon(),
        )
        specificities = ops.divide(
            self.true_negatives,
            self.true_negatives + self.false_positives + backend.epsilon(),
        )
        return self._find_max_under_constraint(
            sensitivities, specificities, ops.greater_equal
        )

    def get_config(self):
        config = {
            "num_thresholds": self.num_thresholds,
            "sensitivity": self.sensitivity,
        }
        base_config = super().get_config()
        return {**base_config, **config}


@keras_export("keras.metrics.PrecisionAtRecall")
class PrecisionAtRecall(SensitivitySpecificityBase):
    

    def __init__(
        self, recall, num_thresholds=200, class_id=None, name=None, dtype=None
    ):
        if recall < 0 or recall > 1:
            raise ValueError(
                "Argument `recall` must be in the range [0, 1]. "
                f"Received: recall={recall}"
            )
        self.recall = recall
        self.num_thresholds = num_thresholds
        super().__init__(
            value=recall,
            num_thresholds=num_thresholds,
            class_id=class_id,
            name=name,
            dtype=dtype,
        )

    def result(self):
        recalls = ops.divide(
            self.true_positives,
            self.true_positives + self.false_negatives + backend.epsilon(),
        )
        precisions = ops.divide(
            self.true_positives,
            self.true_positives + self.false_positives + backend.epsilon(),
        )
        return self._find_max_under_constraint(
            recalls, precisions, ops.greater_equal
        )

    def get_config(self):
        config = {"num_thresholds": self.num_thresholds, "recall": self.recall}
        base_config = super().get_config()
        return {**base_config, **config}


@keras_export("keras.metrics.RecallAtPrecision")
class RecallAtPrecision(SensitivitySpecificityBase):
    

    def __init__(
        self,
        precision,
        num_thresholds=200,
        class_id=None,
        name=None,
        dtype=None,
    ):
        if precision < 0 or precision > 1:
            raise ValueError(
                "Argument `precision` must be in the range [0, 1]. "
                f"Received: precision={precision}"
            )
        self.precision = precision
        self.num_thresholds = num_thresholds
        super().__init__(
            value=precision,
            num_thresholds=num_thresholds,
            class_id=class_id,
            name=name,
            dtype=dtype,
        )

    def result(self):
        recalls = ops.divide(
            self.true_positives,
            self.true_positives + self.false_negatives + backend.epsilon(),
        )
        precisions = ops.divide(
            self.true_positives,
            self.true_positives + self.false_positives + backend.epsilon(),
        )
        return self._find_max_under_constraint(
            precisions, recalls, ops.greater_equal
        )

    def get_config(self):
        config = {
            "num_thresholds": self.num_thresholds,
            "precision": self.precision,
        }
        base_config = super().get_config()
        return {**base_config, **config}


@keras_export("keras.metrics.AUC")
class AUC(Metric):
    

    def __init__(
        self,
        num_thresholds=200,
        curve="ROC",
        summation_method="interpolation",
        name=None,
        dtype=None,
        thresholds=None,
        multi_label=False,
        num_labels=None,
        label_weights=None,
        from_logits=False,
    ):
        
        self._direction = "up"

        
        if isinstance(curve, metrics_utils.AUCCurve) and curve not in list(
            metrics_utils.AUCCurve
        ):
            raise ValueError(
                f'Invalid `curve` argument value "{curve}". '
                f"Expected one of: {list(metrics_utils.AUCCurve)}"
            )
        if isinstance(
            summation_method, metrics_utils.AUCSummationMethod
        ) and summation_method not in list(metrics_utils.AUCSummationMethod):
            raise ValueError(
                "Invalid `summation_method` "
                f'argument value "{summation_method}". '
                f"Expected one of: {list(metrics_utils.AUCSummationMethod)}"
            )

        
        self._init_from_thresholds = thresholds is not None
        if thresholds is not None:
            
            self.num_thresholds = len(thresholds) + 2
            thresholds = sorted(thresholds)
            self._thresholds_distributed_evenly = (
                metrics_utils.is_evenly_distributed_thresholds(
                    np.array([0.0] + thresholds + [1.0])
                )
            )
        else:
            if num_thresholds <= 1:
                raise ValueError(
                    "Argument `num_thresholds` must be an integer > 1. "
                    f"Received: num_thresholds={num_thresholds}"
                )

            
            
            self.num_thresholds = num_thresholds
            thresholds = [
                (i + 1) * 1.0 / (num_thresholds - 1)
                for i in range(num_thresholds - 2)
            ]
            self._thresholds_distributed_evenly = True

        
        
        self._thresholds = np.array(
            [0.0 - backend.epsilon()] + thresholds + [1.0 + backend.epsilon()]
        )

        if isinstance(curve, metrics_utils.AUCCurve):
            self.curve = curve
        else:
            self.curve = metrics_utils.AUCCurve.from_str(curve)
        if isinstance(summation_method, metrics_utils.AUCSummationMethod):
            self.summation_method = summation_method
        else:
            self.summation_method = metrics_utils.AUCSummationMethod.from_str(
                summation_method
            )
        super().__init__(name=name, dtype=dtype)

        
        self.multi_label = multi_label
        self.num_labels = num_labels
        if label_weights is not None:
            label_weights = ops.array(label_weights, dtype=self.dtype)
            self.label_weights = label_weights

        else:
            self.label_weights = None

        self._from_logits = from_logits

        self._built = False
        if self.multi_label:
            if num_labels:
                shape = [None, num_labels]
                self._build(shape)
        else:
            if num_labels:
                raise ValueError(
                    "`num_labels` is needed only when `multi_label` is True."
                )
            self._build(None)

    @property
    def thresholds(self):
        
        return list(self._thresholds)

    def _build(self, shape):
        
        if self.multi_label:
            if len(shape) != 2:
                raise ValueError(
                    "`y_pred` must have rank 2 when `multi_label=True`. "
                    f"Found rank {len(shape)}. "
                    f"Full shape received for `y_pred`: {shape}"
                )
            self._num_labels = shape[1]
            variable_shape = [self.num_thresholds, self._num_labels]
        else:
            variable_shape = [self.num_thresholds]

        self._build_input_shape = shape
        
        self.true_positives = self.add_variable(
            shape=variable_shape,
            initializer=initializers.Zeros(),
            name="true_positives",
        )
        self.false_positives = self.add_variable(
            shape=variable_shape,
            initializer=initializers.Zeros(),
            name="false_positives",
        )
        self.true_negatives = self.add_variable(
            shape=variable_shape,
            initializer=initializers.Zeros(),
            name="true_negatives",
        )
        self.false_negatives = self.add_variable(
            shape=variable_shape,
            initializer=initializers.Zeros(),
            name="false_negatives",
        )

        self._built = True

    def update_state(self, y_true, y_pred, sample_weight=None):
        
        if not self._built:
            self._build(y_pred.shape)

        if self.multi_label or (self.label_weights is not None):
            
            shapes = [(y_true, ("N", "L"))]
            if self.multi_label:
                
                
                shapes.extend(
                    [
                        (self.true_positives, ("T", "L")),
                        (self.true_negatives, ("T", "L")),
                        (self.false_positives, ("T", "L")),
                        (self.false_negatives, ("T", "L")),
                    ]
                )
            if self.label_weights is not None:
                
                
                shapes.append((self.label_weights, ("L",)))

        
        
        
        label_weights = None if self.multi_label else self.label_weights

        if self._from_logits:
            y_pred = activations.sigmoid(y_pred)

        metrics_utils.update_confusion_matrix_variables(
            {
                metrics_utils.ConfusionMatrix.TRUE_POSITIVES: self.true_positives,  
                metrics_utils.ConfusionMatrix.TRUE_NEGATIVES: self.true_negatives,  
                metrics_utils.ConfusionMatrix.FALSE_POSITIVES: self.false_positives,  
                metrics_utils.ConfusionMatrix.FALSE_NEGATIVES: self.false_negatives,  
            },
            y_true,
            y_pred,
            self._thresholds,
            thresholds_distributed_evenly=self._thresholds_distributed_evenly,
            sample_weight=sample_weight,
            multi_label=self.multi_label,
            label_weights=label_weights,
        )

    def interpolate_pr_auc(self):
        

        dtp = (
            self.true_positives[: self.num_thresholds - 1]
            - self.true_positives[1:]
        )
        p = ops.add(self.true_positives, self.false_positives)
        dp = p[: self.num_thresholds - 1] - p[1:]
        prec_slope = ops.divide(dtp, ops.maximum(dp, backend.epsilon()))
        intercept = self.true_positives[1:] - ops.multiply(prec_slope, p[1:])

        safe_p_ratio = ops.where(
            ops.logical_and(p[: self.num_thresholds - 1] > 0, p[1:] > 0),
            ops.divide(
                p[: self.num_thresholds - 1],
                ops.maximum(p[1:], backend.epsilon()),
            ),
            ops.ones_like(p[1:]),
        )

        pr_auc_increment = ops.divide(
            prec_slope * (dtp + intercept * ops.log(safe_p_ratio)),
            ops.maximum(
                self.true_positives[1:] + self.false_negatives[1:],
                backend.epsilon(),
            ),
        )

        if self.multi_label:
            by_label_auc = ops.sum(pr_auc_increment, axis=0)
            if self.label_weights is None:
                
                return ops.mean(by_label_auc)
            else:
                
                return ops.divide(
                    ops.sum(ops.multiply(by_label_auc, self.label_weights)),
                    ops.maximum(ops.sum(self.label_weights), backend.epsilon()),
                )
        else:
            return ops.sum(pr_auc_increment)

    def result(self):
        if (
            self.curve == metrics_utils.AUCCurve.PR
            and self.summation_method
            == metrics_utils.AUCSummationMethod.INTERPOLATION
        ):
            
            return self.interpolate_pr_auc()

        
        recall = ops.divide(
            self.true_positives,
            ops.maximum(
                ops.add(self.true_positives, self.false_negatives),
                backend.epsilon(),
            ),
        )
        if self.curve == metrics_utils.AUCCurve.ROC:
            fp_rate = ops.divide(
                self.false_positives,
                ops.maximum(
                    ops.add(self.false_positives, self.true_negatives),
                    backend.epsilon(),
                ),
            )
            x = fp_rate
            y = recall
        else:  
            precision = ops.divide(
                self.true_positives,
                ops.maximum(
                    ops.add(self.true_positives, self.false_positives),
                    backend.epsilon(),
                ),
            )
            x = recall
            y = precision

        
        if (
            self.summation_method
            == metrics_utils.AUCSummationMethod.INTERPOLATION
        ):
            
            heights = (y[: self.num_thresholds - 1] + y[1:]) / 2.0
        elif self.summation_method == metrics_utils.AUCSummationMethod.MINORING:
            heights = ops.minimum(y[: self.num_thresholds - 1], y[1:])
        
        else:
            heights = ops.maximum(y[: self.num_thresholds - 1], y[1:])

        
        if self.multi_label:
            riemann_terms = ops.multiply(
                x[: self.num_thresholds - 1] - x[1:], heights
            )
            by_label_auc = ops.sum(riemann_terms, axis=0)

            if self.label_weights is None:
                
                return ops.mean(by_label_auc)
            else:
                
                return ops.divide(
                    ops.sum(ops.multiply(by_label_auc, self.label_weights)),
                    ops.maximum(ops.sum(self.label_weights), backend.epsilon()),
                )
        else:
            return ops.sum(
                ops.multiply(x[: self.num_thresholds - 1] - x[1:], heights)
            )

    def reset_state(self):
        if self._built:
            if self.multi_label:
                variable_shape = (self.num_thresholds, self._num_labels)
            else:
                variable_shape = (self.num_thresholds,)

            self.true_positives.assign(ops.zeros(variable_shape))
            self.false_positives.assign(ops.zeros(variable_shape))
            self.true_negatives.assign(ops.zeros(variable_shape))
            self.false_negatives.assign(ops.zeros(variable_shape))

    def get_config(self):
        label_weights = self.label_weights
        config = {
            "num_thresholds": self.num_thresholds,
            "curve": self.curve.value,
            "summation_method": self.summation_method.value,
            "multi_label": self.multi_label,
            "num_labels": self.num_labels,
            "label_weights": label_weights,
            "from_logits": self._from_logits,
        }
        
        
        if self._init_from_thresholds:
            
            
            
            config["thresholds"] = self.thresholds[1:-1]
        base_config = super().get_config()
        return {**base_config, **config}