# Path: ehr_prediction_modeling/metrics/calibration.py
# DEFAULT_NUM_BINS = 50
# class Error(Exception):
# class MismatchedPredictionArraysError(Error):
# def reliability_histogram(
#     predictions: ListOrArray,
#     labels: ListOrArray,
#     confidence_scores: ListOrArray,
#     num_bins: int = DEFAULT_NUM_BINS,
#     example_weights: Optional[ListOrArray] = None
# ) -> Tuple[np.ndarray, np.ndarray, np.ndarray]:
# def risk_reliability_histogram(
#     labels: ListOrArray,
#     risk_scores: ListOrArray,
#     num_bins=DEFAULT_NUM_BINS,
#     example_weights: Optional[ListOrArray] = None
# ) -> Tuple[np.ndarray, np.ndarray, np.ndarray]:
# def expected_calibration_error(observed_score_histogram: ListOrArray,
#                                predicted_score_histogram: ListOrArray,
#                                bin_example_counts: ListOrArray) -> float:
# def maximum_calibration_error(observed_score_histogram: np.ndarray,
#                               predicted_score_histogram: np.ndarray) -> float:
# def reliability_diagram_from_posneg_lists(
#     pos: ListOrArray,
#     neg: ListOrArray,
#     pos_weights: ListOrArray,
#     neg_weights: ListOrArray,
#     num_bins: int = DEFAULT_NUM_BINS,
#     class_threshold: float = 0.5) -> Tuple[np.ndarray, np.ndarray, np.ndarray]:
# def risk_reliability_diagram_from_posneg_lists(
#     pos: ListOrArray,
#     neg: ListOrArray,
#     pos_weights: ListOrArray,
#     neg_weights: ListOrArray,
#     num_bins: int = DEFAULT_NUM_BINS
# ) -> Tuple[np.ndarray, np.ndarray, np.ndarray]:
# 
# Path: ehr_prediction_modeling/metrics/pr_roc.py
# def area_under_curve_from_raw_scores(scores_pos: ListOrArray,
#                                      scores_neg: ListOrArray) -> float:
# def pr_area_under_curve_from_raw_scores(scores_pos: ListOrArray,
#                                         scores_neg: ListOrArray) -> float:
# def weighted_area_under_curve_from_raw_scores(
#     scores_pos: ListOrArray, scores_neg: ListOrArray,
#     weights_pos: ListOrArray, weights_neg: ListOrArray) -> float:
# def weighted_pr_area_under_curve_from_raw_scores(
#     scores_pos: ListOrArray, scores_neg: ListOrArray,
#     weights_pos: ListOrArray, weights_neg: ListOrArray) -> float:
# def roc_curve_from_raw_scores(
#     scores_pos: ListOrArray,
#     scores_neg: ListOrArray,
#     weights_pos: Optional[ListOrArray] = None,
#     weights_neg: Optional[ListOrArray] = None) -> CurvePoints:
# def pr_curve_from_raw_scores(
#     scores_pos: ListOrArray,
#     scores_neg: ListOrArray,
#     weights_pos: Optional[ListOrArray] = None,
#     weights_neg: Optional[ListOrArray] = None) -> CurvePoints:
# def _curve_area(curve: CurvePoints) -> float:
# def _trapezoid_area(point_left: List[float], point_right: List[float]) -> float:
# 
# Path: ehr_prediction_modeling/metrics/calculators/base.py
# class Error(Exception):
# class InvalidClassThresholdError(Error):
# class MetricCalculatorBase():
#   ALL_METRICS = []
#   def get_metrics_dict(self) -> Mapping[str, Any]:


import math
import numpy as np
from typing import List, Mapping, Optional, Union
from ehr_prediction_modeling.metrics import calibration
from ehr_prediction_modeling.metrics import pr_roc
from ehr_prediction_modeling.metrics.calculators import base
      num_calibration_bins: The number of bins to use for the reliability
        diagram and the computation of the expected/max miscalibration.
      class_threshold: The point that splits the soft predictions into positives
        and negatives.
    """
    self._pos = pos
    self._neg = neg
    self._pos_weights = pos_weights
    self._neg_weights = neg_weights
    self._num_calibration_bins = num_calibration_bins
    self._class_threshold = class_threshold
    self._precompute_confusion_matrix()
    self._precompute_reliability_diagrams()

  def _precompute_reliability_diagrams(self) -> None:
    """Computes a set of histograms.

    These are for accuracy vs confidence and risk vs positive rate.
    """
    (self._accuracy_histogram, self._confidence_histogram,
     self._confidence_bin_counts) = (
         calibration.reliability_diagram_from_posneg_lists(
             self._pos,
             self._neg,
             self._pos_weights,
             self._neg_weights,
             num_bins=self._num_calibration_bins,
             class_threshold=self._class_threshold))
    (self._positive_rate_histogram, self._risk_histogram,
     self._risk_bin_counts) = (

# Path: ehr_prediction_modeling/metrics/calibration.py
# DEFAULT_NUM_BINS = 50
# class Error(Exception):
# class MismatchedPredictionArraysError(Error):
# def reliability_histogram(
#     predictions: ListOrArray,
#     labels: ListOrArray,
#     confidence_scores: ListOrArray,
#     num_bins: int = DEFAULT_NUM_BINS,
#     example_weights: Optional[ListOrArray] = None
# ) -> Tuple[np.ndarray, np.ndarray, np.ndarray]:
# def risk_reliability_histogram(
#     labels: ListOrArray,
#     risk_scores: ListOrArray,
#     num_bins=DEFAULT_NUM_BINS,
#     example_weights: Optional[ListOrArray] = None
# ) -> Tuple[np.ndarray, np.ndarray, np.ndarray]:
# def expected_calibration_error(observed_score_histogram: ListOrArray,
#                                predicted_score_histogram: ListOrArray,
#                                bin_example_counts: ListOrArray) -> float:
# def maximum_calibration_error(observed_score_histogram: np.ndarray,
#                               predicted_score_histogram: np.ndarray) -> float:
# def reliability_diagram_from_posneg_lists(
#     pos: ListOrArray,
#     neg: ListOrArray,
#     pos_weights: ListOrArray,
#     neg_weights: ListOrArray,
#     num_bins: int = DEFAULT_NUM_BINS,
#     class_threshold: float = 0.5) -> Tuple[np.ndarray, np.ndarray, np.ndarray]:
# def risk_reliability_diagram_from_posneg_lists(
#     pos: ListOrArray,
#     neg: ListOrArray,
#     pos_weights: ListOrArray,
#     neg_weights: ListOrArray,
#     num_bins: int = DEFAULT_NUM_BINS
# ) -> Tuple[np.ndarray, np.ndarray, np.ndarray]:
# 
# Path: ehr_prediction_modeling/metrics/pr_roc.py
# def area_under_curve_from_raw_scores(scores_pos: ListOrArray,
#                                      scores_neg: ListOrArray) -> float:
# def pr_area_under_curve_from_raw_scores(scores_pos: ListOrArray,
#                                         scores_neg: ListOrArray) -> float:
# def weighted_area_under_curve_from_raw_scores(
#     scores_pos: ListOrArray, scores_neg: ListOrArray,
#     weights_pos: ListOrArray, weights_neg: ListOrArray) -> float:
# def weighted_pr_area_under_curve_from_raw_scores(
#     scores_pos: ListOrArray, scores_neg: ListOrArray,
#     weights_pos: ListOrArray, weights_neg: ListOrArray) -> float:
# def roc_curve_from_raw_scores(
#     scores_pos: ListOrArray,
#     scores_neg: ListOrArray,
#     weights_pos: Optional[ListOrArray] = None,
#     weights_neg: Optional[ListOrArray] = None) -> CurvePoints:
# def pr_curve_from_raw_scores(
#     scores_pos: ListOrArray,
#     scores_neg: ListOrArray,
#     weights_pos: Optional[ListOrArray] = None,
#     weights_neg: Optional[ListOrArray] = None) -> CurvePoints:
# def _curve_area(curve: CurvePoints) -> float:
# def _trapezoid_area(point_left: List[float], point_right: List[float]) -> float:
# 
# Path: ehr_prediction_modeling/metrics/calculators/base.py
# class Error(Exception):
# class InvalidClassThresholdError(Error):
# class MetricCalculatorBase():
#   ALL_METRICS = []
#   def get_metrics_dict(self) -> Mapping[str, Any]:


# Path: ehr_prediction_modeling/metrics/calculators/classification_task.py
import math
import numpy as np
from typing import List, Mapping, Optional, Union
from ehr_prediction_modeling.metrics import calibration
from ehr_prediction_modeling.metrics import pr_roc
from ehr_prediction_modeling.metrics.calculators import base

      num_calibration_bins: The number of bins to use for the reliability
        diagram and the computation of the expected/max miscalibration.
      class_threshold: The point that splits the soft predictions into positives
        and negatives.
    """
    self._pos = pos
    self._neg = neg
    self._pos_weights = pos_weights
    self._neg_weights = neg_weights
    self._num_calibration_bins = num_calibration_bins
    self._class_threshold = class_threshold
    self._precompute_confusion_matrix()
    self._precompute_reliability_diagrams()

  def _precompute_reliability_diagrams(self) -> None:
    """Computes a set of histograms.

    These are for accuracy vs confidence and risk vs positive rate.
    """
    (self._accuracy_histogram, self._confidence_histogram,
     self._confidence_bin_counts) = (
         calibration.reliability_diagram_from_posneg_lists(
             self._pos,
             self._neg,
             self._pos_weights,
             self._neg_weights,
             num_bins=self._num_calibration_bins,
             class_threshold=self._class_threshold))
    (self._positive_rate_histogram, self._risk_histogram,
     self._risk_bin_counts) = (

         calibration.risk_reliability_diagram_from_posneg_lists(