# Path: ehr_prediction_modeling/metrics/calibration.py
# DEFAULT_NUM_BINS = 50
# class Error(Exception):
# class MismatchedPredictionArraysError(Error):
# def reliability_histogram(
#     predictions: ListOrArray,
#     labels: ListOrArray,
#     confidence_scores: ListOrArray,
#     num_bins: int = DEFAULT_NUM_BINS,
#     example_weights: Optional[ListOrArray] = None
# ) -> Tuple[np.ndarray, np.ndarray, np.ndarray]:
# def risk_reliability_histogram(
#     labels: ListOrArray,
#     risk_scores: ListOrArray,
#     num_bins=DEFAULT_NUM_BINS,
#     example_weights: Optional[ListOrArray] = None
# ) -> Tuple[np.ndarray, np.ndarray, np.ndarray]:
# def expected_calibration_error(observed_score_histogram: ListOrArray,
#                                predicted_score_histogram: ListOrArray,
#                                bin_example_counts: ListOrArray) -> float:
# def maximum_calibration_error(observed_score_histogram: np.ndarray,
#                               predicted_score_histogram: np.ndarray) -> float:
# def reliability_diagram_from_posneg_lists(
#     pos: ListOrArray,
#     neg: ListOrArray,
#     pos_weights: ListOrArray,
#     neg_weights: ListOrArray,
#     num_bins: int = DEFAULT_NUM_BINS,
#     class_threshold: float = 0.5) -> Tuple[np.ndarray, np.ndarray, np.ndarray]:
# def risk_reliability_diagram_from_posneg_lists(
#     pos: ListOrArray,
#     neg: ListOrArray,
#     pos_weights: ListOrArray,
#     neg_weights: ListOrArray,
#     num_bins: int = DEFAULT_NUM_BINS
# ) -> Tuple[np.ndarray, np.ndarray, np.ndarray]:
# 
# Path: ehr_prediction_modeling/metrics/pr_roc.py
# def area_under_curve_from_raw_scores(scores_pos: ListOrArray,
#                                      scores_neg: ListOrArray) -> float:
# def pr_area_under_curve_from_raw_scores(scores_pos: ListOrArray,
#                                         scores_neg: ListOrArray) -> float:
# def weighted_area_under_curve_from_raw_scores(
#     scores_pos: ListOrArray, scores_neg: ListOrArray,
#     weights_pos: ListOrArray, weights_neg: ListOrArray) -> float:
# def weighted_pr_area_under_curve_from_raw_scores(
#     scores_pos: ListOrArray, scores_neg: ListOrArray,
#     weights_pos: ListOrArray, weights_neg: ListOrArray) -> float:
# def roc_curve_from_raw_scores(
#     scores_pos: ListOrArray,
#     scores_neg: ListOrArray,
#     weights_pos: Optional[ListOrArray] = None,
#     weights_neg: Optional[ListOrArray] = None) -> CurvePoints:
# def pr_curve_from_raw_scores(
#     scores_pos: ListOrArray,
#     scores_neg: ListOrArray,
#     weights_pos: Optional[ListOrArray] = None,
#     weights_neg: Optional[ListOrArray] = None) -> CurvePoints:
# def _curve_area(curve: CurvePoints) -> float:
# def _trapezoid_area(point_left: List[float], point_right: List[float]) -> float:
# 
# Path: ehr_prediction_modeling/metrics/calculators/base.py
# class Error(Exception):
# class InvalidClassThresholdError(Error):
# class MetricCalculatorBase():
#   ALL_METRICS = []
#   def get_metrics_dict(self) -> Mapping[str, Any]:


import math
import numpy as np
from typing import List, Mapping, Optional, Union
from ehr_prediction_modeling.metrics import calibration
from ehr_prediction_modeling.metrics import pr_roc
from ehr_prediction_modeling.metrics.calculators import base
        "confidence_bin_counts": self._confidence_bin_counts
    }

  def full_risk_calibration_histogram(self) -> Mapping[str, np.ndarray]:
    """Gives the full risk vs positive rate calibration histogram."""
    return {
        "positive_rate_histogram": self._positive_rate_histogram,
        "risk_histogram": self._risk_histogram,
        "risk_bin_counts": self._risk_bin_counts
    }

  def expected_confidence_calibration_error(self) -> float:
    """Compute the expected confidence calibration error."""
    return calibration.expected_calibration_error(self._accuracy_histogram,
                                                  self._confidence_histogram,
                                                  self._confidence_bin_counts)

  def maximum_confidence_calibration_error(self) -> float:
    """Compute the maximum confidence calibration error."""
    return calibration.maximum_calibration_error(self._accuracy_histogram,
                                                 self._confidence_histogram)

  def expected_risk_calibration_error(self) -> float:
    """Compute the expected risk calibration error."""
    return calibration.expected_calibration_error(self._positive_rate_histogram,
                                                  self._risk_histogram,
                                                  self._risk_bin_counts)

  def maximum_risk_calibration_error(self) -> float:
    """Compute the maximum risk calibration error."""

# Path: ehr_prediction_modeling/metrics/calibration.py
# DEFAULT_NUM_BINS = 50
# class Error(Exception):
# class MismatchedPredictionArraysError(Error):
# def reliability_histogram(
#     predictions: ListOrArray,
#     labels: ListOrArray,
#     confidence_scores: ListOrArray,
#     num_bins: int = DEFAULT_NUM_BINS,
#     example_weights: Optional[ListOrArray] = None
# ) -> Tuple[np.ndarray, np.ndarray, np.ndarray]:
# def risk_reliability_histogram(
#     labels: ListOrArray,
#     risk_scores: ListOrArray,
#     num_bins=DEFAULT_NUM_BINS,
#     example_weights: Optional[ListOrArray] = None
# ) -> Tuple[np.ndarray, np.ndarray, np.ndarray]:
# def expected_calibration_error(observed_score_histogram: ListOrArray,
#                                predicted_score_histogram: ListOrArray,
#                                bin_example_counts: ListOrArray) -> float:
# def maximum_calibration_error(observed_score_histogram: np.ndarray,
#                               predicted_score_histogram: np.ndarray) -> float:
# def reliability_diagram_from_posneg_lists(
#     pos: ListOrArray,
#     neg: ListOrArray,
#     pos_weights: ListOrArray,
#     neg_weights: ListOrArray,
#     num_bins: int = DEFAULT_NUM_BINS,
#     class_threshold: float = 0.5) -> Tuple[np.ndarray, np.ndarray, np.ndarray]:
# def risk_reliability_diagram_from_posneg_lists(
#     pos: ListOrArray,
#     neg: ListOrArray,
#     pos_weights: ListOrArray,
#     neg_weights: ListOrArray,
#     num_bins: int = DEFAULT_NUM_BINS
# ) -> Tuple[np.ndarray, np.ndarray, np.ndarray]:
# 
# Path: ehr_prediction_modeling/metrics/pr_roc.py
# def area_under_curve_from_raw_scores(scores_pos: ListOrArray,
#                                      scores_neg: ListOrArray) -> float:
# def pr_area_under_curve_from_raw_scores(scores_pos: ListOrArray,
#                                         scores_neg: ListOrArray) -> float:
# def weighted_area_under_curve_from_raw_scores(
#     scores_pos: ListOrArray, scores_neg: ListOrArray,
#     weights_pos: ListOrArray, weights_neg: ListOrArray) -> float:
# def weighted_pr_area_under_curve_from_raw_scores(
#     scores_pos: ListOrArray, scores_neg: ListOrArray,
#     weights_pos: ListOrArray, weights_neg: ListOrArray) -> float:
# def roc_curve_from_raw_scores(
#     scores_pos: ListOrArray,
#     scores_neg: ListOrArray,
#     weights_pos: Optional[ListOrArray] = None,
#     weights_neg: Optional[ListOrArray] = None) -> CurvePoints:
# def pr_curve_from_raw_scores(
#     scores_pos: ListOrArray,
#     scores_neg: ListOrArray,
#     weights_pos: Optional[ListOrArray] = None,
#     weights_neg: Optional[ListOrArray] = None) -> CurvePoints:
# def _curve_area(curve: CurvePoints) -> float:
# def _trapezoid_area(point_left: List[float], point_right: List[float]) -> float:
# 
# Path: ehr_prediction_modeling/metrics/calculators/base.py
# class Error(Exception):
# class InvalidClassThresholdError(Error):
# class MetricCalculatorBase():
#   ALL_METRICS = []
#   def get_metrics_dict(self) -> Mapping[str, Any]:


# Path: ehr_prediction_modeling/metrics/calculators/classification_task.py
import math
import numpy as np
from typing import List, Mapping, Optional, Union
from ehr_prediction_modeling.metrics import calibration
from ehr_prediction_modeling.metrics import pr_roc
from ehr_prediction_modeling.metrics.calculators import base

        "confidence_bin_counts": self._confidence_bin_counts
    }

  def full_risk_calibration_histogram(self) -> Mapping[str, np.ndarray]:
    """Gives the full risk vs positive rate calibration histogram."""
    return {
        "positive_rate_histogram": self._positive_rate_histogram,
        "risk_histogram": self._risk_histogram,
        "risk_bin_counts": self._risk_bin_counts
    }

  def expected_confidence_calibration_error(self) -> float:
    """Compute the expected confidence calibration error."""
    return calibration.expected_calibration_error(self._accuracy_histogram,
                                                  self._confidence_histogram,
                                                  self._confidence_bin_counts)

  def maximum_confidence_calibration_error(self) -> float:
    """Compute the maximum confidence calibration error."""
    return calibration.maximum_calibration_error(self._accuracy_histogram,
                                                 self._confidence_histogram)

  def expected_risk_calibration_error(self) -> float:
    """Compute the expected risk calibration error."""
    return calibration.expected_calibration_error(self._positive_rate_histogram,
                                                  self._risk_histogram,
                                                  self._risk_bin_counts)

  def maximum_risk_calibration_error(self) -> float:
    """Compute the maximum risk calibration error."""

    return calibration.maximum_calibration_error(self._positive_rate_histogram,