# Path: ehr_prediction_modeling/metrics/calibration.py
# DEFAULT_NUM_BINS = 50
# class Error(Exception):
# class MismatchedPredictionArraysError(Error):
# def reliability_histogram(
#     predictions: ListOrArray,
#     labels: ListOrArray,
#     confidence_scores: ListOrArray,
#     num_bins: int = DEFAULT_NUM_BINS,
#     example_weights: Optional[ListOrArray] = None
# ) -> Tuple[np.ndarray, np.ndarray, np.ndarray]:
# def risk_reliability_histogram(
#     labels: ListOrArray,
#     risk_scores: ListOrArray,
#     num_bins=DEFAULT_NUM_BINS,
#     example_weights: Optional[ListOrArray] = None
# ) -> Tuple[np.ndarray, np.ndarray, np.ndarray]:
# def expected_calibration_error(observed_score_histogram: ListOrArray,
#                                predicted_score_histogram: ListOrArray,
#                                bin_example_counts: ListOrArray) -> float:
# def maximum_calibration_error(observed_score_histogram: np.ndarray,
#                               predicted_score_histogram: np.ndarray) -> float:
# def reliability_diagram_from_posneg_lists(
#     pos: ListOrArray,
#     neg: ListOrArray,
#     pos_weights: ListOrArray,
#     neg_weights: ListOrArray,
#     num_bins: int = DEFAULT_NUM_BINS,
#     class_threshold: float = 0.5) -> Tuple[np.ndarray, np.ndarray, np.ndarray]:
# def risk_reliability_diagram_from_posneg_lists(
#     pos: ListOrArray,
#     neg: ListOrArray,
#     pos_weights: ListOrArray,
#     neg_weights: ListOrArray,
#     num_bins: int = DEFAULT_NUM_BINS
# ) -> Tuple[np.ndarray, np.ndarray, np.ndarray]:
# 
# Path: ehr_prediction_modeling/metrics/pr_roc.py
# def area_under_curve_from_raw_scores(scores_pos: ListOrArray,
#                                      scores_neg: ListOrArray) -> float:
# def pr_area_under_curve_from_raw_scores(scores_pos: ListOrArray,
#                                         scores_neg: ListOrArray) -> float:
# def weighted_area_under_curve_from_raw_scores(
#     scores_pos: ListOrArray, scores_neg: ListOrArray,
#     weights_pos: ListOrArray, weights_neg: ListOrArray) -> float:
# def weighted_pr_area_under_curve_from_raw_scores(
#     scores_pos: ListOrArray, scores_neg: ListOrArray,
#     weights_pos: ListOrArray, weights_neg: ListOrArray) -> float:
# def roc_curve_from_raw_scores(
#     scores_pos: ListOrArray,
#     scores_neg: ListOrArray,
#     weights_pos: Optional[ListOrArray] = None,
#     weights_neg: Optional[ListOrArray] = None) -> CurvePoints:
# def pr_curve_from_raw_scores(
#     scores_pos: ListOrArray,
#     scores_neg: ListOrArray,
#     weights_pos: Optional[ListOrArray] = None,
#     weights_neg: Optional[ListOrArray] = None) -> CurvePoints:
# def _curve_area(curve: CurvePoints) -> float:
# def _trapezoid_area(point_left: List[float], point_right: List[float]) -> float:
# 
# Path: ehr_prediction_modeling/metrics/calculators/base.py
# class Error(Exception):
# class InvalidClassThresholdError(Error):
# class MetricCalculatorBase():
#   ALL_METRICS = []
#   def get_metrics_dict(self) -> Mapping[str, Any]:


import math
import numpy as np
from typing import List, Mapping, Optional, Union
from ehr_prediction_modeling.metrics import calibration
from ehr_prediction_modeling.metrics import pr_roc
from ehr_prediction_modeling.metrics.calculators import base
# limitations under the License.

# Lint as: python3
"""Metric calculator for classification tasks."""

CONFIDENCE_CALIBRATION_HISTOGRAM_KEY = "full_confidence_calibration_histogram"
RISK_CALIBRATION_HISTOGRAM_KEY = "full_risk_calibration_histogram"

METRICS_EPSILON = 1e-5
ListOrArray = Union[List[Union[float, int]], np.ndarray]


class ClassificationTaskMetricCalculator(base.MetricCalculatorBase):
  """Calculator class for evaluation metrics for classification tasks."""

  ALL_METRICS = [
      "true_positives", "false_positives", "true_negatives", "false_negatives",
      "rocauc", "prauc", "normalised_prauc", "nb_ex", "nb_ex_pos", "nb_ex_neg",
      "tp", "tn", "ppv", "npv", "acc", "f1", "f2", "mcc",
      "expected_confidence_calibration_error",
      "maximum_confidence_calibration_error",
      CONFIDENCE_CALIBRATION_HISTOGRAM_KEY, "expected_risk_calibration_error",
      "maximum_risk_calibration_error", RISK_CALIBRATION_HISTOGRAM_KEY
  ]

  def __init__(self,
               pos: ListOrArray,
               neg: ListOrArray,
               pos_weights: ListOrArray,
               neg_weights: ListOrArray,

# Path: ehr_prediction_modeling/metrics/calibration.py
# DEFAULT_NUM_BINS = 50
# class Error(Exception):
# class MismatchedPredictionArraysError(Error):
# def reliability_histogram(
#     predictions: ListOrArray,
#     labels: ListOrArray,
#     confidence_scores: ListOrArray,
#     num_bins: int = DEFAULT_NUM_BINS,
#     example_weights: Optional[ListOrArray] = None
# ) -> Tuple[np.ndarray, np.ndarray, np.ndarray]:
# def risk_reliability_histogram(
#     labels: ListOrArray,
#     risk_scores: ListOrArray,
#     num_bins=DEFAULT_NUM_BINS,
#     example_weights: Optional[ListOrArray] = None
# ) -> Tuple[np.ndarray, np.ndarray, np.ndarray]:
# def expected_calibration_error(observed_score_histogram: ListOrArray,
#                                predicted_score_histogram: ListOrArray,
#                                bin_example_counts: ListOrArray) -> float:
# def maximum_calibration_error(observed_score_histogram: np.ndarray,
#                               predicted_score_histogram: np.ndarray) -> float:
# def reliability_diagram_from_posneg_lists(
#     pos: ListOrArray,
#     neg: ListOrArray,
#     pos_weights: ListOrArray,
#     neg_weights: ListOrArray,
#     num_bins: int = DEFAULT_NUM_BINS,
#     class_threshold: float = 0.5) -> Tuple[np.ndarray, np.ndarray, np.ndarray]:
# def risk_reliability_diagram_from_posneg_lists(
#     pos: ListOrArray,
#     neg: ListOrArray,
#     pos_weights: ListOrArray,
#     neg_weights: ListOrArray,
#     num_bins: int = DEFAULT_NUM_BINS
# ) -> Tuple[np.ndarray, np.ndarray, np.ndarray]:
# 
# Path: ehr_prediction_modeling/metrics/pr_roc.py
# def area_under_curve_from_raw_scores(scores_pos: ListOrArray,
#                                      scores_neg: ListOrArray) -> float:
# def pr_area_under_curve_from_raw_scores(scores_pos: ListOrArray,
#                                         scores_neg: ListOrArray) -> float:
# def weighted_area_under_curve_from_raw_scores(
#     scores_pos: ListOrArray, scores_neg: ListOrArray,
#     weights_pos: ListOrArray, weights_neg: ListOrArray) -> float:
# def weighted_pr_area_under_curve_from_raw_scores(
#     scores_pos: ListOrArray, scores_neg: ListOrArray,
#     weights_pos: ListOrArray, weights_neg: ListOrArray) -> float:
# def roc_curve_from_raw_scores(
#     scores_pos: ListOrArray,
#     scores_neg: ListOrArray,
#     weights_pos: Optional[ListOrArray] = None,
#     weights_neg: Optional[ListOrArray] = None) -> CurvePoints:
# def pr_curve_from_raw_scores(
#     scores_pos: ListOrArray,
#     scores_neg: ListOrArray,
#     weights_pos: Optional[ListOrArray] = None,
#     weights_neg: Optional[ListOrArray] = None) -> CurvePoints:
# def _curve_area(curve: CurvePoints) -> float:
# def _trapezoid_area(point_left: List[float], point_right: List[float]) -> float:
# 
# Path: ehr_prediction_modeling/metrics/calculators/base.py
# class Error(Exception):
# class InvalidClassThresholdError(Error):
# class MetricCalculatorBase():
#   ALL_METRICS = []
#   def get_metrics_dict(self) -> Mapping[str, Any]:


# Path: ehr_prediction_modeling/metrics/calculators/classification_task.py
import math
import numpy as np
from typing import List, Mapping, Optional, Union
from ehr_prediction_modeling.metrics import calibration
from ehr_prediction_modeling.metrics import pr_roc
from ehr_prediction_modeling.metrics.calculators import base

# limitations under the License.

# Lint as: python3
"""Metric calculator for classification tasks."""

CONFIDENCE_CALIBRATION_HISTOGRAM_KEY = "full_confidence_calibration_histogram"
RISK_CALIBRATION_HISTOGRAM_KEY = "full_risk_calibration_histogram"

METRICS_EPSILON = 1e-5
ListOrArray = Union[List[Union[float, int]], np.ndarray]


class ClassificationTaskMetricCalculator(base.MetricCalculatorBase):
  """Calculator class for evaluation metrics for classification tasks."""

  ALL_METRICS = [
      "true_positives", "false_positives", "true_negatives", "false_negatives",
      "rocauc", "prauc", "normalised_prauc", "nb_ex", "nb_ex_pos", "nb_ex_neg",
      "tp", "tn", "ppv", "npv", "acc", "f1", "f2", "mcc",
      "expected_confidence_calibration_error",
      "maximum_confidence_calibration_error",
      CONFIDENCE_CALIBRATION_HISTOGRAM_KEY, "expected_risk_calibration_error",
      "maximum_risk_calibration_error", RISK_CALIBRATION_HISTOGRAM_KEY
  ]

  def __init__(self,
               pos: ListOrArray,
               neg: ListOrArray,
               pos_weights: ListOrArray,
               neg_weights: ListOrArray,

               num_calibration_bins: int = calibration.DEFAULT_NUM_BINS,