# Path: mimic3models/metrics.py
# def print_metrics_multilabel(y_true, predictions, verbose=1):
#     y_true = np.array(y_true)
#     predictions = np.array(predictions)
# 
#     auc_scores = metrics.roc_auc_score(y_true, predictions, average=None)
#     ave_auc_micro = metrics.roc_auc_score(y_true, predictions,
#                                           average="micro")
#     ave_auc_macro = metrics.roc_auc_score(y_true, predictions,
#                                           average="macro")
#     ave_auc_weighted = metrics.roc_auc_score(y_true, predictions,
#                                              average="weighted")
# 
#     if verbose:
#         print("ROC AUC scores for labels:", auc_scores)
#         print("ave_auc_micro = {}".format(ave_auc_micro))
#         print("ave_auc_macro = {}".format(ave_auc_macro))
#         print("ave_auc_weighted = {}".format(ave_auc_weighted))
# 
#     return {"auc_scores": auc_scores,
#             "ave_auc_micro": ave_auc_micro,
#             "ave_auc_macro": ave_auc_macro,
#             "ave_auc_weighted": ave_auc_weighted}
# 
# def print_metrics_binary(y_true, predictions, verbose=1):
#     predictions = np.array(predictions)
#     if len(predictions.shape) == 1:
#         predictions = np.stack([1 - predictions, predictions]).transpose((1, 0))
# 
#     cf = metrics.confusion_matrix(y_true, predictions.argmax(axis=1))
#     if verbose:
#         print("confusion matrix:")
#         print(cf)
#     cf = cf.astype(np.float32)
# 
#     acc = (cf[0][0] + cf[1][1]) / np.sum(cf)
#     prec0 = cf[0][0] / (cf[0][0] + cf[1][0])
#     prec1 = cf[1][1] / (cf[1][1] + cf[0][1])
#     rec0 = cf[0][0] / (cf[0][0] + cf[0][1])
#     rec1 = cf[1][1] / (cf[1][1] + cf[1][0])
#     auroc = metrics.roc_auc_score(y_true, predictions[:, 1])
# 
#     (precisions, recalls, thresholds) = metrics.precision_recall_curve(y_true, predictions[:, 1])
#     auprc = metrics.auc(recalls, precisions)
#     minpse = np.max([min(x, y) for (x, y) in zip(precisions, recalls)])
# 
#     if verbose:
#         print("accuracy = {}".format(acc))
#         print("precision class 0 = {}".format(prec0))
#         print("precision class 1 = {}".format(prec1))
#         print("recall class 0 = {}".format(rec0))
#         print("recall class 1 = {}".format(rec1))
#         print("AUC of ROC = {}".format(auroc))
#         print("AUC of PRC = {}".format(auprc))
#         print("min(+P, Se) = {}".format(minpse))
# 
#     return {"acc": acc,
#             "prec0": prec0,
#             "prec1": prec1,
#             "rec0": rec0,
#             "rec1": rec1,
#             "auroc": auroc,
#             "auprc": auprc,
#             "minpse": minpse}


from mimic3models.metrics import print_metrics_multilabel, print_metrics_binary
import sklearn.utils as sk_utils
import numpy as np
import pandas as pd
import argparse
import json
import os
    n_tasks = 25
    labels_cols = ["label_{}".format(i) for i in range(1, n_tasks + 1)]
    test_df.columns = list(test_df.columns[:2]) + labels_cols

    df = test_df.merge(pred_df, left_on='stay', right_on='stay', how='left', suffixes=['_l', '_r'])
    assert (df['pred_1'].isnull().sum() == 0)
    assert (df['period_length_l'].equals(df['period_length_r']))
    for i in range(1, n_tasks + 1):
        assert (df['label_{}_l'.format(i)].equals(df['label_{}_r'.format(i)]))

    metrics = [('Macro ROC AUC', 'ave_auc_macro'),
               ('Micro ROC AUC', 'ave_auc_micro'),
               ('Weighted ROC AUC', 'ave_auc_weighted')]

    data = np.zeros((df.shape[0], 50))
    for i in range(1, n_tasks + 1):
        data[:, i - 1] = df['pred_{}'.format(i)]
        data[:, 25 + i - 1] = df['label_{}_l'.format(i)]

    results = dict()
    results['n_iters'] = args.n_iters
    ret = print_metrics_multilabel(data[:, 25:], data[:, :25], verbose=0)
    for (m, k) in metrics:
        results[m] = dict()
        results[m]['value'] = ret[k]
        results[m]['runs'] = []

    for i in range(1, n_tasks + 1):
        m = 'ROC AUC of task {}'.format(i)
        results[m] = dict()

# Path: mimic3models/metrics.py
# def print_metrics_multilabel(y_true, predictions, verbose=1):
#     y_true = np.array(y_true)
#     predictions = np.array(predictions)
# 
#     auc_scores = metrics.roc_auc_score(y_true, predictions, average=None)
#     ave_auc_micro = metrics.roc_auc_score(y_true, predictions,
#                                           average="micro")
#     ave_auc_macro = metrics.roc_auc_score(y_true, predictions,
#                                           average="macro")
#     ave_auc_weighted = metrics.roc_auc_score(y_true, predictions,
#                                              average="weighted")
# 
#     if verbose:
#         print("ROC AUC scores for labels:", auc_scores)
#         print("ave_auc_micro = {}".format(ave_auc_micro))
#         print("ave_auc_macro = {}".format(ave_auc_macro))
#         print("ave_auc_weighted = {}".format(ave_auc_weighted))
# 
#     return {"auc_scores": auc_scores,
#             "ave_auc_micro": ave_auc_micro,
#             "ave_auc_macro": ave_auc_macro,
#             "ave_auc_weighted": ave_auc_weighted}
# 
# def print_metrics_binary(y_true, predictions, verbose=1):
#     predictions = np.array(predictions)
#     if len(predictions.shape) == 1:
#         predictions = np.stack([1 - predictions, predictions]).transpose((1, 0))
# 
#     cf = metrics.confusion_matrix(y_true, predictions.argmax(axis=1))
#     if verbose:
#         print("confusion matrix:")
#         print(cf)
#     cf = cf.astype(np.float32)
# 
#     acc = (cf[0][0] + cf[1][1]) / np.sum(cf)
#     prec0 = cf[0][0] / (cf[0][0] + cf[1][0])
#     prec1 = cf[1][1] / (cf[1][1] + cf[0][1])
#     rec0 = cf[0][0] / (cf[0][0] + cf[0][1])
#     rec1 = cf[1][1] / (cf[1][1] + cf[1][0])
#     auroc = metrics.roc_auc_score(y_true, predictions[:, 1])
# 
#     (precisions, recalls, thresholds) = metrics.precision_recall_curve(y_true, predictions[:, 1])
#     auprc = metrics.auc(recalls, precisions)
#     minpse = np.max([min(x, y) for (x, y) in zip(precisions, recalls)])
# 
#     if verbose:
#         print("accuracy = {}".format(acc))
#         print("precision class 0 = {}".format(prec0))
#         print("precision class 1 = {}".format(prec1))
#         print("recall class 0 = {}".format(rec0))
#         print("recall class 1 = {}".format(rec1))
#         print("AUC of ROC = {}".format(auroc))
#         print("AUC of PRC = {}".format(auprc))
#         print("min(+P, Se) = {}".format(minpse))
# 
#     return {"acc": acc,
#             "prec0": prec0,
#             "prec1": prec1,
#             "rec0": rec0,
#             "rec1": rec1,
#             "auroc": auroc,
#             "auprc": auprc,
#             "minpse": minpse}


# Path: mimic3benchmark/evaluation/evaluate_pheno.py
from mimic3models.metrics import print_metrics_multilabel, print_metrics_binary
import sklearn.utils as sk_utils
import numpy as np
import pandas as pd
import argparse
import json
import os

    n_tasks = 25
    labels_cols = ["label_{}".format(i) for i in range(1, n_tasks + 1)]
    test_df.columns = list(test_df.columns[:2]) + labels_cols

    df = test_df.merge(pred_df, left_on='stay', right_on='stay', how='left', suffixes=['_l', '_r'])
    assert (df['pred_1'].isnull().sum() == 0)
    assert (df['period_length_l'].equals(df['period_length_r']))
    for i in range(1, n_tasks + 1):
        assert (df['label_{}_l'.format(i)].equals(df['label_{}_r'.format(i)]))

    metrics = [('Macro ROC AUC', 'ave_auc_macro'),
               ('Micro ROC AUC', 'ave_auc_micro'),
               ('Weighted ROC AUC', 'ave_auc_weighted')]

    data = np.zeros((df.shape[0], 50))
    for i in range(1, n_tasks + 1):
        data[:, i - 1] = df['pred_{}'.format(i)]
        data[:, 25 + i - 1] = df['label_{}_l'.format(i)]

    results = dict()
    results['n_iters'] = args.n_iters
    ret = print_metrics_multilabel(data[:, 25:], data[:, :25], verbose=0)
    for (m, k) in metrics:
        results[m] = dict()
        results[m]['value'] = ret[k]
        results[m]['runs'] = []

    for i in range(1, n_tasks + 1):
        m = 'ROC AUC of task {}'.format(i)
        results[m] = dict()

        results[m]['value'] = print_metrics_binary(data[:, 25 + i - 1], data[:, i - 1], verbose=0)['auroc']