def test_auc_ci():
    "Validates the AUC confidence interval by comparing with R's pROC"

    def checkme(y_true, y_pred):
        'Test utility'
        roc = ROC.from_scores(y_true, y_pred)
        print(roc.auc_ci)
        np.testing.assert_allclose(roc.auc_ci.estimate, roc.auc, atol=0.01)
        proc = importr('pROC')
        r_ci_obj = proc.ci(proc.roc(FloatVector(y_true), FloatVector(y_pred), ci=True), method='bootstrap')
        r_ci_dict = dict(list(r_ci_obj.items()))
        np.testing.assert_allclose(r_ci_dict['2.5%'], roc.auc_ci.low, atol=0.02)
        np.testing.assert_allclose(r_ci_dict['97.5%'], roc.auc_ci.high, atol=0.02)
    np.random.seed(12648430)
    (yield (checkme, ([1, 1, 1, 1, 0, 0, 0, 0] * 10), ([1, 1, 1, 1, 0, 0, 0, 0] * 10)))
    (yield (checkme, ([1, 1, 1, 1, 0, 0, 0, 0] * 10), ([1, 0, 1, 0, 1, 0, 1, 0] * 10)))
    for _ in range(5):
        y_true = np.random.choice([0, 1], size=100)
        y_pred = np.random.normal(size=100)
        y_pred[(y_true == 1)] += np.abs(np.random.normal())
        (yield (checkme, y_true, y_pred))
Validates the AUC confidence interval by comparing with R's pROC
def test_auc_ci():
    

    def checkme(y_true, y_pred):
        'Test utility'
        roc = ROC.from_scores(y_true, y_pred)
        print(roc.auc_ci)
        np.testing.assert_allclose(roc.auc_ci.estimate, roc.auc, atol=0.01)
        proc = importr('pROC')
        r_ci_obj = proc.ci(proc.roc(FloatVector(y_true), FloatVector(y_pred), ci=True), method='bootstrap')
        r_ci_dict = dict(list(r_ci_obj.items()))
        np.testing.assert_allclose(r_ci_dict['2.5%'], roc.auc_ci.low, atol=0.02)
        np.testing.assert_allclose(r_ci_dict['97.5%'], roc.auc_ci.high, atol=0.02)
    np.random.seed(12648430)
    (yield (checkme, ([1, 1, 1, 1, 0, 0, 0, 0] * 10), ([1, 1, 1, 1, 0, 0, 0, 0] * 10)))
    (yield (checkme, ([1, 1, 1, 1, 0, 0, 0, 0] * 10), ([1, 0, 1, 0, 1, 0, 1, 0] * 10)))
    for _ in range(5):
        y_true = np.random.choice([0, 1], size=100)
        y_pred = np.random.normal(size=100)
        y_pred[(y_true == 1)] += np.abs(np.random.normal())
        (yield (checkme, y_true, y_pred))