<jupyter_start><jupyter_text>ROC-AUCIn this notebook I compare the display of ROC and PR curves in a balanced and imbalanced dataset, which have similar class separability.We can change the separability parameter when we create the toy dataset, to compare results on datasets with different degree of class separability.<jupyter_code>import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import seaborn as sns

from sklearn.datasets import make_blobs
from sklearn.ensemble import RandomForestClassifier
from sklearn.model_selection import train_test_split

from sklearn.metrics import (
    roc_auc_score,
    plot_precision_recall_curve,
    plot_roc_curve,
)<jupyter_output><empty_output><jupyter_text>Create data<jupyter_code># function to create data

def create_data(imbalanced=True, separability=3):
    # Configuration options
    blobs_random_seed = 42
    centers = [(0, 0), (separability, separability)]
    cluster_std = 1.5
    num_features_for_samples = 2
    num_samples_total = 1600

    # Generate X
    X, y = make_blobs(
        n_samples=num_samples_total,
        centers=centers,
        n_features=num_features_for_samples,
        cluster_std=cluster_std)

    # transform arrays to pandas formats
    X = pd.DataFrame(X, columns=['VarA', 'VarB'])
    y = pd.Series(y)
    
    if imbalanced:
    # create an imbalancced Xset
    # (make blobs creates same number of obs per class
    # we need to downsample manually)
        X = pd.concat([
            X[y == 0].sample(1500,replace=True, random_state=34),
            X[y == 1].sample(100, random_state=42)
        ], axis=0)

        y = y.loc[X.index]

    # display size
    print(X.shape, y.shape)
    return X, y<jupyter_output><empty_output><jupyter_text>Create balanced data<jupyter_code>X, y = create_data(imbalanced=False, separability=3)

sns.scatterplot(
    data=X, x="VarA", y="VarB", hue=y, alpha=0.5
)

plt.title('Toy dataset')
plt.show()
# imbalanced target

y.value_counts() / len(y)<jupyter_output><empty_output><jupyter_text>Create imbalanced dataset<jupyter_code>X_im, y_im = create_data(imbalanced=True, separability=3)

sns.scatterplot(
    data=X_im, x="VarA", y="VarB", hue=y_im, alpha=0.5
)

plt.title('Toy dataset')
plt.show()
# imbalanced target

y_im.value_counts() / len(y_im)<jupyter_output><empty_output><jupyter_text>separate in train and test<jupyter_code># separate dataset into train and test

X_train, X_test, y_train, y_test = train_test_split(
    X, y, test_size=0.3, random_state=0)

X_train.shape, X_test.shape
# separate dataset into train and test

X_train_im, X_test_im, y_train_im, y_test_im = train_test_split(
    X_im, y_im, test_size=0.3, random_state=0)

X_train_im.shape, X_test_im.shape<jupyter_output><empty_output><jupyter_text>Train Random Forests<jupyter_code>rf = RandomForestClassifier(n_estimators=100, random_state=39, max_depth=2, n_jobs=4)

rf.fit(X_train, y_train)
rf_im = RandomForestClassifier(n_estimators=100, random_state=39, max_depth=2, n_jobs=4)

rf_im.fit(X_train_im, y_train_im)<jupyter_output><empty_output><jupyter_text>ROC-AUC<jupyter_code>print('ROC-AUC Balanced data test:', roc_auc_score(y_test, rf.predict_proba(X_test)[:,1]))
print('ROC-AUC Imbalanced data test:', roc_auc_score(y_test_im, rf_im.predict_proba(X_test_im)[:,1]))<jupyter_output>ROC-AUC Balanced data test: 0.970719546939059
ROC-AUC Imbalanced data test: 0.9559151785714286<jupyter_text>Plot ROC curvehttps://scikit-learn.org/stable/modules/generated/sklearn.metrics.plot_roc_curve.html<jupyter_code>rf_disp = plot_roc_curve(rf, X_test, y_test)
rf_im_disp = plot_roc_curve(rf_im, X_test_im, y_test_im)

ax = plt.gca()
rf_disp.plot(ax=ax, alpha=0.8)
rf_im_disp.plot(ax=ax, alpha=0.8)<jupyter_output><empty_output><jupyter_text>PRC<jupyter_code>rf_disp = plot_precision_recall_curve(rf, X_test, y_test)
rf_im_disp = plot_precision_recall_curve(rf_im, X_test_im, y_test_im)

ax = plt.gca()
rf_disp.plot(ax=ax, alpha=0.8)
rf_im_disp.plot(ax=ax, alpha=0.8)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.10798706145807414, 'fraction_numerical': 0.02363772082607614, 'mean_word_length': 5.5793780687397705, 'pattern_counts': {'":': 0, '<': 36, '<?xml version=': 0, '>': 36, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '9', 'dir': 'jupyter-structured-clean-dedup', 'id': '298876', 'license': 'BSD-3-Clause', 'n_tokens_mistral': 1625, 'n_tokens_neox': 1540, 'n_words': 297, 'path': 'Section-03-Metrics/03-08-Optional-Comparison-ROC-PRC.ipynb', 'repo_name': 'bkiselgof/machine-learning-imbalanced-data', 'size': '4019'}