<jupyter_start><jupyter_text>Cross-ValidationIn this notebook, we will estimate the generalization error of a machine learning model using Cross-Validation Schemes.<jupyter_code>import numpy as np
import pandas as pd
from scipy.special import comb

from sklearn.datasets import load_breast_cancer
from sklearn.linear_model import LogisticRegression
from sklearn.metrics import accuracy_score

from sklearn.model_selection import (
    KFold,
    RepeatedKFold,
    LeaveOneOut,
    LeavePOut,
    StratifiedKFold,
    cross_validate,
    train_test_split,
)
# if you want more information about the dataset for this demo:

# scikit-learn dataset
# https://scikit-learn.org/stable/datasets/toy_dataset.html#breast-cancer-dataset

# dataset information: UCI Machine Learning Repository
# https://archive.ics.uci.edu/ml/datasets/Breast+Cancer+Wisconsin+(Diagnostic)
    
# in short, classification problem, trying to predict whether the tumor
# is malignant or benign

# load dataset
breast_cancer_X, breast_cancer_y = load_breast_cancer(return_X_y=True)
X = pd.DataFrame(breast_cancer_X)
y = pd.Series(breast_cancer_y).map({0:1, 1:0})

X.head()
# percentage of benign (0) and malign tumors (1)

y.value_counts() / len(y)
# split dataset into a train and test set

X_train, X_test, y_train, y_test = train_test_split(
    X, y, test_size=0.3, random_state=0)

X_train.shape, X_test.shape<jupyter_output><empty_output><jupyter_text>K-Fold Cross-Validation<jupyter_code># Logistic Regression
logit = LogisticRegression(
    penalty ='l2', C=10, solver='liblinear', random_state=4, max_iter=10000)

# K-Fold Cross-Validation
kf = KFold(n_splits=5, shuffle=True, random_state=4)

# estimate generalization error
clf =  cross_validate(
    logit,
    X_train, 
    y_train,
    scoring='accuracy',
    return_train_score=True,
    cv=kf, # k-fold
)

clf['test_score']
clf['train_score']
print('mean train set accuracy: ', np.mean(clf['train_score']), ' +- ', np.std(clf['train_score']))
print('mean test set accuracy: ', np.mean(clf['test_score']), ' +- ', np.std(clf['test_score']))<jupyter_output>mean train set accuracy:  0.9679659312710711  +-  0.003649325403713798
mean test set accuracy:  0.9497784810126582  +-  0.013608919570498668<jupyter_text>Repeated K-Fold<jupyter_code># Logistic Regression
logit = LogisticRegression(
    penalty ='l2', C=1, solver='liblinear', random_state=4, max_iter=10000)

# Repeated K-Fold Cross-Validation
rkf = RepeatedKFold(
    n_splits=5,
    n_repeats=10,
    random_state=4,
)

print('We expect K * n performance metrics: ', 5*10)

# estimate generalization error
clf =  cross_validate(
    logit,
    X_train, 
    y_train,
    scoring='accuracy',
    return_train_score=True,
    cv=rkf, # k-fold
)

print('Number of metrics obtained: ', len(clf['test_score']))

clf['test_score']
print('mean train set accuracy: ', np.mean(clf['train_score']), ' +- ', np.std(clf['train_score']))
print('mean test set accuracy: ', np.mean(clf['test_score']), ' +- ', np.std(clf['test_score']))<jupyter_output>mean train set accuracy:  0.9603627688728534  +-  0.006592970819977563
mean test set accuracy:  0.9454651898734177  +-  0.023290636388696693<jupyter_text>Leave One Out<jupyter_code># Logistic Regression
logit = LogisticRegression(
    penalty ='l2', C=1, solver='liblinear', random_state=4, max_iter=10000)

# Leave One Out Cross-Validation
loo = LeaveOneOut()

print('We expect as many metrics as data in the train set: ', len(X_train))

# estimate generalization error
clf =  cross_validate(
    logit,
    X_train, 
    y_train,
    scoring='accuracy',
    return_train_score=True,
    cv=loo, # k-fold
)

print('Number of metrics obtained: ', len(clf['test_score']))

len(clf['test_score'])
print('mean train set accuracy: ', np.mean(clf['train_score']), ' +- ', np.std(clf['train_score']))
print('mean test set accuracy: ', np.mean(clf['test_score']), ' +- ', np.std(clf['test_score']))<jupyter_output>mean train set accuracy:  0.9575079427363519  +-  0.0012575839695838692
mean test set accuracy:  0.9472361809045227  +-  0.22356162123660028<jupyter_text>Leave P Out<jupyter_code># Logistic Regression
logit = LogisticRegression(
    penalty ='l2', C=1, solver='liblinear', random_state=4, max_iter=10000)

# Leave P Out Cross-Validation
lpo = LeavePOut(p=2)

# I take a smaller sample of the data, otherwise
# my computer runs out of memory
X_train_small = X_train.head(100)
y_train_small = y_train.head(100)

# The number of combinations of N things taken k at a time.
print('We expect : ', comb(100,2), ' metrics')


# estimate generalization error
clf =  cross_validate(
    logit,
    X_train_small, 
    y_train_small,
    scoring='accuracy',
    return_train_score=True,
    cv=lpo, # k-fold
)

print('Number of metrics obtained: ', len(clf['test_score']))
print('mean train set accuracy: ', np.mean(clf['train_score']), ' +- ', np.std(clf['train_score']))
print('mean test set accuracy: ', np.mean(clf['test_score']), ' +- ', np.std(clf['test_score']))<jupyter_output>mean train set accuracy:  0.9700020614306328  +-  0.0032367717044687024
mean test set accuracy:  0.9190909090909091  +-  0.19298082235326716<jupyter_text>Stratified K-Fold Cross-Validation<jupyter_code># Logistic Regression
logit = LogisticRegression(
    penalty ='l2', C=1, solver='liblinear', random_state=4, max_iter=10000)

# Leave P Out Cross-Validation
skf = StratifiedKFold(n_splits=5, shuffle=True, random_state=4)

# estimate generalization error
clf =  cross_validate(
    logit,
    X_train, 
    y_train,
    scoring='accuracy',
    return_train_score=True,
    cv=skf, # k-fold
)

len(clf['test_score'])
print('mean train set accuracy: ', np.mean(clf['train_score']), ' +- ', np.std(clf['train_score']))
print('mean test set accuracy: ', np.mean(clf['test_score']), ' +- ', np.std(clf['test_score']))<jupyter_output>mean train set accuracy:  0.9623094970525029  +-  0.004457316368446095
mean test set accuracy:  0.944620253164557  +-  0.02364900328794808
{'fraction_non_alphanumeric': 0.1300785821768935, 'fraction_numerical': 0.07189433205149641, 'mean_word_length': 5.237747653806048, 'pattern_counts': {'":': 0, '<': 20, '<?xml version=': 0, '>': 20, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '6', 'dir': 'jupyter-structured-clean-dedup', 'id': '582714', 'license': 'BSD-3-Clause', 'n_tokens_mistral': 2447, 'n_tokens_neox': 2182, 'n_words': 466, 'path': 'Section-04-Cross-Validation/04-01-Cross-Validation.ipynb', 'repo_name': 'ankitario/hyperparameter-optimization', 'size': '5981'}