<jupyter_start><jupyter_text>Optimizing a performance metricIn this notebook, we are going to see how we can optimize a metric in a grid search for hyperparameters utilizing sklearn.<jupyter_code>import pandas as pd
import matplotlib.pyplot as plt

from sklearn.datasets import load_breast_cancer
from sklearn.ensemble import RandomForestClassifier
from sklearn.model_selection import GridSearchCV
# if you want more information about the dataset for this demo:

# scikit-learn dataset
# https://scikit-learn.org/stable/datasets/toy_dataset.html#breast-cancer-dataset

# dataset information: UCI Machine Learning Repository
# https://archive.ics.uci.edu/ml/datasets/Breast+Cancer+Wisconsin+(Diagnostic)
    
# in short, classification problem, trying to predict whether the tumor
# is malignant or benign

# load dataset
breast_cancer_X, breast_cancer_y = load_breast_cancer(return_X_y=True)
X = pd.DataFrame(breast_cancer_X)
y = pd.Series(breast_cancer_y).map({0:1, 1:0})

X.head()
# percentage of benign (0) and malign tumors (1)

y.value_counts() / len(y)
# random forests
rf_model = RandomForestClassifier(
    n_estimators=100, max_depth=1, random_state=0, n_jobs=4)

# hyperparameter space
rf_param_grid = dict(
    n_estimators=[20, 50, 100, 200, 500, 1000],
    max_depth=[2, 3, 4],
)

# search
clf = GridSearchCV(rf_model,
                   rf_param_grid,
                   scoring='roc_auc',
                   cv=5)

search = clf.fit(X, y)

# best hyperparameters
search.best_params_
results = pd.DataFrame(search.cv_results_)[['params', 'mean_test_score', 'std_test_score']]

results.head()
results.sort_values(by='mean_test_score', ascending=False, inplace=True)

results.reset_index(drop=True, inplace=True)

results['mean_test_score'].plot(yerr=[results['std_test_score'], results['std_test_score']], subplots=True)

plt.ylabel('Mean ROC_AUC')
plt.xlabel('Hyperparameter space')<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.10649350649350649, 'fraction_numerical': 0.017142857142857144, 'mean_word_length': 5.398671096345515, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '377550', 'license': 'BSD-3-Clause', 'n_tokens_mistral': 693, 'n_tokens_neox': 642, 'n_words': 150, 'path': 'Section-03-Metrics/03-02-Optimizing-an-available-metric.ipynb', 'repo_name': 'ankitario/hyperparameter-optimization', 'size': '1925'}