<jupyter_start><jupyter_text>Multi-fidelity demoIn this notebook, we show how we can already obtain a good idea of where the best hyperparameters lie with low fidelity models.Low fidelity models are those that would deliver a sub-optimal performance but at a much lower computational cost.<jupyter_code>import datetime

import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
from scipy import stats

from sklearn.datasets import load_breast_cancer
from sklearn.model_selection import (
    RandomizedSearchCV,
    train_test_split,
)

# the models to optimize
from sklearn.ensemble import RandomForestClassifier
from sklearn.svm import SVC
# load dataset

breast_cancer_X, breast_cancer_y = load_breast_cancer(return_X_y=True)

X_train = pd.DataFrame(breast_cancer_X)
y_train = pd.Series(breast_cancer_y).map({0:1, 1:0})

X_train.head()
# the target:
# percentage of benign (0) and malign tumors (1)

y_train.value_counts() / len(y_train)<jupyter_output><empty_output><jupyter_text>Multi-fidelity data size<jupyter_code># set up the model
svm_model = SVC(kernel='rbf', random_state=0)

# determine the hyperparameter space
# we will search over 2 hyperparameters for this demo

param_grid = dict(
    C = stats.loguniform(0.001, 100),
    gamma = stats.loguniform(0.001, 100),
)
# set up the search

search = RandomizedSearchCV(svm_model,
                            param_grid,
                            scoring='accuracy',
                            cv=2,
                            n_iter = 50,
                            random_state=10,
                            n_jobs=4,
                            refit=False,
                           )
# determine the best hyperparameters and performance
# searching over datasets of different sizes
# that is, different fidelities

print('Darker orange means better accuracy')

for size in [0.1, 0.2, 0.4, 0.8]:

    # subsample the data
    n_samples = int(size * len(X_train))

    X_train_ = X_train.sample(n_samples, random_state=0)
    y_train_ = y_train[X_train_.index]

    # search
    ts = datetime.datetime.now()

    search.fit(X_train_, y_train_)

    td = (datetime.datetime.now()-ts).microseconds

    results = pd.DataFrame(search.cv_results_)

    # plot results
    plt.scatter(results['param_gamma'],
                results['param_C'],
                c=results['mean_test_score'],
                cmap='Wistia',
               )
    
    plt.title(f"Data fraction: {size}, time: {td}")
    plt.xlabel('gamma')
    plt.ylabel('C')
    plt.show()<jupyter_output>Darker orange means better accuracy<jupyter_text>For this particular dataset, utilizing 20% of the data already tells us where the best hyperparameter combination lies.  number of estimators<jupyter_code># set up the model
rf = RandomForestClassifier(random_state=0)

# determine the hyperparameter space
# we will search over 2 hyperparameters for this demo

param_grid = dict(
    min_samples_split=stats.uniform(0, 1),
    max_depth=stats.randint(1, 5),
    )
# set up the search

search = RandomizedSearchCV(rf,
                            param_grid,
                            scoring='accuracy',
                            cv=2,
                            n_iter = 50,
                            random_state=10,
                            n_jobs=4,
                            refit=False,
                           )
# determine the best hyperparameters and performance
# searching over models with different number of trees
# that is, different fidelities

print('Darker orange means better accuracy')

for n_estimators in [5, 10, 20, 50]:

    # fix the n_estimators parameter
    rf.set_params(**{'n_estimators': n_estimators})

    # search
    ts = datetime.datetime.now()

    search.fit(X_train, y_train)

    td = (datetime.datetime.now()-ts).seconds

    results = pd.DataFrame(search.cv_results_)

    # plot results
    plt.scatter(results['param_min_samples_split'],
                results['param_max_depth'],
                c=results['mean_test_score'],
                cmap='Wistia',
               )
    
    plt.title(f"n_estimators: {n_estimators}, time: {td}")
    plt.xlabel('min_samples_split')
    plt.ylabel('max_depth')
    plt.show()<jupyter_output>Darker orange means better accuracy
{'fraction_non_alphanumeric': 0.08771929824561403, 'fraction_numerical': 0.013567251461988304, 'mean_word_length': 2.5692821368948247, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '503289', 'license': 'BSD-3-Clause', 'n_tokens_mistral': 1344, 'n_tokens_neox': 1270, 'n_words': 333, 'path': 'Section-08-Multi-fidelity-Optimization/01-Multi--fidelity-demo.ipynb', 'repo_name': 'ankitario/hyperparameter-optimization', 'size': '4275'}