<jupyter_start><jupyter_text>Search algorithms within OptunaWe can select the search algorithm from the [optuna.study.create_study()](https://optuna.readthedocs.io/en/stable/reference/generated/optuna.study.create_study.htmloptuna.study.create_study) class.<jupyter_code>import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
from sklearn.preprocessing import MinMaxScaler
import xgboost as xgb

from sklearn.datasets import load_breast_cancer
from sklearn.metrics import accuracy_score, roc_auc_score
from sklearn.model_selection import cross_val_score, train_test_split
from sklearn.ensemble import RandomForestClassifier

import optuna<jupyter_output><empty_output><jupyter_text>Data Pre-processingThe data is related with direct marketing campaigns of a Portuguese banking institution. The marketing campaigns were based on phone calls. Often, more than one contact to the same client was required, in order to access if the product (bank term deposit) would be ('yes') or not ('no') subscribed.<jupyter_code>data = pd.read_csv('balanced_bank.csv', index_col = 0)
data
data.info()
data.head()
data.columns
data.loan.value_counts()
data.isnull().sum()<jupyter_output><empty_output><jupyter_text>Categorical Encoding<jupyter_code>categorical_cols = ['job','marital','education','month','day_of_week'] 
data_encoded = pd.get_dummies(data, drop_first=True,columns = categorical_cols)<jupyter_output><empty_output><jupyter_text>Binary encoding<jupyter_code>binary_cat_cols = ['contact','housing','loan','poutcome','default']

for i in binary_cat_cols:
    if len(data_encoded[i].unique())==2:
        data_encoded[i].replace({data_encoded[i].unique()[0]: 0, data_encoded[i].unique()[1]: 1, 'unknown':-1}, inplace=True)
    else:
        data_encoded[i].replace({data_encoded[i].unique()[0]: 0, data_encoded[i].unique()[1]: 1, data_encoded[i].unique()[2]: 2, 'unknown':-1}, inplace=True)<jupyter_output><empty_output><jupyter_text>Label encoding<jupyter_code>from sklearn.preprocessing import LabelEncoder

label_encoder = LabelEncoder()
data_encoded['y'] = label_encoder.fit_transform(data_encoded['y'])
data_encoded.dtypes<jupyter_output><empty_output><jupyter_text>Scaling<jupyter_code>print(data_encoded.shape)

X = data_encoded.loc[:,data_encoded.columns!='y']
print(X.shape)

y = data_encoded.loc[:,'y']
print(y.shape)
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.33, random_state=1)
scaler = MinMaxScaler()
scaler.fit(X_train)
X_train = scaler.transform(X_train)
X_test = scaler.transform(X_test)<jupyter_output><empty_output><jupyter_text>Baseline model<jupyter_code>model = xgb.XGBClassifier()  
model.fit(X_train, y_train)
preds = model.predict(X_test)
accuracy = accuracy_score(y_test, preds)
accuracy<jupyter_output><empty_output><jupyter_text>HyperParameter optimization with OptunaIn this notebook, I will demo how to select the search algorithm with Optuna. We will compare the use of:- Randomized search- Tree-structured Parzen Estimators- CMA-ES  Define the objective functionThis is the hyperparameter response space, the function we want to minimize.<jupyter_code>def objective(trial):
    param = {
        "objective": 'binary:logistic',
        'reg_lambda': trial.suggest_loguniform('lambda', 1e-3, 10.0),
        'reg_alpha': trial.suggest_loguniform('alpha', 1e-3, 10.0),
        'gamma':trial.suggest_loguniform('gamma', 1e-3,1),
        'colsample_bytree': trial.suggest_categorical('colsample_bytree', [i/10.0 for i in range(4,11)]),
        'subsample': trial.suggest_categorical('subsample', [i/10.0 for i in range(4,11)]),
        'learning_rate': trial.suggest_categorical('learning_rate', [0.008,0.009,0.01,0.012,0.014,0.016,0.018, 0.02,0.300000012]),
        'n_estimators': trial.suggest_int('n_estimators',100,500),
        'max_depth': trial.suggest_categorical('max_depth', [5,6,7,9,11,13,15,17,20]),
        'min_child_weight': trial.suggest_int('min_child_weight', 1, 300),
    }
    model = xgb.XGBClassifier(**param)  
    model.fit(X_train, y_train)
    preds = model.predict(X_test)
    
    accuracy = accuracy_score(y_test, preds)
    
    return accuracy
random_study = optuna.create_study(
    direction="maximize",
    sampler=optuna.samplers.RandomSampler(),
)

tpe_study = optuna.create_study(
    direction="maximize",
    sampler=optuna.samplers.TPESampler(),
)

cmaes_study = optuna.create_study(
    direction="maximize",
    sampler=optuna.samplers.CmaEsSampler(),
)

cmaes_study.optimize(objective, n_trials=100)
tpe_study.optimize(objective, n_trials=100)
random_study.optimize(objective, n_trials=100)

search_space = {
    "n_estimators": [100, 500, 1000],
    "max_depth": [5,6,7,9,11,13,15,17,20],
    "min_samples_split": [0.1, 1.0],
    'min_child_weight': [1, 5, 10],
    'gamma': [0.5, 1, 1.5, 2, 5],
    'subsample': [0.6, 0.8, 1.0],
    'colsample_bytree': [0.6, 0.8, 1.0],

}

def objective(trial):
    
    
    param = {
        'n_estimators': trial.suggest_int('n_estimators',100,500),
        'max_depth': trial.suggest_categorical('max_depth', [5,6,7,9,11,13,15,17,20]),
    }
    model = xgb.XGBClassifier(**param)  
    model.fit(X_train, y_train)
    preds = model.predict(X_test)
    
    accuracy = accuracy_score(y_test, preds)
    
    return accuracy
grid_study = optuna.create_study(
    direction="maximize",
    sampler=optuna.samplers.GridSampler(search_space),
)

grid_study.optimize(objective)
tstamp1 = cmaes_study.trials_dataframe()['datetime_complete'].min()
tstamp2 = grid_study.trials_dataframe()['datetime_complete'].max()
temp = tstamp1 - tstamp2 if tstamp1 > tstamp2 else tstamp2 - tstamp1
temp
print('GRID SEARCH: ',study.best_value)
print('CMAES: ',cmaes_study.best_value)
print('TPE: ',tpe_study.best_value)
print('RANDOM SEARCH : ',random_study.best_value)
print('BASELINE MODEL: ',accuracy)
tstamp1 = tpe_study.trials_dataframe()['datetime_complete'].min()
tstamp2 = tpe_study.trials_dataframe()['datetime_complete'].max()
temp = tstamp1 - tstamp2 if tstamp1 > tstamp2 else tstamp2 - tstamp1
temp<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.13631119169276412, 'fraction_numerical': 0.03345970001648261, 'mean_word_length': 5.958715596330276, 'pattern_counts': {'":': 4, '<': 33, '<?xml version=': 0, '>': 35, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '8', 'dir': 'jupyter-structured-clean-dedup', 'id': '116563', 'license': 'Apache-2.0', 'n_tokens_mistral': 2338, 'n_tokens_neox': 2193, 'n_words': 385, 'path': 'Optuna + XgBoost.ipynb', 'repo_name': 'NijatZeynalov/optuna-hyperparameter-optimization', 'size': '6067'}