<jupyter_start><jupyter_text>Import Libraries<jupyter_code>import sys
import warnings
from typing import List
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import seaborn as sns
from sklearn.ensemble import RandomForestRegressor
from sklearn.linear_model import SGDRegressor
from sklearn.neighbors import KNeighborsRegressor
from xgboost import XGBRegressor
from skopt.space import Real, Categorical, Integer
sys.path.append("../src")
warnings.simplefilter('ignore')
from preprocessing.load_data import load_processed
from models.tune_models import GridSearchTuner, RandomSearchTuner, BayesianSearchTuner<jupyter_output><empty_output><jupyter_text>Functions<jupyter_code>def check_param_estimate_with_model(estimator, parameter: str, parameter_values: List, metric="mae", test_on_new_data=True, verbose=True):
    parameter_grid = {parameter: parameter_values}
    model = GridSearchTuner(data= df, estimator=estimator, params=parameter_grid, cv=5, metric=metric)
    
    if verbose:
        print("searching for the best parameters...")
        best_score, best_params = model.search_best()
        print("found best parameters...")
        print(f"Best score: {best_score}")
        print(f"Best params: {best_params}")
        
    if test_on_new_data:
        # check if model still overfits
        print("Estimate of the data on a new sample data: ")
        model.validate()
        model.evaluate_model("mae")
    return model
def plot_params(model, parameter_values: List, plot_title: str, x_label: str, y_label: str, verbose: bool = False):
    means = model.model.cv_results_['mean_test_score']
    stds = model.model.cv_results_['std_test_score']
    params = model.model.cv_results_['params']
    
    if verbose:
        for mean, stdev, param in zip(means, stds, params):
            print("%f (%f) with: %r" % (mean, stdev, param))
  
    plt.errorbar(parameter_values, means, yerr=stds)
    plt.title(plot_title)
    plt.xlabel(x_label)
    plt.ylabel(y_label)
    plt.show()<jupyter_output><empty_output><jupyter_text>Data Loading<jupyter_code>df = load_processed("../data/bmw.csv")<jupyter_output><empty_output><jupyter_text>HyperParameter Tuning  Random Forest   Tuning with Random Sampling<jupyter_code># define parameter search space
n_estimators = [100, 200, 500, 750]
max_depth = [5, 10, 20, 50, None]

rf_params = {'n_estimators': n_estimators,
               'max_depth': max_depth}
# search for best parameters
random_rf = RandomSearchTuner(data= df, estimator=RandomForestRegressor, params = rf_params, cv=4, n_iter=50, metric="mae", n_jobs=1)
best_score, best_params = random_rf.search_best()
print(f"Best score: {best_score}")
print(f"Best params: {best_params}")
# check if model still overfits
random_rf.validate()
random_rf.evaluate_model("mae")<jupyter_output>The Train mae: 784.8855860100639
The Test mae: 1994.0362731777757<jupyter_text>Tuning with Grid Search<jupyter_code># create the grid for the parameters
grid_params = {'n_estimators': [200, 250, 300, 350], 'max_depth': [20, 35, 45]}
grid_rf = GridSearchTuner(data= df, estimator=RandomForestRegressor, params = grid_params, cv=4, metric="mae")
best_score, best_params = grid_rf.search_best()
print(f"Best score: {best_score}")
print(f"Best params: {best_params}")
# check if model still overfits
grid_rf.validate()
grid_rf.evaluate_model("mae")<jupyter_output>The Train mae: 781.9009185293344
The Test mae: 1984.2777404408762<jupyter_text>Tuning with Bayesian Search<jupyter_code># define the search space for the parameters
search_params = {
     'n_estimators': Integer(200,400),
     'max_depth': Integer(20, 200),
    'min_samples_leaf': Integer(1, 20),
    'min_samples_split': Integer(2, 20)
      }
bayes_rf = BayesianSearchTuner(data= df, estimator=RandomForestRegressor, params = search_params, cv=5, n_iter=100, metric="mae")
best_score, best_params = bayes_rf.search_best()
print(f"Best score: {best_score}")
print(f"Best params: {best_params}")
bayes_rf.validate()
bayes_rf.evaluate_model("mae")<jupyter_output>The Train mae: 1036.7342301101246
The Test mae: 1975.7707397146223<jupyter_text>XgBoost  Tuning with Random Sampling<jupyter_code># define the parameter search space for XgBoost
n_estimators = [100, 200, 500]
max_depth = [1, 3, 5, 7, 9]
learning_rate = [0.001, 0.01, 0.1, 0.2, 0.3]
subsample = [0.1, 0.5, 0.7, 1.0]
colsample_bytree = [0.1, 0.4, 0.5, 0.8, 1.0]
reg_alpha = [0.001, 0.01, 0.1, 1.0, 2.0]
reg_lambda = [0.001, 0.01, 0.1, 1.0, 2.0]

            
xgb_params = {'n_estimators': n_estimators,
             'max_depth': max_depth,
             'learning_rate': learning_rate,
             'subsample': subsample,
             'colsample_bytree': colsample_bytree,
             'reg_alpha': reg_alpha,
              'reg_lambda': reg_lambda
             }
# find the best hyperparameters
random_xgb = RandomSearchTuner(data= df, estimator=XGBRegressor, params = xgb_params, cv=4, n_iter=50, metric="mae")
best_score, best_params = random_xgb.search_best()
print(f"Best score: {best_score}")
print(f"Best params: {best_params}")
# check if the model overfits
random_xgb.validate()
random_xgb.evaluate_model("mae")<jupyter_output>The Train mae: 984.9749992673164
The Test mae: 1948.0504411244392<jupyter_text>Tuning with Grid Search  Tune number of trees<jupyter_code>parameter_values=[100, 150, 300, 350, 500, 1000]
grid_xgb = check_param_estimate_with_model(estimator=XGBRegressor, parameter='n_estimators', parameter_values=parameter_values)
plot_params(model=grid_xgb, parameter_values=parameter_values, plot_title="XGB Boost number trees vs MAE", x_label="n_estimators", y_label="MAE", verbose=False)<jupyter_output><empty_output><jupyter_text>Tune max_dept<jupyter_code>parameter_values=[1, 3, 5, 7, 9, 12, 15, 25]
grid_xgb = check_param_estimate_with_model(estimator=XGBRegressor, parameter='max_depth', parameter_values=parameter_values)
plot_params(model=grid_xgb, parameter_values=parameter_values, plot_title="XGBoost max_depth vs MAE", x_label="max_depth", y_label="MAE", verbose=False)<jupyter_output><empty_output><jupyter_text>Tune number of learning_rate<jupyter_code>parameter_values=[0.001, 0.01, 0.1, 0.2, 0.25, 0.3, 0.4, 0.6, 0.8, 0.9, 1, 2]
grid_xgb = check_param_estimate_with_model(estimator=XGBRegressor, parameter='learning_rate', parameter_values=parameter_values)
plot_params(model=grid_xgb, parameter_values=parameter_values, plot_title="XGBoost learning_rate vs MAE", x_label="learning_rate", y_label="MAE", verbose=False)<jupyter_output><empty_output><jupyter_text>Tune number of subsample<jupyter_code>parameter_values=[0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1.0]
grid_xgb = check_param_estimate_with_model(estimator=XGBRegressor, parameter='subsample', parameter_values=parameter_values)
plot_params(model=grid_xgb, parameter_values=parameter_values, plot_title="XGBoost subsample vs MAE", x_label="subsample", y_label="MAE", verbose=False)<jupyter_output><empty_output><jupyter_text>Tune number of colsample_bytree<jupyter_code>parameter_values=[0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1.0]
grid_xgb = check_param_estimate_with_model(estimator=XGBRegressor, parameter='colsample_bytree', parameter_values=parameter_values)
plot_params(model=grid_xgb, parameter_values=parameter_values, plot_title="XGBoost colsample_bytree vs MAE", x_label="colsample_bytree", y_label="MAE", verbose=False)<jupyter_output><empty_output><jupyter_text>Tune number of reg_alpha<jupyter_code>parameter_values=[0.0001, 0.001, 0.01, 0.1, 1.0, 2.0, 5.0, 10.0]
grid_xgb = check_param_estimate_with_model(estimator=XGBRegressor, parameter='reg_alpha', parameter_values=parameter_values)
plot_params(model=grid_xgb, parameter_values=parameter_values, plot_title="XGBoost reg_alpha vs MAE", x_label="reg_alpha", y_label="MAE", verbose=False)<jupyter_output><empty_output><jupyter_text>Tune number of reg_lambda<jupyter_code>parameter_values=[0.0001, 0.001, 0.01, 0.1, 1.0, 2.0, 5.0, 10.0]
grid_xgb = check_param_estimate_with_model(estimator=XGBRegressor, parameter='reg_lambda', parameter_values=parameter_values)
plot_params(model=grid_xgb, parameter_values=parameter_values, plot_title="XGBoost reg_lambda vs MAE", x_label="reg_lambda", y_label="MAE", verbose=False)<jupyter_output><empty_output><jupyter_text>XGBoost -> Test all BEST combinations for sanity check<jupyter_code># Test all possible combinations
from models.estimators import XGBoostRegressor
xgb = XGBoostRegressor(df, colsample_bytree=0.5, subsample=1.0, learning_rate=0.2, max_depth=7, n_estimators=100)
xgb.train()
xgb.validate()
xgb.evaluate_model(metric="mae")<jupyter_output>The Train mae: 1365.8734383864162
The Test mae: 1904.9824111557007<jupyter_text>Tuning with Bayesian Search<jupyter_code>refined_params = {"colsample_bytree": Real(0.4, 1), "subsample": Real(0.4, 1), "learning_rate": Real(0.01, 1), "max_dept": Integer(3,15), "n_estimators": Integer(100,300)}
# search best params
bayes_xgb = BayesianSearchTuner(data= df, estimator=XGBRegressor, params = refined_params, cv=5, n_iter=100, metric="mae", verbosity = 0)
best_score, best_params = bayes_xgb.search_best()
print(f"Best score: {best_score}")
print(f"Best params: {best_params}")
bayes_xgb.validate()
bayes_xgb.evaluate_model("mae")<jupyter_output>The Train mae: 1411.8353786325404
The Test mae: 1941.668847427368
{'fraction_non_alphanumeric': 0.12113733905579399, 'fraction_numerical': 0.05515021459227468, 'mean_word_length': 6.754575707154742, 'pattern_counts': {'":': 5, '<': 59, '<?xml version=': 0, '>': 60, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '16', 'dir': 'jupyter-structured-clean-dedup', 'id': '223282', 'license': 'MIT', 'n_tokens_mistral': 3790, 'n_tokens_neox': 3407, 'n_words': 676, 'path': 'notebooks/tune_algoirthms.ipynb', 'repo_name': 'JohnAnih/BMW-used-car-prediction', 'size': '9320'}