<jupyter_start><jupyter_text>Import library<jupyter_code># !pip install --upgrade tables
# !pip install eli5
# !pip install xgboost
# !pip install hyperopt
import pandas as pd
import numpy as np
import xgboost as xgb

from sklearn.metrics import mean_absolute_error as mae
from sklearn.model_selection import cross_val_score

from hyperopt import hp, fmin, tpe, STATUS_OK

import eli5
from eli5.sklearn import PermutationImportance
cd "drive/My Drive/Colab Notebooks/dw_matrix_car"
df = pd.read_hdf('data/car.h5')
df.shape<jupyter_output><empty_output><jupyter_text>Feature Engineering<jupyter_code>SUFFIX_CAT = '__cat'

for feat in df.columns:
    if isinstance(df[feat][0], list):
        continue
    factorized_values = df[feat].factorize()[0]

    if SUFFIX_CAT in feat:
        continue
    df[feat + SUFFIX_CAT] = factorized_values

df['param_pojemność-skokowa'] = df['param_pojemność-skokowa'].map(lambda x: -1 if str(x) == 'None' else int(str(x).split('cm')[0].replace(' ', '')) )
df['param_moc'] = df['param_moc'].map(lambda x: -1 if str(x) == 'None' else int(x.split(' ')[0]) )
df['param_rok-produkcji'] = df['param_rok-produkcji'].map(lambda x: -1 if str(x) == 'None' else int(x))

def run_model(model, feats):
    X = df[feats].values
    y = df['price_value'].values

    scores = cross_val_score(model, X, y, cv = 3, scoring = 'neg_mean_absolute_error')
    return np.mean(scores), np.std(scores)
feats = ['param_napęd__cat', 'param_rok-produkcji', 'param_stan__cat', 'param_skrzynia-biegów__cat', 'param_faktura-vat__cat', 'param_moc', 'param_marka-pojazdu__cat', 'feature_kamera-cofania__cat', 'param_typ__cat', 'param_pojemność-skokowa', 'seller_name__cat', 'feature_wspomaganie-kierownicy__cat', 'param_model-pojazdu__cat', 'param_wersja__cat', 'param_kod-silnika__cat', 'feature_system-start-stop__cat', 'feature_asystent-pasa-ruchu__cat', 'feature_czujniki-parkowania-przednie__cat', 'feature_łopatki-zmiany-biegów__cat', 'feature_regulowane-zawieszenie__cat']
xgb_parms = {
    'max_depth': 5,
    'n_estimators': 50,
    'learning_rate': 0.1,
    'seed': 0
}

run_model(xgb.XGBRegressor( **xgb_parms), feats)

def obj_func(params):
    mean_mae, score_std = run_model(xgb.XGBRegressor(**params), feats)
   
    print("Training with params:")
    print(params, np.abs(mean_mae))
 
    return {'loss': np.abs(mean_mae), 'status': STATUS_OK}

# space

xgb_reg_params = {
    'learning_rate':    hp.choice('learning_rate',  np.arange(0.05, 0.31, 0.05,dtype=float)),
    'max_depth':        hp.choice('max_depth',      np.arange(5, 16, 1, dtype = int)),
    'subsample':        hp.quniform('subsample', 0.5, 1, 0.05),
    'colsample_bytree': hp.quniform('colsample_bytree', 0.5, 1, 0.05),
    'objective':        'reg:squarederror',
    'n_estimators':     100,
    'seed':             0
}
# run

best = fmin(obj_func, xgb_reg_params, algo = tpe.suggest, max_evals = 5)<jupyter_output>Training with params:
{'colsample_bytree': 0.6000000000000001, 'learning_rate': 0.15000000000000002, 'max_depth': 13, 'n_estimators': 100, 'objective': 'reg:squarederror', 'seed': 0, 'subsample': 0.7000000000000001}
8021.26782298684
Training with params:
{'colsample_bytree': 0.8500000000000001, 'learning_rate': 0.2, 'max_depth': 12, 'n_estimators': 100, 'objective': 'reg:squarederror', 'seed': 0, 'subsample': 0.7000000000000001}
7670.749769854843
Training with params:
{'colsample_bytree': 0.7000000000000001, 'learning_rate': 0.1, 'max_depth': 9, 'n_estimators': 100, 'objective': 'reg:squarederror', 'seed': 0, 'subsample': 0.55}
8072.788374825047
Training with params:
{'colsample_bytree': 0.9500000000000001, 'learning_rate': 0.2, 'max_depth': 14, 'n_estimators': 100, 'objective': 'reg:squarederror', 'seed': 0, 'subsample': 0.9}
7661.6167853284
Training with params:
{'colsample_bytree': 0.8, 'learning_rate': 0.1, 'max_depth': 13, 'n_estimators': 100, 'objective': 'reg:squarederror', 'see[...]<jupyter_text>Best Config XGBoost<jupyter_code>feats = ['param_napęd__cat', 'param_rok-produkcji', 'param_stan__cat', 'param_skrzynia-biegów__cat', 'param_faktura-vat__cat', 'param_moc', 'param_marka-pojazdu__cat', 'feature_kamera-cofania__cat', 'param_typ__cat', 'param_pojemność-skokowa', 'seller_name__cat', 'feature_wspomaganie-kierownicy__cat', 'param_model-pojazdu__cat', 'param_wersja__cat', 'param_kod-silnika__cat', 'feature_system-start-stop__cat', 'feature_asystent-pasa-ruchu__cat', 'feature_czujniki-parkowania-przednie__cat', 'feature_łopatki-zmiany-biegów__cat', 'feature_regulowane-zawieszenie__cat']
xgb_best_params = {
    'learning_rate':    0.1,
    'max_depth':        13,
    'subsample':        0.7,
    'colsample_bytree': 0.8,
    'objective':        'reg:squarederror',
    'n_estimators':     100,
    'seed':             0
}
run_model(xgb.XGBRegressor( **xgb_best_params), feats)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.15417956656346749, 'fraction_numerical': 0.058617131062951494, 'mean_word_length': 5.656593406593407, 'pattern_counts': {'":': 0, '<': 12, '<?xml version=': 0, '>': 12, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 7, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '668608', 'license': 'MIT', 'n_tokens_mistral': 2133, 'n_tokens_neox': 1895, 'n_words': 335, 'path': 'matrix_two/day5.ipynb', 'repo_name': 'jedrzejd/dw_matrix_car', 'size': '4845'}