<jupyter_start><jupyter_text>XGBoost Hyperparameter Tuning<jupyter_code># Global variables for testing changes to this notebook quickly
RANDOM_SEED = 0
NUM_FOLDS = 6
NUM_TREES = 25000
EARLY_STOP = 200
import numpy as np
import pandas as pd
import time
import os
import pyarrow
import gc

# Hide warnings
import warnings
warnings.filterwarnings('ignore')
os.environ['TF_CPP_MIN_LOG_LEVEL'] = '3' 

# display options
pd.set_option('display.max_rows', None)
pd.set_option('display.max_columns', None)

# Model/Evaluation
from IPython.display import Image
from functools import partial
from sklearn.base import clone
from sklearn.pipeline import make_pipeline
from sklearn.preprocessing import LabelEncoder, RobustScaler
from sklearn.model_selection import StratifiedKFold, train_test_split
from sklearn.metrics import accuracy_score, recall_score
from sklearn.metrics import confusion_matrix, ConfusionMatrixDisplay
from xgboost import XGBClassifier

# Plotting
import matplotlib
import seaborn as sns
from matplotlib import pyplot as plt

# Optuna
import optuna
from optuna.pruners import PercentilePruner<jupyter_output><empty_output><jupyter_text>Load and Prepare Data<jupyter_code>%%time

train = pd.read_feather('../data/xgboost_train.feather')
test = pd.read_feather('../data/xgboost_test.feather')
submission = pd.read_csv('../data/sample_submission.csv')

# Drop columns, cover type 5
train.drop(["Soil_Type7", "Id", "Soil_Type15"], axis=1, inplace=True)
test.drop(["Soil_Type7", "Id", "Soil_Type15"], axis=1, inplace=True)
train = train[train.Cover_Type != 5]

# Label Encoding
encoder = LabelEncoder()
train["Cover_Type"] = encoder.fit_transform(train["Cover_Type"])<jupyter_output>Wall time: 1.59 s<jupyter_text>Model and Scoring Function<jupyter_code># Default XGBoost
default_params = dict(
    tree_method = 'hist',
    n_estimators = NUM_TREES,
    random_state = RANDOM_SEED,
)
def score_xgboost(model_params = {}, fit_params = {}):
    
    # Original Training/Test Split
    features = [x for x in train.columns if x not in ['Id','Cover_Type']]
    X_temp, X_test = train[features], test[features]
    y_temp = train['Cover_Type']
    
    # Store the out-of-fold predictions
    test_preds = np.zeros((X_test.shape[0],6))
    oof_preds = np.zeros((X_temp.shape[0],))
    scores = np.zeros(NUM_FOLDS)
    
    # Stratified k-fold cross-validation
    skf = StratifiedKFold(n_splits = NUM_FOLDS, shuffle = True, random_state = RANDOM_SEED)
    for fold, (train_idx, valid_idx) in enumerate(skf.split(train[features],train['Cover_Type'])):
       
        # Training and Validation Sets
        start = time.time()
        X_train, X_valid = X_temp.iloc[train_idx], X_temp.iloc[valid_idx]
        y_train, y_valid = y_temp.iloc[train_idx], y_temp.iloc[valid_idx]
        
        # Create model
        start = time.time()
        model = XGBClassifier(**default_params, **model_params)
        
        model.fit(
            X_train, y_train,
            eval_set = [(X_valid, y_valid)],
            eval_metric = "mlogloss",
            early_stopping_rounds = EARLY_STOP,
            verbose = 500,
            **fit_params
        )

        # validation/holdout predictions
        valid_preds = np.ravel(model.predict(X_valid))
        oof_preds[valid_idx] = valid_preds
        test_preds += model.predict_proba(X_test)

        # Save scores and times
        scores[fold] = accuracy_score(y_valid, valid_preds)
        end = time.time()
        
    
    test_preds = np.argmax(test_preds, axis = 1)
    oof_accuracy = accuracy_score(y_temp, oof_preds)
    print(f'OOF Accuracy: {round(100*oof_accuracy, 5)}% accuracy.')
    time.sleep(0.5)
    
    return oof_preds, test_preds
oof_preds, test_preds = score_xgboost({
        'max_depth': 5, 
        'learning_rate': 0.034134457831963505, 
        'gamma': 0.009393716737203944, 
        'min_child_weight': 0.05489584210577205, 
        'subsample': 0.9199999999999999, 
        'colsample_bytree': 0.5800000000000001, 
        'colsample_bylevel': 0.56, 
        'reg_lambda': 0.020044735970500934, 
        'reg_alpha': 2.7703392595849
    })
fig, ax = plt.subplots(figsize = (8,6))
cm = confusion_matrix(train['Cover_Type'], oof_preds)
disp = ConfusionMatrixDisplay(confusion_matrix=cm)
disp.plot(ax = ax)
plt.title("Confusion Matrix (% of Actual Labels)", fontsize=16)
plt.xlabel("Predicted Labels", fontsize=14)
plt.ylabel("Actual Labels", fontsize=14)
plt.show()
submission['Cover_Type'] = encoder.inverse_transform(test_preds)
submission.to_csv('../output/xgboost_submission.csv')<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.10973220117570215, 'fraction_numerical': 0.037448290877422165, 'mean_word_length': 4.031763417305586, 'pattern_counts': {'":': 0, '<': 12, '<?xml version=': 0, '>': 12, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 2, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '477068', 'license': 'MIT', 'n_tokens_mistral': 1742, 'n_tokens_neox': 1584, 'n_words': 323, 'path': 'tps-2021-12/notebooks/Notebook 5b - XGBoost.ipynb', 'repo_name': 'rsizem2/tabular-playground-series', 'size': '4593'}