<jupyter_start><jupyter_text>XGBoostXGBoost (Extreme Gradient Boosting) belongs to a family of boosting algorithms and uses the gradient boosting (GBM) framework. Boosting is a sequential technique which works on the principle of an ensemble. It combines a set of weak learners and delivers improved prediction accuracy.Sources:- https://xgboost.readthedocs.io/en/latest/tutorials/model.html- https://www.datacamp.com/community/tutorials/xgboost-in-python- https://www.datacamp.com/community/open-courses/basic-statistics- https://medium.com/@juniormiranda_23768/ensemble-methods-tuning-a-xgboost-model-with-scikit-learn-54ff669f988a- https://hackernoon.com/want-a-complete-guide-for-xgboost-model-in-python-using-scikit-learn-sc11f31bq- https://scikit-learn.org/stable/modules/generated/sklearn.ensemble.GradientBoostingClassifier.html- https://www.kaggle.com/stuarthallows/using-xgboost-with-scikit-learn- https://github.com/dmlc/xgboost/blob/master/demo/guide-python/sklearn_examples.py  Gradient Boosting<jupyter_code># https://scikit-learn.org/stable/auto_examples/ensemble/plot_gradient_boosting_regularization.html#sphx-glr-auto-examples-ensemble-plot-gradient-boosting-regularization-py
#
# Author: <NAME> <<EMAIL>>
#
# License: BSD 3 clause

import numpy as np
import matplotlib.pyplot as plt

from sklearn import ensemble
from sklearn import datasets


# Generate data for binary classification
X, y = datasets.make_hastie_10_2(n_samples=12000, random_state=1)
X = X.astype(np.float32)

# map labels from {-1, 1} to {0, 1}
labels, y = np.unique(y, return_inverse=True)

X_train, X_test = X[:2000], X[2000:]
y_train, y_test = y[:2000], y[2000:]

original_params = {'n_estimators': 1000, 'max_leaf_nodes': 4, 'max_depth': None, 'random_state': 2, 'min_samples_split': 5}

plt.figure()

for label, color, setting in [('No shrinkage', 'orange', {'learning_rate': 1.0, 'subsample': 1.0}),
                              ('learning_rate=0.1', 'turquoise', {'learning_rate': 0.1, 'subsample': 1.0}),
                              ('subsample=0.5', 'blue', {'learning_rate': 1.0, 'subsample': 0.5}),
                              ('learning_rate=0.1, subsample=0.5', 'gray', {'learning_rate': 0.1, 'subsample': 0.5}),
                              ('learning_rate=0.1, max_features=2', 'magenta', {'learning_rate': 0.1, 'max_features': 2})]:
    
    params = dict(original_params)
    params.update(setting)

    clf = ensemble.GradientBoostingClassifier(**params)
    clf.fit(X_train, y_train)

    # compute test set deviance
    test_deviance = np.zeros((params['n_estimators'],), dtype=np.float64)

    for i, y_pred in enumerate(clf.staged_decision_function(X_test)):
        # clf.loss_ assumes that y_test[i] in {0, 1}
        test_deviance[i] = clf.loss_(y_test, y_pred)

    plt.plot((np.arange(test_deviance.shape[0]) + 1)[::5], test_deviance[::5],
            '-', color=color, label=label)

plt.legend(loc='upper left')
plt.xlabel('Boosting Iterations')
plt.ylabel('Test Set Deviance')

plt.show()<jupyter_output><empty_output><jupyter_text>XGBoost demo<jupyter_code># Import the Boston Housing dataset
from sklearn.datasets import load_boston
boston = load_boston()
print(boston.keys())
boston.data.shape
boston.feature_names
# CRIM     per capita crime rate by town
# ZN       proportion of residential land zoned for lots over 25,000 sq.ft.
# INDUS    proportion of non-retail business acres per town
# CHAS     Charles River dummy variable (= 1 if tract bounds river; 0 otherwise)
# NOX      nitric oxides concentration (parts per 10 million)
# RM       average number of rooms per dwelling
# AGE      proportion of owner-occupied units built prior to 1940
# DIS      weighted distances to five Boston employment centres
# RAD      index of accessibility to radial highways
# TAX      full-value property-tax rate per $10,000
# PTRATIO  pupil-teacher ratio by town
# B        1000(Bk - 0.63)^2 where Bk is the proportion of blacks by town
# LSTAT    lower status of the population
# MEDV     Median value of owner-occupied homes in $1000's
import pandas as pd

data = pd.DataFrame(boston.data)
data.columns = boston.feature_names
data['PRICE'] = boston.target

data.head()
data.describe()
from sklearn.model_selection import train_test_split

X, y = data.iloc[:,:-1],data.iloc[:,-1]
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=123)
# pip3 install xgboost
import xgboost as xgb

data_dmatrix = xgb.DMatrix(data=X,label=y)
# parameters: https://xgboost.readthedocs.io/en/latest/parameter.html#general-parameters
#   learning_rate: step size shrinkage used to prevent overfitting. Range is [0,1]
#   max_depth: determines how deeply each tree is allowed to grow during any boosting round.
#   subsample: percentage of samples used per tree. Low value can lead to underfitting.
#   colsample_bytree: percentage of features used per tree. High value can lead to overfitting.
#   n_estimators: number of trees you want to build.
#   objective: determines the loss function to be used like reg:linear for regression problems, reg:logistic for classification problems with only decision, binary:logistic for classification problems with probability.
# regularization parameters to penalize models as they become more complex:
#   gamma: controls whether a given node will split based on the expected reduction in loss after the split. A higher value leads to fewer splits. Supported only for tree-based learners.
#   alpha: L1 regularization on leaf weights. A large value leads to more regularization.
#   lambda: L2 regularization on leaf weights and is smoother than L1 regularization.

xg_reg = xgb.XGBRegressor(objective ='reg:squarederror', colsample_bytree = 0.3, learning_rate = 0.1,
                max_depth = 5, alpha = 10, n_estimators = 10)
# learn
xg_reg.fit(X_train,y_train)

# make prediction
preds = xg_reg.predict(X_test)
from sklearn.metrics import mean_squared_error

rmse = np.sqrt(mean_squared_error(y_test, preds))
print("RMSE: %f" % (rmse))
# pip3 install graphviz

# xgb.plot_tree(xg_reg,num_trees=0)
# plt.rcParams['figure.figsize'] = [50, 10]
# plt.show()<jupyter_output><empty_output><jupyter_text>XGBoost Regression<jupyter_code>from sklearn.datasets import load_breast_cancer, load_diabetes, load_wine
from sklearn.metrics import auc, accuracy_score, confusion_matrix, mean_squared_error
from sklearn.model_selection import cross_val_score, GridSearchCV, KFold, RandomizedSearchCV, train_test_split
from scipy.stats import uniform, randint
X,y = load_diabetes(return_X_y=True)
print ('X shape:', X.shape, 'Y shape:', y.shape)
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=123)

# reg:squarederror: regression with squared loss
# reg:logistic: logistic regression
# binary:logistic: logistic regression for binary classification, output probability
# ...

xgb_model = xgb.XGBRegressor(objective="reg:squarederror", random_state=42)

xgb_model.fit(X_train, y_train)

y_pred = xgb_model.predict(X_test)

mse = mean_squared_error(y_test, y_pred)
print('Root mean square error:', np.sqrt(mse))<jupyter_output>X shape: (442, 10) Y shape: (442,)
Root mean square error: 58.379260646934036<jupyter_text>XGBoost Binary Classification<jupyter_code>X,y = load_breast_cancer(return_X_y=True)
print ('X shape:', X.shape, 'Y shape:', y.shape)
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=123)

xgb_model = xgb.XGBClassifier(objective="binary:logistic", random_state=42)
xgb_model.fit(X_train, y_train)

y_pred = xgb_model.predict(X_test)

print(confusion_matrix(y_test, y_pred))<jupyter_output>X shape: (569, 30) Y shape: (569,)
[[38  3]
 [ 1 72]]<jupyter_text>XGBoost Multiclass Classification<jupyter_code>X,y = load_wine(return_X_y=True)
print ('X shape:', X.shape, 'Y shape:', y.shape)
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=123)

xgb_model = xgb.XGBClassifier(objective="multi:softprob", random_state=42)
xgb_model.fit(X_train, y_train)

y_pred = xgb_model.predict(X_test)

print(confusion_matrix(y_test, y_pred))<jupyter_output>X shape: (178, 13) Y shape: (178,)
[[ 8  0  0]
 [ 0 11  0]
 [ 0  0 17]]<jupyter_text>XGBoost Cross Validation<jupyter_code>def display_scores(scores):
    print("Scores: {0}\nMean: {1:.3f}\nStd: {2:.3f}".format(scores, np.mean(scores), np.std(scores)))

X,y = load_diabetes(return_X_y=True)
print ('X shape:', X.shape, 'Y shape:', y.shape)
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=123)
kfold = KFold(n_splits=5, shuffle=True, random_state=42)

scores = []

for train_index, test_index in kfold.split(X):   
    X_train, X_test = X[train_index], X[test_index]
    y_train, y_test = y[train_index], y[test_index]

    xgb_model = xgb.XGBRegressor(objective="reg:squarederror")
    xgb_model.fit(X_train, y_train)
    
    y_pred = xgb_model.predict(X_test)
    
    scores.append(mean_squared_error(y_test, y_pred))
    
display_scores(np.sqrt(scores))<jupyter_output>X shape: (442, 10) Y shape: (442,)
Scores: [63.94113147 61.42459229 67.48347369 69.49735072 59.90352078]
Mean: 64.450
Std: 3.599<jupyter_text>XGBoost Hyperparameter Searching<jupyter_code>def report_best_scores(results, n_top=3):
    for i in range(1, n_top + 1):
        candidates = np.flatnonzero(results['rank_test_score'] == i)
        for candidate in candidates:
            print("Model with rank: {0}".format(i))
            print("Mean validation score: {0:.3f} (std: {1:.3f})".format(
                  results['mean_test_score'][candidate],
                  results['std_test_score'][candidate]))
            print("Parameters: {0}\n".format(results['params'][candidate]))

X,y = load_diabetes(return_X_y=True)

xgb_model = xgb.XGBRegressor()

params = {
    "colsample_bytree": uniform(0.7, 0.3),
    "gamma": uniform(0, 0.5),
    "learning_rate": uniform(0.03, 0.3), # default 0.1 
    "max_depth": randint(2, 6), # default 3
    "n_estimators": randint(100, 150), # default 100
    "subsample": uniform(0.6, 0.4)
}

searchCV = RandomizedSearchCV(xgb_model, param_distributions=params, random_state=42, n_iter=200, cv=3, verbose=1, n_jobs=1, return_train_score=True)
searchCV.fit(X, y)
report_best_scores(searchCV.cv_results_, 1)<jupyter_output>Fitting 3 folds for each of 200 candidates, totalling 600 fits<jupyter_text>XGBoost Early Stopping<jupyter_code>X,y = load_breast_cancer(return_X_y=True)
print ('X shape:', X.shape, 'Y shape:', y.shape)
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42)

# if more than one evaluation metric are given the last one is used for early stopping
xgb_model = xgb.XGBClassifier(objective="binary:logistic", random_state=42, eval_metric="auc")
# learn
xgb_model.fit(X_train, y_train, early_stopping_rounds=5, eval_set=[(X_test, y_test)])
# predict
y_pred = xgb_model.predict(X_test)

accuracy_score(y_test, y_pred)
print("best score: {0}, best iteration: {1}, best ntree limit {2}".format(xgb_model.best_score, xgb_model.best_iteration, xgb_model.best_ntree_limit))<jupyter_output>best score: 0.991484, best iteration: 10, best ntree limit 11<jupyter_text>XGBoost Evaluations<jupyter_code>X,y = load_breast_cancer(return_X_y=True)
print ('X shape:', X.shape, 'Y shape:', y.shape)
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42)

xgb_model = xgb.XGBClassifier(objective="binary:logistic", n_estimators=20, random_state=42, eval_metric=["auc", "error", "error@0.6"])
# learn
xgb_model.fit(X_train, y_train, eval_set=[(X_test, y_test)])
# predict
y_pred = xgb_model.predict(X_test)<jupyter_output>X shape: (569, 30) Y shape: (569,)
[0]	validation_0-auc:0.95644	validation_0-error:0.05263	validation_0-error@0.6:0.06140
[1]	validation_0-auc:0.97101	validation_0-error:0.05263	validation_0-error@0.6:0.05263
[2]	validation_0-auc:0.96479	validation_0-error:0.04386	validation_0-error@0.6:0.06140
[3]	validation_0-auc:0.96479	validation_0-error:0.04386	validation_0-error@0.6:0.06140
[4]	validation_0-auc:0.96544	validation_0-error:0.04386	validation_0-error@0.6:0.05263
[5]	validation_0-auc:0.96479	validation_0-error:0.04386	validation_0-error@0.6:0.04386
[6]	validation_0-auc:0.97330	validation_0-error:0.04386	validation_0-error@0.6:0.04386
[7]	validation_0-auc:0.98886	validation_0-error:0.04386	validation_0-error@0.6:0.04386
[8]	validation_0-auc:0.98985	validation_0-error:0.04386	validation_0-error@0.6:0.05263
[9]	validation_0-auc:0.98985	validation_0-error:0.04386	validation_0-error@0.6:0.04386
[10]	validation_0-auc:0.99148	validation_0-error:0.04386	validation_0-error@0.6:0.04386
[11]	va[...]
{'fraction_non_alphanumeric': 0.1262427015938141, 'fraction_numerical': 0.05302193466940192, 'mean_word_length': 5.225442043222004, 'pattern_counts': {'":': 6, '<': 33, '<?xml version=': 0, '>': 33, 'https://': 10, 'lorem ipsum': 0, 'www.': 3, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '9', 'dir': 'jupyter-structured-clean-dedup', 'id': '610139', 'license': 'MIT', 'n_tokens_mistral': 5114, 'n_tokens_neox': 4642, 'n_words': 1024, 'path': 'ml_xgboost_classification.ipynb', 'repo_name': 'lspevak/ml-samples', 'size': '12707'}