<jupyter_start><jupyter_text>Table of Contents1&nbsp;&nbsp;Load Data2&nbsp;&nbsp;Cross Validation3&nbsp;&nbsp;Viz4&nbsp;&nbsp;Feature Importance5&nbsp;&nbsp;Train Test Split6&nbsp;&nbsp;Model6.1&nbsp;&nbsp;OOB: Out of Bag6.2&nbsp;&nbsp;Results7&nbsp;&nbsp;Plotting the Tree8&nbsp;&nbsp;Variable Importance  XGBoostCredit:<jupyter_code>import warnings
warnings.filterwarnings("always")

import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import seaborn as sns
from sklearn import datasets
import xgboost as xgb

# python version 
import sys
print('python vers:', sys.version[:31])<jupyter_output>python vers: 3.6.5 |Anaconda custom (64-bit)<jupyter_text>Load Data<jupyter_code>from sklearn.datasets import load_boston
boston = load_boston()

data = pd.DataFrame(boston.data)
data.columns = boston.feature_names

data['PRICE'] = boston.target

data.head()

print(boston.keys())
print(boston.data.shape)
print(boston.feature_names)
# Separate the target variable and rest of the variables
X, y = data.iloc[:,:-1],data.iloc[:,-1]
# convert the dataset into an optimized data structure called Dmatrix that XGBoost supports
data_dmatrix = xgb.DMatrix(data=X,label=y)
data_dmatrix
from sklearn.model_selection import train_test_split

X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=123)

xg_reg = xgb.XGBRegressor(objective ='reg:linear', colsample_bytree = 0.3, learning_rate = 0.1,
                max_depth = 5, alpha = 10, n_estimators = 10)

xg_reg.fit(X_train,y_train)

preds = xg_reg.predict(X_test)
from sklearn.metrics import mean_squared_error

rmse = np.sqrt(mean_squared_error(y_test, preds))
print("RMSE: %f" % (rmse))<jupyter_output>RMSE: 10.868649<jupyter_text>Cross Validation<jupyter_code>params = {"objective":"reg:linear",'colsample_bytree': 0.3,'learning_rate': 0.1,
                'max_depth': 5, 'alpha': 10}

cv_results = xgb.cv(dtrain=data_dmatrix, params=params, nfold=3,
                    num_boost_round=50,early_stopping_rounds=10,metrics="rmse", as_pandas=True, seed=123)

cv_results.head()
print((cv_results["test-rmse-mean"]).tail(1))<jupyter_output>49    4.031162
Name: test-rmse-mean, dtype: float64<jupyter_text>Viz<jupyter_code>xg_reg = xgb.train(params=params, dtrain=data_dmatrix, num_boost_round=10)
import matplotlib.pyplot as plt

xgb.plot_tree(xg_reg,num_trees=0)
plt.rcParams['figure.figsize'] = [500, 100]
plt.show()<jupyter_output><empty_output><jupyter_text>Feature Importance<jupyter_code>xgb.plot_importance(xg_reg)
plt.rcParams['figure.figsize'] = [5, 5]
plt.show()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.12933025404157045, 'fraction_numerical': 0.03117782909930716, 'mean_word_length': 7.021604938271605, 'pattern_counts': {'":': 1, '<': 18, '<?xml version=': 0, '>': 18, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '5', 'dir': 'jupyter-structured-clean-dedup', 'id': '486135', 'license': 'MIT', 'n_tokens_mistral': 1050, 'n_tokens_neox': 988, 'n_words': 175, 'path': '*Machine_Learning/Trees/XGBoost.ipynb', 'repo_name': 'jasjung/Machine_Learnin', 'size': '2598'}