<jupyter_start><jupyter_text>XGBoost model. Tuning and creating submission<jupyter_code>%store -r X_train
%store -r X_test

%store -r y_train
%store -r y_test

%store -r X_train_sub
%store -r X_test_sub
%store -r y_train_sub
%store -r y_test_sub<jupyter_output><empty_output><jupyter_text>Import libraries<jupyter_code>send_submission = False
__da
import sys
import os
sys.pa### Import libraries

send_submission = False

_dath.append(os.path.realpath('./../../scripts'))
import numpy as np
import pandas as pd 

import xgboost as xgb
from xgboost import plot_importance

import warnings
import matplotlib.pyplot as plt
%matplotlib inline 


from IPython.core.debugger import set_trace
from time import time

# SKLEARN

from sklearn.model_selection import RandomizedSearchCV, TimeSeriesSplit, KFold
from sklearn.metrics import r2_score

import xgboost as xgb

# Bayessian Optimization
from bayes_opt import BayesianOptimization
from hyperopt import hp
from hyperopt import fmin, tpe, space_eval
from hyperopt import STATUS_OK, Trials, fmin, hp, tpe

# vecstack
from vecstack import stacking
def get_xgb(params = {}):
    return xgb.XGBRegressor(
           boosting_type= 'gbdt',
           objective = 'reg:squarederror',
           metric = "rmse",
           random_state= 1,
           silent = True,
           **params)

def create_submission(model, submission_name, send_submission=False, message="XGBoost"):
    result = model.predict(X_test_sub)
    submission = pd.DataFrame({
        "ID": np.arange(result.shape[0]), 
        "item_cnt_month": result
    })
    submission.to_csv(submission_name, index=False)
   
    if send_submission:
        !kaggle competitions submit -c competitive-data-science-predict-future-sales -f {submission_name} -m {message}
%%notify -m "Grid search XGBoost was ended"

gridParams = {
    'learning_rate': np.linspace(0.01, 2, 10),
    'max_depth': np.arange(0, 14),
    'n_estimators': np.arange(100, 1000, 100),
    'num_leaves': np.arange(8, 65, 8),
    'colsample_bytree' : [0.65, 0.66],
    'subsample' : np.arange(0.5, 1.1, 0.1),
    'reg_alpha' : np.arange(0, 100),
    'reg_lambda' : np.arange(0, 100),
    'min_weight_sample': np.arange(1, 300, 10)
}

xgb_model = get_xgb()

# Create the grid
grid = RandomizedSearchCV(xgb_model, gridParams, verbose=0, cv=TimeSeriesSplit(n_splits=5), n_iter=50)

grid.fit(X_train, y_train, 
    eval_metric="rmse", 
    eval_set=[(X_train, y_train), (X_test, y_test)], 
    verbose=10, 
    early_stopping_rounds=20)
print("Score", grid.best_score_)
print('Fitting with params: ')
print(grid.best_params_)
rmse =  np.sqrt(mean_squared_error(y_test, grid.best_estimator.predict(X_test))) 
print(f"RMSE {rmse}")<jupyter_output><empty_output><jupyter_text>Tuned parameters from RandomizedSearch.<jupyter_code>params = {'subsample': 1,
 'reg_lambda': 11,
 'reg_alpha': 32,
 'num_leaves': 40,
 'n_estimators': 10,
 'max_depth': 7,
 'learning_rate': 0.45,
 'colsample_bytree': 0.66}
%%notify -m "XGBoost submission have been fit"

xgb_model = get_xgb(params)
xgb_model.fit(X_train_sub, y_train_sub, 
    eval_metric="rmse", 
    eval_set=[(X_train_sub, y_train_sub)], 
    verbose=10, 
    early_stopping_rounds=20)    
%%notify -m "Submission was created"
create_submission(xgb_model, 'xgb_submission_gs.csv', True)
fig, ax = plt.subplots(figsize=(12,10))
xgb.plot_importance(xgb_model, max_num_features=20, height=0.8, ax=ax)
plt.show()
progress
# make predictions for test data
progress = xgb_model.evals_result()
y_pred = xgb_model.predict(X_test)
predictions = np.round(y_pred)
# evaluate predictions
accuracy = np.sqrt( 1/(len(y_test)) * (y_test - predictions)**2)
# retrieve performance metrics
epochs = len(progress['validation_0']['rmse'])
x_axis = range(0, epochs)
# plot log loss
fig, ax = plt.subplots()
ax.plot(x_axis, progress['validation_0']['rmse'], label='Train')
#ax.plot(x_axis, progress['train']['rmse'], label='Test')
ax.legend()
plt.ylabel('RMSE')
plt.title('XGBoost Log Loss')
plt.show()
%store xgb_model<jupyter_output>Stored 'xgb_model' (XGBRegressor)
{'fraction_non_alphanumeric': 0.12915947744638895, 'fraction_numerical': 0.022923342371210253, 'mean_word_length': 4.924087591240876, 'pattern_counts': {'":': 2, '<': 12, '<?xml version=': 0, '>': 12, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '277872', 'license': 'MIT', 'n_tokens_mistral': 1593, 'n_tokens_neox': 1503, 'n_words': 330, 'path': 'notebooks/model_train/sales-prediction.model_train.xgboost.1.0.ipynb', 'repo_name': 'Saphirox/sales-prediction', 'size': '4057'}