<jupyter_start><jupyter_text>Optimizing Individual Models  Ridge Regression<jupyter_code>#base model
rdg = linear_model.RidgeCV()
base_results = model_selection.cross_validate(rdg, df[x], df[y], cv=cv_split, n_jobs=-1)
rdg.fit(df[x], df[y])

print('BEFORE Parameters: ', rdg.get_params())
print("BEFORE Training  score mean: {:.2f}". format(base_results['train_score'].mean()*100)) 
print("BEFORE Test score mean: {:.2f}". format(base_results['test_score'].mean()*100))
print("BEFORE Test score 3*std: +/- {:.2f}". format(base_results['test_score'].std()*100*3))
print('-'*10)

param_grid = {'alphas': ((165.0,), (170.0,), (175.0,), (180.0,), (190.0,), (200.0,))
             }
tune_model = model_selection.GridSearchCV(linear_model.RidgeCV(), param_grid=param_grid, cv=cv_split, n_jobs=-1)
tune_model.fit(df[x], df[y])


print('AFTER Parameters: ', tune_model.best_params_)
print("AFTER Training score mean: {:.2f}". format(tune_model.cv_results_['mean_train_score'][tune_model.best_index_]*100)) 
print("AFTER Test score mean: {:.2f}". format(tune_model.cv_results_['mean_test_score'][tune_model.best_index_]*100))
print("AFTER Test score 3*std: +/- {:.2f}". format(tune_model.cv_results_['std_test_score'][tune_model.best_index_]*100*3))
print('-'*10)
### Feature Tuning
rdg = linear_model.RidgeCV(alphas=(175.0,))
base_results = model_selection.cross_validate(rdg, df[x], df[y], cv=cv_split, n_jobs=-1)
rdg.fit(df[x], df[y])


print('BEFORE RFE Training Shape Old: ', df[x].shape) 
print('BEFORE RFE Training Columns Old: ', df[x].columns.values)

print("BEFORE RFE Training score mean: {:.2f}". format(base_results['train_score'].mean()*100)) 
print("BEFORE RFE Test score mean: {:.2f}". format(base_results['test_score'].mean()*100))
print("BEFORE RFE Test score 3*std: +/- {:.2f}". format(base_results['test_score'].std()*100*3))
print('-'*10)



#feature selection
rdg_rfe = feature_selection.RFECV(rdg, step = 1, cv = cv_split, n_jobs=-1)
rdg_rfe.fit(df[x], df[y])

#transform x&y to reduced features and fit new model
X_rfe = df[x].columns.values[rdg_rfe.get_support()]
rfe_results = model_selection.cross_validate(rdg, df[X_rfe], df[y], cv  = cv_split, n_jobs=-1)

print('AFTER RFE Training Shape New: ', df[X_rfe].shape) 
print('AFTER RFE Training Columns New: ', X_rfe)

print("AFTER RFE Training score mean: {:.2f}". format(rfe_results['train_score'].mean()*100)) 
print("AFTER RFE Test score mean: {:.2f}". format(rfe_results['test_score'].mean()*100))
print("AFTER RFE Test score 3*std: +/- {:.2f}". format(rfe_results['test_score'].std()*100*3))
print('-'*10)
rdg_x = ['age_BLK%', 'age_DRB%', 'age_DRtg', 'age_PER', 'age_PPR', 'age_STL%', 'age_TRB%',
 'age_BPM', 'age_MIN:GP', 'age_USGxTS', 'age_FTRate', 'age_STLK%',
 'age_DRB%_ppctl', 'age_TRB%_ppctl', 'age_rFG%', 'Height', 'Weight_ppctl', 'Age',
 'age_pDBPM_ppctl', 'age_pDBPM']

#base model
rdg = linear_model.RidgeCV(alphas=(175.0,))
base_results = model_selection.cross_validate(rdg, df[rdg_x], df[y], cv=cv_split, n_jobs=-1)
rdg.fit(df[rdg_x], df[y])

print('BEFORE Parameters: ', rdg.get_params())
print("BEFORE Training  score mean: {:.2f}". format(base_results['train_score'].mean()*100)) 
print("BEFORE Test score mean: {:.2f}". format(base_results['test_score'].mean()*100))
print("BEFORE Test score 3*std: +/- {:.2f}". format(base_results['test_score'].std()*100*3))
print('-'*10)

param_grid = {'alphas': ((110.0,), (115.0,), (90.0,), (95.0,), (100.0,), (105.0,))
             }
tune_model = model_selection.GridSearchCV(linear_model.RidgeCV(), param_grid=param_grid, cv=cv_split, n_jobs=-1)
tune_model.fit(df[rdg_x], df[y])


print('AFTER Parameters: ', tune_model.best_params_)
print("AFTER Training score mean: {:.2f}". format(tune_model.cv_results_['mean_train_score'][tune_model.best_index_]*100)) 
print("AFTER Test score mean: {:.2f}". format(tune_model.cv_results_['mean_test_score'][tune_model.best_index_]*100))
print("AFTER Test score 3*std: +/- {:.2f}". format(tune_model.cv_results_['std_test_score'][tune_model.best_index_]*100*3))
print('-'*10)
# final ridge
rdg_x = ['age_BLK%', 'age_DRB%', 'age_DRtg', 'age_PER', 'age_PPR', 'age_STL%', 'age_TRB%',
 'age_BPM', 'age_MIN:GP', 'age_USGxTS', 'age_FTRate', 'age_STLK%',
 'age_DRB%_ppctl', 'age_TRB%_ppctl', 'age_rFG%', 'Height', 'Weight_ppctl', 'Age',
 'age_pDBPM_ppctl', 'age_pDBPM']

#base model
rdg = linear_model.RidgeCV(alphas=(110.0,))
base_results = model_selection.cross_validate(rdg, df[rdg_x], df[y], cv=cv_split, n_jobs=-1)
rdg.fit(df[rdg_x], df[y])

print('Parameters: ', tune_model.best_params_)
print("Training score mean: {:.2f}". format(tune_model.cv_results_['mean_train_score'][tune_model.best_index_]*100)) 
print("Test score mean: {:.2f}". format(tune_model.cv_results_['mean_test_score'][tune_model.best_index_]*100))
print("Test score 3*std: +/- {:.2f}". format(tune_model.cv_results_['std_test_score'][tune_model.best_index_]*100*3))
print('-'*10)<jupyter_output><empty_output><jupyter_text>Gradient Boosting<jupyter_code># Model selection process with cross validation.
cv_split = model_selection.ShuffleSplit(n_splits = 10, test_size = .3, train_size = .6, random_state = 0 ) 
# run model 10x with 60/30 split intentionally leaving out 10%

#base model
grb = ensemble.GradientBoostingRegressor(random_state=2)
base_results = model_selection.cross_validate(grb, df[x], df[y], cv=cv_split, n_jobs=-1)
grb.fit(df[x], df[y])


print('BEFORE Parameters: ', grb.get_params())
print("BEFORE Training score mean: {:.2f}". format(base_results['train_score'].mean()*100)) 
print("BEFORE Test score mean: {:.2f}". format(base_results['test_score'].mean()*100))
print("BEFORE Test score 3*std: +/- {:.2f}". format(base_results['test_score'].std()*100*3))
print("BEFORE Test set score min: {:.2f}". format(base_results['test_score'].min()*100))
print('-'*10)


#tune hyper-parameters: 
param_grid = {
                'max_depth': [3, 4, 5],
                'n_estimators': [35, 40, 45],
                'min_samples_split': [2],
                'min_samples_leaf': [9, 10, 11],
                'subsample': [.7, .75, .8],
                'max_features': [None, .1, .2, .3]
             }


# choose best model with grid_search: 
tune_model = model_selection.GridSearchCV(grb, param_grid=param_grid, cv=cv_split, n_jobs=-1)
tune_model.fit(df[x], df[y])

print('AFTER Parameters: ', tune_model.best_params_)
print("AFTER Training score mean: {:.2f}". format(tune_model.cv_results_['mean_train_score'][tune_model.best_index_]*100)) 
print("AFTER Test score mean: {:.2f}". format(tune_model.cv_results_['mean_test_score'][tune_model.best_index_]*100))
print("AFTER Test score 3*std: +/- {:.2f}". format(tune_model.cv_results_['std_test_score'][tune_model.best_index_]*100*3))
print('-'*10)
### Feature Tuning

#base model
grb = ensemble.GradientBoostingRegressor(max_depth=3, min_samples_leaf=10, n_estimators=40, max_features=.1, subsample=.75)
base_results = model_selection.cross_validate(grb, df[x], df[y], cv=cv_split, n_jobs=-1)
grb.fit(df[x], df[y])

print('BEFORE RFE Training Shape Old: ', df[x].shape) 
print('BEFORE RFE Training Columns Old: ', df[x].columns.values)

print("BEFORE RFE Training score mean: {:.2f}". format(base_results['train_score'].mean()*100)) 
print("BEFORE RFE Test score mean: {:.2f}". format(base_results['test_score'].mean()*100))
print("BEFORE RFE Test score 3*std: +/- {:.2f}". format(base_results['test_score'].std()*100*3))
print('-'*10)



#feature selection
grb_rfe = feature_selection.RFECV(grb, step = 1, cv = cv_split, n_jobs=-1)
grb_rfe.fit(df[x], df[y])

#transform x&y to reduced features and fit new model
X_rfe = df[x].columns.values[grb_rfe.get_support()]
rfe_results = model_selection.cross_validate(grb, df[X_rfe], df[y], cv  = cv_split, n_jobs=-1)

print('AFTER RFE Training Shape New: ', df[X_rfe].shape) 
print('AFTER RFE Training Columns New: ', X_rfe)

print("AFTER RFE Training score mean: {:.2f}". format(rfe_results['train_score'].mean()*100)) 
print("AFTER RFE Test score mean: {:.2f}". format(rfe_results['test_score'].mean()*100))
print("AFTER RFE Test score 3*std: +/- {:.2f}". format(rfe_results['test_score'].std()*100*3))
print('-'*10)
grb_x = ['age_AST%', 'age_DRB%', 'age_DRtg', 'age_ORtg', 'age_PER', 'age_PPR', 'age_PPS',
 'age_STL%', 'age_TOV%', 'age_TRB%', 'age_TS%', 'age_Total S %', 'age_eFG%',
 'age_BPM', 'age_MIN:GP', 'age_USGxTS', 'age_FTRate', 'age_PF:STLK',
 'age_AST%_ppctl', 'age_BLK%_ppctl', 'age_DRB%_ppctl', 'age_DRtg_ppctl',
 'age_ORtg_ppctl', 'age_PPR_ppctl', 'age_PPS_ppctl', 'age_STL%_ppctl',
 'age_TS%_ppctl', 'age_Total S %_ppctl', 'age_USG%_ppctl', 'age_eFG%_ppctl',
 'age_BPM_ppctl', 'age_USGxTS_ppctl', 'age_STLK%_ppctl', 'age_PF:STLK_ppctl',
 'age_rFG%', 'age_rFT%', 'Height_ppctl', 'Weight_ppctl', 'Age',
 'age_pDBPM_ppctl', 'age_PER_ppctl']

# final grb
grb = ensemble.GradientBoostingRegressor(max_depth=3, min_samples_leaf=10, n_estimators=40, max_features=.1, subsample=.75)
base_results = model_selection.cross_validate(grb, df[grb_x], df[y], cv=cv_split, n_jobs=-1)
grb.fit(df[grb_x], df[y])


print('Parameters: ', grb.get_params())
print("Training score mean: {:.2f}". format(base_results['train_score'].mean()*100)) 
print("Test score mean: {:.2f}". format(base_results['test_score'].mean()*100))
print("Test score 3*std: +/- {:.2f}". format(base_results['test_score'].std()*100*3))
print("Test set score min: {:.2f}". format(base_results['test_score'].min()*100))
print('-'*10)<jupyter_output><empty_output><jupyter_text>XG Boost<jupyter_code># Model selection process with cross validation.
cv_split = model_selection.ShuffleSplit(n_splits = 10, test_size = .3, train_size = .6, random_state = 0 ) 
# run model 10x with 60/30 split intentionally leaving out 10%

#base model
xgb = XGBRegressor()
base_results = model_selection.cross_validate(xgb, df[x], df[y], cv=cv_split, n_jobs=-1)
xgb.fit(df[x], df[y])


print('BEFORE Parameters: ', xgb.get_params())
print("BEFORE Training score mean: {:.2f}". format(base_results['train_score'].mean()*100)) 
print("BEFORE Test score mean: {:.2f}". format(base_results['test_score'].mean()*100))
print("BEFORE Test score 3*std: +/- {:.2f}". format(base_results['test_score'].std()*100*3))
print("BEFORE Test set score min: {:.2f}". format(base_results['test_score'].min()*100))
print('-'*10)


#tune hyper-parameters: 
param_grid = {
                'max_depth': [3, 4, 5],
                'n_estimators': [25, 30, 35],
                'subsample': [.8, .85, .9, .95, 1.0],
                'colsample_bylevel': [.2, .25, .3],
                'colsample_bytree': [.2, .25, .3]
             }

# choose best model with grid_search: 
tune_model = model_selection.GridSearchCV(XGBRegressor(), param_grid=param_grid, cv=cv_split, n_jobs=-1)
tune_model.fit(df[x], df[y])


print('AFTER Parameters: ', tune_model.best_params_)
print("AFTER Training score mean: {:.2f}". format(tune_model.cv_results_['mean_train_score'][tune_model.best_index_]*100)) 
print("AFTER Test score mean: {:.2f}". format(tune_model.cv_results_['mean_test_score'][tune_model.best_index_]*100))
print("AFTER Test score 3*std: +/- {:.2f}". format(tune_model.cv_results_['std_test_score'][tune_model.best_index_]*100*3))
print('-'*10)
#base model
xgb = XGBRegressor(colsample_bylevel=0.25, colsample_bytree=0.25, max_depth=4, n_estimators=30)
base_results = model_selection.cross_validate(xgb, df[x], df[y], cv=cv_split, n_jobs=-1)
xgb.fit(df[x], df[y])


print('Parameters: ', xgb.get_params())
print("Training score mean: {:.2f}". format(base_results['train_score'].mean()*100)) 
print("Test score mean: {:.2f}". format(base_results['test_score'].mean()*100))
print("Test score 3*std: +/- {:.2f}". format(base_results['test_score'].std()*100*3))
print("Test set score min: {:.2f}". format(base_results['test_score'].min()*100))
print('-'*10)<jupyter_output><empty_output><jupyter_text>Bagging Regressor<jupyter_code>#base model
bag = ensemble.BaggingRegressor()
base_results = model_selection.cross_validate(bag, df[x], df[y], cv=cv_split, n_jobs=-1)
bag.fit(df[x], df[y])


print('BEFORE Parameters: ', bag.get_params())
print("BEFORE Training  score mean: {:.2f}". format(base_results['train_score'].mean()*100)) 
print("BEFORE Test score mean: {:.2f}". format(base_results['test_score'].mean()*100))
print("BEFORE Test score 3*std: +/- {:.2f}". format(base_results['test_score'].std()*100*3))
print("BEFORE Test set score min: {:.2f}". format(base_results['test_score'].min()*100))
print('-'*10)


#tune hyper-parameters: 
param_grid = {
                'base_estimator': [ensemble.GradientBoostingRegressor(n_estimators=125, max_depth=3, min_samples_split=7, min_samples_leaf=1)],
                'n_estimators': [35],
                'max_samples': [.4],
                'bootstrap_features': [False],
                'random_state': [2]
             }


# choose best model with grid_search: 
tune_model = model_selection.GridSearchCV(ensemble.BaggingRegressor(), param_grid=param_grid, cv=cv_split, n_jobs=-1)
tune_model.fit(df[x], df[y])

print('AFTER DT Parameters: ', tune_model.best_params_)
print("AFTER DT Training score mean: {:.2f}". format(tune_model.cv_results_['mean_train_score'][tune_model.best_index_]*100)) 
print("AFTER DT Test score mean: {:.2f}". format(tune_model.cv_results_['mean_test_score'][tune_model.best_index_]*100))
print("AFTER DT Test score 3*std: +/- {:.2f}". format(tune_model.cv_results_['std_test_score'][tune_model.best_index_]*100*3))
print('-'*10)
#final model
bag = ensemble.BaggingRegressor(ensemble.GradientBoostingRegressor(n_estimators=125, max_depth=3, min_samples_split=7),
                                n_estimators=35,
                                max_samples=.4,
                                bootstrap_features=False)
base_results = model_selection.cross_validate(bag, df[x], df[y], cv=cv_split, n_jobs=-1)
bag.fit(df[x], df[y])

print("Training score mean: {:.2f}". format(base_results['train_score'].mean()*100)) 
print("Test score mean: {:.2f}". format(base_results['test_score'].mean()*100))
print("Test score 3*std: +/- {:.2f}". format(base_results['test_score'].std()*100*3))
print('-'*10)<jupyter_output><empty_output><jupyter_text>Adaboost (base: decision tree regressor)<jupyter_code>#base model
ada = ensemble.AdaBoostRegressor()
base_results = model_selection.cross_validate(ada, df[x], df[y], cv=cv_split, n_jobs=-1)
ada.fit(df[x], df[y])


print('BEFORE Parameters: ', ada.get_params())
print("BEFORE Training  score mean: {:.2f}". format(base_results['train_score'].mean()*100)) 
print("BEFORE Test score mean: {:.2f}". format(base_results['test_score'].mean()*100))
print("BEFORE Test score 3*std: +/- {:.2f}". format(base_results['test_score'].std()*100*3))
print("BEFORE Test set score min: {:.2f}". format(base_results['test_score'].min()*100))
print('-'*10)


#tune hyper-parameters: 
param_grid = {
                'base_estimator': [tree.DecisionTreeRegressor(max_depth=5)],
                'n_estimators': [60],
                'random_state': [2]
             }


# choose best model with grid_search: 
tune_model = model_selection.GridSearchCV(ensemble.AdaBoostRegressor(), param_grid=param_grid, cv=cv_split, n_jobs=-1)
tune_model.fit(df[x], df[y])

print('AFTER Parameters: ', tune_model.best_params_)
print("AFTER Training score mean: {:.2f}". format(tune_model.cv_results_['mean_train_score'][tune_model.best_index_]*100)) 
print("AFTER Test score mean: {:.2f}". format(tune_model.cv_results_['mean_test_score'][tune_model.best_index_]*100))
print("AFTER Test score 3*std: +/- {:.2f}". format(tune_model.cv_results_['std_test_score'][tune_model.best_index_]*100*3))
print('-'*10)
### Feature Tuning

#base model
ada = ensemble.AdaBoostRegressor(tree.DecisionTreeRegressor(max_depth=5), n_estimators=60)
base_results = model_selection.cross_validate(ada, df[x], df[y], cv=cv_split, n_jobs=-1)
ada.fit(df[x], df[y])

print('BEFORE RFE Training Shape Old: ', df[x].shape) 
print('BEFORE RFE Training Columns Old: ', df[x].columns.values)

print("BEFORE RFE Training score mean: {:.2f}". format(base_results['train_score'].mean()*100)) 
print("BEFORE RFE Test score mean: {:.2f}". format(base_results['test_score'].mean()*100))
print("BEFORE RFE Test score 3*std: +/- {:.2f}". format(base_results['test_score'].std()*100*3))
print('-'*10)



#feature selection
ada_rfe = feature_selection.RFECV(ada, step = 1, cv = cv_split, n_jobs=-1)
ada_rfe.fit(df[x], df[y])

#transform x&y to reduced features and fit new model
X_rfe = df[x].columns.values[ada_rfe.get_support()]
rfe_results = model_selection.cross_validate(ada, df[X_rfe], df[y], cv  = cv_split, n_jobs=-1)

print('AFTER RFE Training Shape New: ', df[X_rfe].shape) 
print('AFTER RFE Training Columns New: ', X_rfe)

print("AFTER RFE Training score mean: {:.2f}". format(rfe_results['train_score'].mean()*100)) 
print("AFTER RFE Test score mean: {:.2f}". format(rfe_results['test_score'].mean()*100))
print("AFTER RFE Test score 3*std: +/- {:.2f}". format(rfe_results['test_score'].std()*100*3))
print('-'*10)
ada_x = ['age_AST%', 'age_DRB%', 'age_DRtg', 'age_ORtg', 'age_PER', 'age_PPR', 'age_STL%',
 'age_BPM', 'age_MIN:GP', 'age_USGxTS', 'age_STLK%', 'age_BLK%_ppctl',
 'age_DRtg_ppctl', 'age_PPR_ppctl', 'age_PPS_ppctl', 'age_STL%_ppctl',
 'age_TOV%_ppctl', 'age_TRB%_ppctl', 'age_USG%_ppctl', 'age_eFG%_ppctl',
 'age_BPM_ppctl', 'age_PF:STLK_ppctl', 'age_rFG%', 'age_r3P%', 'age_rFT%',
 'Height_ppctl', 'Weight_ppctl', 'Age', 'RSCI', 'age_pDBPM_ppctl', 'age_pDBPM',
 'age_PER_ppctl', 'age_FTRate_ppctl']

#base model
ada = ensemble.AdaBoostRegressor(tree.DecisionTreeRegressor(max_depth=5), n_estimators=60)
base_results = model_selection.cross_validate(ada, df[ada_x], df[y], cv=cv_split, n_jobs=-1)
ada.fit(df[ada_x], df[y])


print('BEFORE Parameters: ', ada.get_params())
print("BEFORE Training score mean: {:.2f}". format(base_results['train_score'].mean()*100)) 
print("BEFORE Test score mean: {:.2f}". format(base_results['test_score'].mean()*100))
print("BEFORE Test score 3*std: +/- {:.2f}". format(base_results['test_score'].std()*100*3))
print("BEFORE Test set score min: {:.2f}". format(base_results['test_score'].min()*100))
print('-'*10)


#tune hyper-parameters: 
param_grid = {
                'base_estimator': [tree.DecisionTreeRegressor(max_depth=5, max_features=None),
                                   tree.DecisionTreeRegressor(max_depth=5, max_features=.1),
                                   tree.DecisionTreeRegressor(max_depth=5, max_features=.2),
                                   tree.DecisionTreeRegressor(max_depth=5, max_features=.3),
                                   tree.DecisionTreeRegressor(max_depth=5, max_features=.4),
                                   tree.DecisionTreeRegressor(max_depth=5, max_features=.5),
                                   tree.DecisionTreeRegressor(max_depth=5, max_features=.6),
                                   tree.DecisionTreeRegressor(max_depth=5, max_features=.7),
                                   tree.DecisionTreeRegressor(max_depth=5, max_features=.8),
                                   tree.DecisionTreeRegressor(max_depth=5, max_features=.9)],
                'n_estimators': [60],
                'random_state': [2]
             }

# choose best model with grid_search: 
tune_model = model_selection.GridSearchCV(ensemble.AdaBoostRegressor(), param_grid=param_grid, cv=cv_split, n_jobs=-1)
tune_model.fit(df[ada_x], df[y])

print('AFTER DT Parameters: ', tune_model.best_params_)
print("AFTER DT Training score mean: {:.2f}". format(tune_model.cv_results_['mean_train_score'][tune_model.best_index_]*100)) 
print("AFTER DT Test score mean: {:.2f}". format(tune_model.cv_results_['mean_test_score'][tune_model.best_index_]*100))
print("AFTER DT Test score 3*std: +/- {:.2f}". format(tune_model.cv_results_['std_test_score'][tune_model.best_index_]*100*3))
print('-'*10)
ada_x = ['age_AST%', 'age_DRB%', 'age_DRtg', 'age_ORtg', 'age_PER', 'age_PPR', 'age_STL%',
 'age_BPM', 'age_MIN:GP', 'age_USGxTS', 'age_STLK%', 'age_BLK%_ppctl',
 'age_DRtg_ppctl', 'age_PPR_ppctl', 'age_PPS_ppctl', 'age_STL%_ppctl',
 'age_TOV%_ppctl', 'age_TRB%_ppctl', 'age_USG%_ppctl', 'age_eFG%_ppctl',
 'age_BPM_ppctl', 'age_PF:STLK_ppctl', 'age_rFG%', 'age_r3P%', 'age_rFT%',
 'Height_ppctl', 'Weight_ppctl', 'Age', 'RSCI', 'age_pDBPM_ppctl', 'age_pDBPM',
 'age_PER_ppctl', 'age_FTRate_ppctl']

#final model
ada = ensemble.AdaBoostRegressor(tree.DecisionTreeRegressor(max_depth=5, max_features=0.2), n_estimators=60)
base_results = model_selection.cross_validate(ada, df[ada_x], df[y], cv=cv_split, n_jobs=-1)
ada.fit(df[ada_x], df[y])


print('Parameters: ', ada.get_params())
print("Training score mean: {:.2f}". format(base_results['train_score'].mean()*100)) 
print("Test score mean: {:.2f}". format(base_results['test_score'].mean()*100))
print("Test score 3*std: +/- {:.2f}". format(base_results['test_score'].std()*100*3))
print("Test set score min: {:.2f}". format(base_results['test_score'].min()*100))
print('-'*10)<jupyter_output><empty_output><jupyter_text>Random Forest<jupyter_code>#base model
rfr = ensemble.RandomForestRegressor(random_state=2, n_jobs=-1)
base_results = model_selection.cross_validate(rfr, df[x], df[y], cv=cv_split, n_jobs=-1)
rfr.fit(df[x], df[y])


print('BEFORE Parameters: ', rfr.get_params())
print("BEFORE Training score mean: {:.2f}". format(base_results['train_score'].mean()*100)) 
print("BEFORE DT Test score mean: {:.2f}". format(base_results['test_score'].mean()*100))
print("BEFORE DT Test score 3*std: +/- {:.2f}". format(base_results['test_score'].std()*100*3))
print("BEFORE DT Test w/bin set score min: {:.2f}". format(base_results['test_score'].min()*100))
print('-'*10)


#tune hyper-parameters: 
param_grid = {
                'n_estimators': [750, 800, 850],
                'max_features': [.25, .3, .35],
                'min_samples_split': [10, 12, 14],
                'min_samples_leaf': [1],
                'max_leaf_nodes': [17, 18, 19],
                'bootstrap': [True],
                'random_state': [2]
             }


# choose best model with grid_search: 
tune_model = model_selection.GridSearchCV(rfr, param_grid=param_grid, cv=cv_split, n_jobs=-1)
tune_model.fit(df[x], df[y])

print('AFTER Parameters: ', tune_model.best_params_)
print("AFTER Training score mean: {:.2f}". format(tune_model.cv_results_['mean_train_score'][tune_model.best_index_]*100)) 
print("AFTER Test score mean: {:.2f}". format(tune_model.cv_results_['mean_test_score'][tune_model.best_index_]*100))
print("AFTER Test score 3*std: +/- {:.2f}". format(tune_model.cv_results_['std_test_score'][tune_model.best_index_]*100*3))
print('-'*10)
# feature tuning

#base model
rfr = ensemble.RandomForestRegressor(bootstrap=True, 
                                     max_features=0.3, 
                                     max_leaf_nodes=18,
                                     min_samples_split=12, 
                                     n_estimators=800,
                                     random_state=2, n_jobs=-1)
base_results = model_selection.cross_validate(rfr, df[x], df[y], cv=cv_split, n_jobs=-1)
rfr.fit(df[x], df[y])

print('BEFORE RFE Training Shape Old: ', df[x].shape) 
print('BEFORE RFE Training Columns Old: ', df[x].columns.values)

print("BEFORE RFE Training score mean: {:.2f}". format(base_results['train_score'].mean()*100)) 
print("BEFORE RFE Test score mean: {:.2f}". format(base_results['test_score'].mean()*100))
print("BEFORE RFE Test score 3*std: +/- {:.2f}". format(base_results['test_score'].std()*100*3))
print('-'*10)



#feature selection
rfr_rfe = feature_selection.RFECV(rfr, step = 1, cv = cv_split, n_jobs=-1)
rfr_rfe.fit(df[x], df[y])

#transform x&y to reduced features and fit new model
X_rfe = df[x].columns.values[rfr_rfe.get_support()]
rfe_results = model_selection.cross_validate(rfr, df[X_rfe], df[y], cv  = cv_split, n_jobs=-1)

print('AFTER RFE Training Shape New: ', df[X_rfe].shape) 
print('AFTER RFE Training Columns New: ', X_rfe)

print("AFTER RFE Training score mean: {:.2f}". format(rfe_results['train_score'].mean()*100)) 
print("AFTER RFE Test score mean: {:.2f}". format(rfe_results['test_score'].mean()*100))
print("AFTER RFE Test score 3*std: +/- {:.2f}". format(rfe_results['test_score'].std()*100*3))
print('-'*10)
rfr_x = ['age_DRtg', 'age_ORtg', 'age_PER', 'age_PPR', 'age_STL%', 'age_BPM',
 'age_DRB%_ppctl', 'age_DRtg_ppctl', 'age_ORtg_ppctl', 'age_PPR_ppctl',
 'age_STL%_ppctl', 'age_TS%_ppctl', 'age_eFG%_ppctl', 'age_BPM_ppctl',
 'age_rFT%', 'Height_ppctl', 'Age', 'age_pDBPM_ppctl', 'age_pDBPM',
 'age_PER_ppctl']

#base model
rfr = ensemble.RandomForestRegressor(random_state=2, n_jobs=-1)
base_results = model_selection.cross_validate(rfr, df[rfr_x], df[y], cv=cv_split, n_jobs=-1)
rfr.fit(df[rfr_x], df[y])


print('BEFORE Parameters: ', rfr.get_params())
print("BEFORE Training score mean: {:.2f}". format(base_results['train_score'].mean()*100)) 
print("BEFORE Test score mean: {:.2f}". format(base_results['test_score'].mean()*100))
print("BEFORE Test score 3*std: +/- {:.2f}". format(base_results['test_score'].std()*100*3))
print("BEFORE Test set score min: {:.2f}". format(base_results['test_score'].min()*100))
print('-'*10)


#tune hyper-parameters: 
param_grid = {
                'n_estimators': [850, 900, 1000],
                'max_features': [.35, .4, .45],
                'min_samples_split': [6, 8, 10],
                'min_samples_leaf': [1],
                'max_leaf_nodes': [19, 20, 21],
                'bootstrap': [True],
                'random_state': [2]
             }


# choose best model with grid_search: 
tune_model = model_selection.GridSearchCV(rfr, param_grid=param_grid, cv=cv_split, n_jobs=-1)
tune_model.fit(df[rfr_x], df[y])

print('AFTER Parameters: ', tune_model.best_params_)
print("AFTER Training score mean: {:.2f}". format(tune_model.cv_results_['mean_train_score'][tune_model.best_index_]*100)) 
print("AFTER Test score mean: {:.2f}". format(tune_model.cv_results_['mean_test_score'][tune_model.best_index_]*100))
print("AFTER Test score 3*std: +/- {:.2f}". format(tune_model.cv_results_['std_test_score'][tune_model.best_index_]*100*3))
print('-'*10)
rfr_x = ['age_DRtg', 'age_ORtg', 'age_PER', 'age_PPR', 'age_STL%', 'age_BPM',
 'age_DRB%_ppctl', 'age_DRtg_ppctl', 'age_ORtg_ppctl', 'age_PPR_ppctl',
 'age_STL%_ppctl', 'age_TS%_ppctl', 'age_eFG%_ppctl', 'age_BPM_ppctl',
 'age_rFT%', 'Height_ppctl', 'Age', 'age_pDBPM_ppctl', 'age_pDBPM',
 'age_PER_ppctl']

# final model
rfr = ensemble.RandomForestRegressor(bootstrap=True, 
                                     max_features=0.35, 
                                     max_leaf_nodes=19,
                                     min_samples_split=10, 
                                     n_estimators=850,
                                     random_state=2, n_jobs=-1)
base_results = model_selection.cross_validate(rfr, df[rfr_x], df[y], cv=cv_split, n_jobs=-1)
rfr.fit(df[rfr_x], df[y])

print('Parameters: ', rfr.get_params())
print("Training score mean: {:.2f}". format(base_results['train_score'].mean()*100)) 
print("Test score mean: {:.2f}". format(base_results['test_score'].mean()*100))
print("Test score 3*std: +/- {:.2f}". format(base_results['test_score'].std()*100*3))
print('-'*10)<jupyter_output><empty_output><jupyter_text>Extra Trees<jupyter_code>#base model
etr = ensemble.ExtraTreesRegressor(random_state=2, n_jobs=-1)
base_results = model_selection.cross_validate(etr, df[x], df[y], cv=cv_split, n_jobs=-1)
etr.fit(df[x], df[y])


print('BEFORE Parameters: ', etr.get_params())
print("BEFORE Training score mean: {:.2f}". format(base_results['train_score'].mean()*100)) 
print("BEFORE Test score mean: {:.2f}". format(base_results['test_score'].mean()*100))
print("BEFORE Test score 3*std: +/- {:.2f}". format(base_results['test_score'].std()*100*3))
print("BEFORE Test set score min: {:.2f}". format(base_results['test_score'].min()*100))
print('-'*10)


#tune hyper-parameters: 
param_grid = {
                'n_estimators': [1800],
                'max_features': [.5],
                'min_samples_split': [8],
                'min_samples_leaf': [1],
                'bootstrap': [True],
                'random_state': [2]
             }


# choose best model with grid_search: 
tune_model = model_selection.GridSearchCV(etr, param_grid=param_grid, cv=cv_split, n_jobs=-1)
tune_model.fit(df[x], df[y])

print('AFTER Parameters: ', tune_model.best_params_)
print("AFTER Training score mean: {:.2f}". format(tune_model.cv_results_['mean_train_score'][tune_model.best_index_]*100)) 
print("AFTER Test score mean: {:.2f}". format(tune_model.cv_results_['mean_test_score'][tune_model.best_index_]*100))
print("AFTER Test score 3*std: +/- {:.2f}". format(tune_model.cv_results_['std_test_score'][tune_model.best_index_]*100*3))
print('-'*10)
# feature tuning

#base model
etr = ensemble.ExtraTreesRegressor(bootstrap=True, 
                                   max_features=0.5, 
                                   min_samples_split=8, 
                                   n_estimators=1800,
                                   random_state=2, n_jobs=-1)
base_results = model_selection.cross_validate(etr, df[x], df[y], cv=cv_split, n_jobs=-1)
etr.fit(df[x], df[y])

print('BEFORE RFE Training Shape Old: ', df[x].shape) 
print('BEFORE RFE Training Columns Old: ', df[x].columns.values)

print("BEFORE RFE Training score mean: {:.2f}". format(base_results['train_score'].mean()*100)) 
print("BEFORE RFE Test score mean: {:.2f}". format(base_results['test_score'].mean()*100))
print("BEFORE RFE Test score 3*std: +/- {:.2f}". format(base_results['test_score'].std()*100*3))
print('-'*10)

#feature selection
etr_rfe = feature_selection.RFECV(etr, step = 1, cv = cv_split, n_jobs=-1)
etr_rfe.fit(df[x], df[y])

#transform x&y to reduced features and fit new model
X_rfe = df[x].columns.values[etr_rfe.get_support()]
rfe_results = model_selection.cross_validate(etr, df[X_rfe], df[y], cv  = cv_split, n_jobs=-1)

print('AFTER RFE Training Shape New: ', df[X_rfe].shape) 
print('AFTER RFE Training Columns New: ', X_rfe)

print("AFTER RFE Training score mean: {:.2f}". format(rfe_results['train_score'].mean()*100)) 
print("AFTER RFE Test score mean: {:.2f}". format(rfe_results['test_score'].mean()*100))
print("AFTER RFE Test score 3*std: +/- {:.2f}". format(rfe_results['test_score'].std()*100*3))
print('-'*10)
etr_x = ['age_DRtg', 'age_ORtg', 'age_PER', 'age_PPR', 'age_STL%', 'age_BPM',
 'age_MIN:GP', 'age_USGxTS', 'age_BLK%_ppctl', 'age_DRB%_ppctl',
 'age_DRtg_ppctl', 'age_ORtg_ppctl', 'age_PPR_ppctl', 'age_PPS_ppctl',
 'age_STL%_ppctl', 'age_Total S %_ppctl', 'age_eFG%_ppctl', 'age_BPM_ppctl',
 'age_STLK%_ppctl', 'age_PF:STLK_ppctl', 'Height', 'Height_ppctl', 'Age',
 'age_pDBPM_ppctl', 'age_PER_ppctl']

#base model
etr = ensemble.ExtraTreesRegressor(random_state=2, n_jobs=-1)
base_results = model_selection.cross_validate(etr, df[etr_x], df[y], cv=cv_split, n_jobs=-1)
etr.fit(df[etr_x], df[y])


print('BEFORE Parameters: ', etr.get_params())
print("BEFORE Training score mean: {:.2f}". format(base_results['train_score'].mean()*100)) 
print("BEFORE Test score mean: {:.2f}". format(base_results['test_score'].mean()*100))
print("BEFORE Test score 3*std: +/- {:.2f}". format(base_results['test_score'].std()*100*3))
print("BEFORE Test set score min: {:.2f}". format(base_results['test_score'].min()*100))
print('-'*10)


#tune hyper-parameters: 
param_grid = {
                'n_estimators': [1800],
                'max_features': [.55, .6],
                'min_samples_split': [7, 8],
                'bootstrap': [True],
                'random_state': [2]
             }


# choose best model with grid_search: 
tune_model = model_selection.GridSearchCV(etr, param_grid=param_grid, cv=cv_split, n_jobs=-1)
tune_model.fit(df[etr_x], df[y])

print('AFTER Parameters: ', tune_model.best_params_)
print("AFTER Training score mean: {:.2f}". format(tune_model.cv_results_['mean_train_score'][tune_model.best_index_]*100)) 
print("AFTER Test score mean: {:.2f}". format(tune_model.cv_results_['mean_test_score'][tune_model.best_index_]*100))
print("AFTER Test score 3*std: +/- {:.2f}". format(tune_model.cv_results_['std_test_score'][tune_model.best_index_]*100*3))
print('-'*10)
etr_x = ['age_DRtg', 'age_ORtg', 'age_PER', 'age_PPR', 'age_STL%', 'age_BPM',
 'age_MIN:GP', 'age_USGxTS', 'age_BLK%_ppctl', 'age_DRB%_ppctl',
 'age_DRtg_ppctl', 'age_ORtg_ppctl', 'age_PPR_ppctl', 'age_PPS_ppctl',
 'age_STL%_ppctl', 'age_Total S %_ppctl', 'age_eFG%_ppctl', 'age_BPM_ppctl',
 'age_STLK%_ppctl', 'age_PF:STLK_ppctl', 'Height', 'Height_ppctl', 'Age',
 'age_pDBPM_ppctl', 'age_PER_ppctl']

#final model
etr = ensemble.ExtraTreesRegressor(bootstrap=True, 
                                   max_features=0.55, 
                                   min_samples_split=7, 
                                   n_estimators=1800,
                                   random_state=2, n_jobs=-1)
base_results = model_selection.cross_validate(etr, df[etr_x], df[y], cv=cv_split, n_jobs=-1)
etr.fit(df[etr_x], df[y])


print("Training score mean: {:.2f}". format(base_results['train_score'].mean()*100)) 
print("Test score mean: {:.2f}". format(base_results['test_score'].mean()*100))
print("Test score 3*std: +/- {:.2f}". format(base_results['test_score'].std()*100*3))
print('-'*10)<jupyter_output><empty_output><jupyter_text>Blend Models<jupyter_code>from mlxtend.regressor import StackingCVRegressor
from sklearn.model_selection import cross_val_score

random_state = 2
np.random.seed(random_state)

# models before feature tuning
etr = ensemble.ExtraTreesRegressor(bootstrap=True, 
                                   max_features=0.5, 
                                   min_samples_split=8, 
                                   n_estimators=1800,
                                   random_state=2, n_jobs=-1)

rfr = ensemble.RandomForestRegressor(bootstrap=True, 
                                     max_features=0.3, 
                                     max_leaf_nodes=18,
                                     min_samples_split=12, 
                                     n_estimators=800,
                                     random_state=2, n_jobs=-1)

ada = ensemble.AdaBoostRegressor(tree.DecisionTreeRegressor(max_depth=5, max_features=0.2), n_estimators=60)

bag = ensemble.BaggingRegressor(ensemble.GradientBoostingRegressor(n_estimators=125, max_depth=3, min_samples_split=7),
                                n_estimators=35,
                                max_samples=.4,
                                bootstrap_features=False)

xgb = XGBRegressor(colsample_bylevel=0.25, colsample_bytree=0.25, max_depth=4, n_estimators=30)

grb = ensemble.GradientBoostingRegressor(max_depth=3, min_samples_leaf=10, n_estimators=40, max_features=.1, subsample=.75)

rdg = linear_model.RidgeCV(alphas=(175.0,))

meta_regr = ensemble.RandomForestRegressor()

models = [etr, rfr, ada, bag, xgb, grb, rdg]

stack = StackingCVRegressor(regressors=models,
                            meta_regressor=meta_regr)

models_plus_stack = models + [stack]

x = ['age_DRtg', 'age_ORtg', 'age_PER', 'age_PPR', 'age_STL%', 'age_BPM',
 'age_MIN:GP', 'age_USGxTS', 'age_BLK%_ppctl', 'age_DRB%_ppctl',
 'age_DRtg_ppctl', 'age_ORtg_ppctl', 'age_PPR_ppctl', 'age_PPS_ppctl',
 'age_STL%_ppctl', 'age_Total S %_ppctl', 'age_eFG%_ppctl', 'age_BPM_ppctl',
 'age_STLK%_ppctl', 'age_PF:STLK_ppctl', 'Height', 'Height_ppctl', 'Age',
 'age_pDBPM_ppctl', 'age_PER_ppctl']
X = np.array(df[x])
y = np.array(df['ep_PIPM'].tolist())


print('5-fold cross validation scores:\n')

for clf, label in zip(models_plus_stack, ['ExtraTrees', 'RandomForest', 'AdaBoost', 'Bagging', 
                                          'XGBoost', 'GradientBoost', 'RidgeRegr', 'StackingCVRegressor']):
    scores = cross_val_score(clf, X, y, cv=5)
    print("R^2 Score: %0.2f (+/- %0.2f) [%s]" % (
        scores.mean(), scores.std(), label))<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.18125277654375832, 'fraction_numerical': 0.03109729009329187, 'mean_word_length': 4.8035771833709315, 'pattern_counts': {'":': 0, '<': 33, '<?xml version=': 0, '>': 33, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 38, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '8', 'dir': 'jupyter-structured-clean-dedup', 'id': '179945', 'license': 'MIT', 'n_tokens_mistral': 14747, 'n_tokens_neox': 13718, 'n_words': 2282, 'path': 'blended_single_value_draft_model.ipynb', 'repo_name': 'sderenth/Draft-Model', 'size': '36016'}