<jupyter_start><jupyter_text>Random Forest<jupyter_code>ran = RandomForestRegressor()


result_r2 = cross_val_score(ran, X, y, cv=kfold, 
                          scoring='r2',n_jobs=-1)

result_mse = cross_val_score(ran, X, y, cv=kfold, 
                          scoring='neg_root_mean_squared_error',n_jobs=-1)

print('r2\n',result_r2,result_r2.mean())

print('rmse\n',result_mse,result_mse.mean())

ran_r2 = result_r2.mean()
ran_rmse = result_rmse.mean()
ran = RandomForestRegressor()
viz = FeatureImportances(ran)
viz.fit(X, y)
viz.show()
parameters = {'n_estimators':np.arange(100,1000,50),
              'min_samples_split':np.arange(10,100),
              'min_samples_leaf':np.arange(1,50),
              'max_features': np.arange(1,30),
              'max_depth':np.arange(20,45)}

model_rf = RandomForestRegressor(random_state=42)
cv = RandomizedSearchCV(model_rf, param_distributions=parameters,
                  cv=kfold,scoring='r2',n_iter=300,n_jobs=-1)

cv.fit( X , y )

results_df = pd.DataFrame(cv.cv_results_)

print(cv.best_params_)

print(cv.best_score_)

print(cv.best_estimator_)<jupyter_output><empty_output><jupyter_text>n_estimators=140		min_samples_split = 20		min_samples_leaf= 1		max_features = 15		max_depth=43<jupyter_code>model_rf = RandomForestRegressor(random_state=42,
                                 n_estimators=cv.best_params_['n_estimators'],
                                 max_features=cv.best_params_['max_features'],\
                                 min_samples_leaf=cv.best_params_['min_samples_leaf'],\
                                 max_depth= cv.best_params_['max_depth'],\
                                 min_samples_split = cv.best_params_['min_samples_split'],\
                                 oob_score=True)

result_r2 = cross_val_score(model_rf, X, y, cv=kfold, 
                          scoring='r2',n_jobs=-1)

result_rmse = cross_val_score(model_rf, X, y, cv=kfold, 
                          scoring='neg_root_mean_squared_error',n_jobs=-1)

print('r2\n',result_r2,result_r2.mean())

print('rmse\n',result_mse,result_mse.mean())

ran_tun_r2 = result_r2.mean()
ran_tun_rmse = result_rmse.mean()<jupyter_output><empty_output><jupyter_text>r2 0.8361	rmse -1560  LightGBM<jupyter_code>lgb = LGBMRegressor()

result_r2 = cross_val_score(lgb, X, y, cv=kfold, 
                          scoring='r2',n_jobs=-1)

result_rmse = cross_val_score(lgb, X, y, cv=kfold, 
                          scoring='neg_root_mean_squared_error',n_jobs=-1)

print('r2\n',result_r2,result_r2.mean())

print('rmse\n',result_rmse,result_rmse.mean())

lgb_r2 = result_r2.mean()
lgb_rmse = result_rmse.mean()
parameters = {'n_estimators':np.arange(100,1000,50),
              'num_leaves ':np.arange(10,100),
              'max_depth':np.arange(20,45),
             'learning_rate':np.arange(0.001,1,0.002)}

model_lgb = LGBMRegressor(random_state=42,n_jobs=-1)

cv = RandomizedSearchCV(model_lgb, param_distributions=parameters,
                  cv=kfold,scoring='r2',n_iter=300,n_jobs=-1)

cv.fit( X , y )

results_df = pd.DataFrame(cv.cv_results_)

print(cv.best_params_)

print(cv.best_score_)

print(cv.best_estimator_)
model_lgb = LGBMRegressor(random_state=42,
                                 n_estimators=cv.best_params_['n_estimators'],\
                                 num_leaves=cv.best_params_['num_leaves '],\
                                 max_depth= cv.best_params_['max_depth'],\
                                 learning_rate = cv.best_params_['learning_rate'])

result_r2 = cross_val_score(model_lgb, X, y, cv=kfold, 
                          scoring='r2',n_jobs=-1)

result_rmse = cross_val_score(model_lgb, X, y, cv=kfold, 
                          scoring='neg_root_mean_squared_error',n_jobs=-1)

print('r2\n',result_r2,result_r2.mean())

print('rmse\n',result_mse,result_mse.mean())

lgb_tun_r2 = result_r2.mean()
lgb_tun_rmse = result_rmse.mean()
model_lgb = LGBMRegressor(random_state=42,
                                 n_estimators=cv.best_params_['n_estimators'],\
                                 num_leaves=cv.best_params_['num_leaves '],\
                                 max_depth= cv.best_params_['max_depth'],\
                                 learning_rate = cv.best_params_['learning_rate'])

viz = FeatureImportances(model_lgb)
viz.fit(X, y)
viz.show()<jupyter_output><empty_output><jupyter_text>XGBOOST<jupyter_code>xgb = XGBRegressor()

result_r2 = cross_val_score(xgb, X, y, cv=kfold, 
                          scoring='r2',n_jobs=-1)

result_rmse = cross_val_score(xgb, X, y, cv=kfold, 
                          scoring='neg_root_mean_squared_error',n_jobs=-1)

print('r2\n',result_r2,result_r2.mean())

print('rmse\n',result_rmse,result_rmse.mean())

xgb_r2 = result_r2.mean()
xgb_rmse = result_rmse.mean()
xgb = XGBRegressor()
viz = FeatureImportances(xgb)
viz.fit(X, y)
viz.show()
r_range = [0.1, 0.2,0.25, 0.3,0.35]
n_est_range = [30,50,75,100,120,150]
md_range = [4,6,8,10]

parameters = dict(learning_rate=lr_range,
                  n_estimators=n_est_range,
                  max_depth=md_range)

from sklearn.model_selection import GridSearchCV,cross_val_score,RandomizedSearchCV
clf = XGBRegressor(random_state=42,tree_method='gpu_hist',gpu_id=0)
cv = GridSearchCV(clf,parameters,
                  cv=5,scoring='r2',n_jobs=-1)

cv.fit(X,y)

print(cv.best_params_)

print(cv.best_score_)
model_xgb = XGBRegressor(max_depth= 6,\
                   learning_rate =0.25,\
                   n_estimators = 100,\
                  random_state=42)<jupyter_output><empty_output><jupyter_text>best params{'learning_rate': 0.25, 'max_depth': 6, 'n_estimators': 100}0.8635339651012288<jupyter_code>model_xgb = XGBRegressor(max_depth= 6,\
                   learning_rate =0.25,\
                   n_estimators = 100,\
                  random_state=42)


result_r2 = cross_val_score(model_xgb, X, y, cv=kfold, 
                          scoring='r2',n_jobs=-1)

result_rmse = cross_val_score(model_xgb, X, y, cv=kfold, 
                          scoring='neg_root_mean_squared_error',n_jobs=-1)

print('r2\n',result_r2,result_r2.mean())

print('rmse\n',result_rmse,result_rmse.mean())

xgb_tun_r2 = result_r2.mean()
xgb_tun_rmse = result_rmse.mean()
model_xgb = XGBRegressor(max_depth= 6,\
                   learning_rate =0.25,\
                   n_estimators = 100,\
                  random_state=42)

viz = FeatureImportances(model_xgb)
viz.fit(X, y)
viz.show()
from sklearn.ensemble import StackingRegressor

models_considered = [('XGBoost Rgressor', model_xgb),
                     ('Random Forest', ran),
                    ('Light GBM',model_lgb)
                    ]
xgb1 = XGBRegressor(random_state=42,max_depth=6,learning_rate=0.25)

stack = StackingRegressor(estimators = models_considered,
                           final_estimator=xgb1)

result_r2 = cross_val_score(stack, X, y, cv=kfold, 
                          scoring='r2',n_jobs=-1)

result_rmse = cross_val_score(stack, X, y, cv=kfold, 
                          scoring='neg_root_mean_squared_error',n_jobs=-1)

print('r2\n',result_r2,result_r2.mean())

print('rmse\n',result_rmse,result_rmse.mean())

stack_r2 = result_r2.mean()
stack_rmse = result_rmse.mean()
stack_passthrough = StackingRegressor(estimators = models_considered,
                           final_estimator=xgb1,passthrough=True)

result_r2 = cross_val_score(stack_passthrough, X, y, cv=kfold, 
                          scoring='r2',n_jobs=-1)

result_rmse = cross_val_score(stack_passthrough, X, y, cv=kfold, 
                          scoring='neg_root_mean_squared_error',n_jobs=-1)

print('r2\n',result_r2,result_r2.mean())

print('rmse\n',result_rmse,result_rmse.mean())

stack_passthrough_r2 = result_r2.mean()
stack_passthrough_rmse = result_rmse.mean()
viz = FeatureImportances(lr)
viz.fit(X, y)
viz.show()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.12787010021565393, 'fraction_numerical': 0.033236077635417985, 'mean_word_length': 2.945945945945946, 'pattern_counts': {'":': 0, '<': 21, '<?xml version=': 0, '>': 21, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 14, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '5', 'dir': 'jupyter-structured-clean-dedup', 'id': '584017', 'license': 'Apache-2.0', 'n_tokens_mistral': 3152, 'n_tokens_neox': 2966, 'n_words': 331, 'path': 'Flight Price_Code/Building Models.ipynb', 'repo_name': 'krishnasrujan/Flight-Price-Prediction', 'size': '7883'}