<jupyter_start><jupyter_text>Create Basline Models<jupyter_code># Read and Clean CSV
churn = data_cleaner(pd.read_csv("data/churn_train.csv"))
churn.head()
# Create X, y arrays from dataframe
X = churn
y = churn.pop("target")

# Train Test Split Data
X_train, X_test, y_train, y_test = train_test_split(X, y)<jupyter_output><empty_output><jupyter_text>Random Forest Classifier<jupyter_code># Create Random Forest Model
model_rf = RandomForestClassifier(
                            oob_score=True,
                            max_features=3,
)

# Fit Data
model_rf.fit(X_train, y_train)
# Feature Importances

# See and plot feature importances 
feat_scores = pd.DataFrame({'Churn Feature Importances' : model_rf.feature_importances_},
                           index=X.columns)
feat_scores = feat_scores.sort_values(by='Churn Feature Importances')
feat_scores.plot(kind='barh', figsize=(9, 6))
# Out of Bag Error
model_rf.oob_score_
# Prediciton
# model_rf.predict(X)<jupyter_output><empty_output><jupyter_text>Random Forest Performance<jupyter_code>get_score(model_rf, X_train, y_train)<jupyter_output>RandomForestClassifier     Train CV | MSE: 0.2538734896943852 | R2: -0.08881632745508536 | Acc: 0.7440653873489694<jupyter_text>Gradient Boosting Classifier<jupyter_code># Create Gradient Boosting Model
model_gb = GradientBoostingClassifier(learning_rate=0.1,
                                      n_estimators=100,
                                      random_state=1)
# Fit Model
model_gb.fit(X_train, y_train)
# Prediction
y_pred_gb = model_gb.predict(X_test)
# Feature Importances
feat_scores = pd.DataFrame({'Churn Feature Importances GDBC' : model_gb.feature_importances_}, index=X.columns)
feat_scores = feat_scores.sort_values(by='Churn Feature Importances GDBC')
# Feature Importances

# See and plot feature importances 
feat_scores = pd.DataFrame({'Churn Feature Importances' : model_rf.feature_importances_},
                           index=X.columns)
feat_scores = feat_scores.sort_values(by='Churn Feature Importances')
feat_scores.plot(kind='barh', figsize=(9, 6))<jupyter_output><empty_output><jupyter_text>Sample Tree Plot<jupyter_code>fig, ax = plt.subplots(figsize=(14, 10))
tree_1 = model_gb.estimators_[0][0]
tree_1_plot = plot_tree(tree_1, ax=ax)

fig.tight_layout()
plt.show()<jupyter_output><empty_output><jupyter_text>Confusion Matrix<jupyter_code># Confusion Matrix GBC
true_neg, false_pos, false_neg, true_pos = confusion_matrix(y_test, y_pred_gb).flatten()
true_neg, false_pos, false_neg, true_pos<jupyter_output><empty_output><jupyter_text>| -        |       Predicted Negative      |  Predicted Positive || ------------- |:-------------:| -----:|| Actual Negative | 5409 (TN)  | 890 (FP)| Actual Positive | 1308 (FN) | 2393 (TP)<jupyter_code># Precision GBC
precision_before = true_pos / (true_pos + false_pos)
precision_before
# Recall GBC
recall_before = true_pos / (true_pos + false_neg)
recall_before<jupyter_output><empty_output><jupyter_text>Gradient Boosting Classifier Peformance<jupyter_code>get_score(model_gb, X_train, y_train)<jupyter_output><empty_output><jupyter_text>Gradient Boosting Regression<jupyter_code>model_gbr = GradientBoostingRegressor(learning_rate=0.1, loss='ls',
                                 n_estimators=100, random_state=1)<jupyter_output><empty_output><jupyter_text>Gradient Boosting Regression Peformance<jupyter_code># get_score(model_gbr, X_train, y_train)<jupyter_output><empty_output><jupyter_text>Gradient Boosting Grid Search> Find best hyper-parameters for Gradient Boosting<jupyter_code># # Create Grid Search
# gradient_boosting_grid = {'learning_rate': [0.1, 0.05, 0.02, 0.01],
#                           'max_depth': [2, 4, 6],
#                           'min_samples_leaf': [1, 2, 5, 10],
#                           'max_features': [1.0, 0.3, 0.1],
#                           'n_estimators': [100],
#                           'random_state': [1]}
# gdbr_best_params, gdbr_best_model = gridsearch_with_output(GradientBoostingRegressor(), 
#                                                            gradient_boosting_grid, 
#                                                            X_train, y_train)
# print("\nComparing model with gridsearch params to initial model on Test set.")
# model_gbr.fit(X_train, y_train)
display_default_and_gsearch_model_results(model_gbr, gdbr_best_model, X_test, y_test)<jupyter_output>Results for GradientBoosting
Gridsearched model mse: 0.147 | r2: 0.365
     Default model mse: 0.149 | r2: 0.354<jupyter_text>Grid Search Best Parameters<jupyter_code>gdbr_best_params
y_pred_gdbr_best = gdbr_best_model.predict(X_test)<jupyter_output><empty_output><jupyter_text>Gradient Boosting Regression Peformance (POST GRID SEARCH)> Baseline Threshold: 0.5<jupyter_code>display_score_metrics(gdbr_best_model, (y_pred_gdbr_best >= 0.5).astype(int), y_test)<jupyter_output>GradientBoostingRegressor     Train CV | MSE: 0.2092 | R2: 0.09452403019390121 | Acc: 0.7908<jupyter_text>> Baseline Threshold: 0.7<jupyter_code>display_score_metrics(gdbr_best_model, (y_pred_gdbr_best >= 0.7).astype(int), y_test)<jupyter_output>GradientBoostingRegressor     Train CV | MSE: 0.2454 | R2: -0.06215967012627455 | Acc: 0.7546<jupyter_text>> Baseline Threshold: 0.3<jupyter_code>display_score_metrics(gdbr_best_model, (y_pred_gdbr_best >= 0.3).astype(int), y_test)<jupyter_output>GradientBoostingRegressor     Train CV | MSE: 0.2347 | R2: -0.01584708467252094 | Acc: 0.7653<jupyter_text>Confusion Matrix<jupyter_code># Confusion Matrix GBR
true_neg, false_pos, false_neg, true_pos = confusion_matrix(y_test, (y_pred_gdbr_best >= 0.5).astype(int)).flatten()
true_neg, false_pos, false_neg, true_pos<jupyter_output><empty_output><jupyter_text>BEFORE| -        |       Predicted Negative      |  Predicted Positive || ------------- |:-------------:| -----:|| Actual Negative | 2386 (TN)  | 1237 (FP)| Actual Positive | 886 (FN) | 5491 (TP)(2386, 1237, 886, 5491) AFTER| -        |       Predicted Negative      |  Predicted Positive || ------------- |:-------------:| -----:|| Actual Negative | 2384 (TN)  | 1239 (FP)| Actual Positive | 853 (FN) | 5524 (TP)(2384, 1239, 853, 5524) PRECISION / RECALL (BEFORE and AFTER)| -        |       Precision      |  Recall || ------------- |:-------------:| -----:|| BEFORE | 0.816  | 0.861| AFTER | 0.817 | 0.87(2423, 1245, 808, 5524)<jupyter_code># Precision GBC
precision_after = true_pos / (true_pos + false_pos)
precision_after
# Recall GBC
recall_after = true_pos / (true_pos + false_neg)
recall_after
(precision_before, recall_before), (precision_after, recall_after)<jupyter_output><empty_output><jupyter_text>GBC MODEL (POST GRID) ROC CURVE<jupyter_code># Plot ROC Curve
fig, ax = plt.subplots(figsize=FSIZE)
plot_roc_curve(model_gb, X_train, y_train, ax=ax)
plot_roc_curve(model_gb, X_train, y_train, ax=ax)<jupyter_output><empty_output><jupyter_text>Gradient Boost Classifier Grid Search<jupyter_code># gb_grid = {'loss': ['deviance','exponential'],'n_estimators': [100,150,200],
#                   'min_samples_leaf': [1,2,3],
#                   'max_depth': [3,4,6],
#                   'max_features': [3,5,9],
#                   'learning_rate': [.05,.1,.15,.2]}

# gb_gridsearch = GridSearchCV(GradientBoostingClassifier(),
#                              gb_grid,
#                              n_jobs=-1,
#                              verbose=True,
#                              scoring='accuracy')

# gb_gridsearch.fit(X_train, y_train)

# print( "Best Params:", gb_gridsearch.best_params_ )

# model_gbc_best = gb_gridsearch.best_estimator_
y_pred = model_gbc_best.predict(X_test)
display_score_metrics(model_gbc_best, y_pred, y_test)
# Random Forest OOB SCORE
model_rf.oob_score_
gb_gridsearch.best_params_<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1267316572601334, 'fraction_numerical': 0.045792714212416624, 'mean_word_length': 3.8159357628165536, 'pattern_counts': {'":': 0, '<': 72, '<?xml version=': 0, '>': 80, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 7, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '19', 'dir': 'jupyter-structured-clean-dedup', 'id': '467954', 'license': 'MIT', 'n_tokens_mistral': 3040, 'n_tokens_neox': 2729, 'n_words': 480, 'path': 'ensemble_modeling.ipynb', 'repo_name': 'boogiedev/churning-a-blind-eye', 'size': '7796'}