<jupyter_start><jupyter_text>Random Forest + Gridsearch CV<jupyter_code>rf = RandomForestClassifier()
rf.fit(X_train, y_train)
rf_pred = rf.predict(X_val)

rf_ac = rf.score(X_val, y_val)
rf_precision = precision_score(y_val, rf_pred)
rf_recall = recall_score(y_val, rf_pred)
rf_f1 = f1_score(y_val, rf_pred)

rf_y_score = rf.predict_proba(X_val)[:, 1]
rf_fpr, rf_tpr, auc_thresholds = roc_curve(y_val, rf_y_score)
roc_auc_rf = auc(rf_fpr, rf_tpr)

print(classification_report(y_val, rf_pred))
print('Random Forest validation metrics: \n Accuracy: %.4f \n Precision: %.4f \n Recall: %.4f \n F1: %.4f \n ROC: %.4f' %
        (rf_ac, 
         rf_precision, 
         rf_recall,
         rf_f1,
         roc_auc_rf)
     )
confusion_matrix(y_val, rf_pred)
# random forest grid search to minimize FP - improve precision
n_estimators = [10,100,200,300,400,500,600,1000]
criterion = ['gini', 'entropy']
param_grid = dict(n_estimators = n_estimators, criterion = criterion)
rf_grid_search = RandomForestClassifier()
rf_grid = GridSearchCV(rf_grid_search, param_grid=param_grid, cv=5, scoring='f1', verbose=2, n_jobs=-1)
rf_grid.fit(X_train, y_train)

rf_grid_preds = rf_grid.predict(X_val)
rf_best_params = rf_grid.best_params_
rf_best_estimator = rf_grid.best_estimator_
rf_best_cm = confusion_matrix(y_val,rf_grid_preds)
rf_best_cr = classification_report(y_val,rf_grid_preds)
print(rf_best_params, rf_best_estimator, rf_best_cm, rf_best_cr)
rf_y_score = rf_grid.predict_proba(X_val)[:, 1]
rf_grid_fpr, rf_grid_tpr, auc_thresholds = roc_curve(y_val, rf_y_score)
roc_auc_rf_grid = auc(rf_grid_fpr, rf_grid_tpr)
print(roc_auc_rf_grid)<jupyter_output>0.7958826507415839<jupyter_text>Logistic Regression + GridsearchCV<jupyter_code>lr = LogisticRegression()
lr.fit(X_train_sc, y_train)
lr_pred = lr.predict(X_val_sc)
lr_confusion = confusion_matrix(y_val, lr_pred)

lr_ac = lr.score(X_val_sc, y_val)
lr_precision = precision_score(y_val, lr_pred)
lr_recall = recall_score(y_val, lr_pred)
lr_f1 = f1_score(y_val, lr_pred)

print(classification_report(y_val, lr_pred))
print('Logistic Regression validation metrics: \n Accuracy: %.4f \n Precision: %.4f \n Recall: %.4f \n F1: %.4f' %
        (lr_ac, 
         lr_precision, 
         lr_recall,
         lr_f1)
     )
# logistic regression grid search to minimize FP - improve precision
penalty = ['l1', 'l2']
C = [0.001,0.01,0.1,1,10,100,1000]
param_grid = dict(C=C, penalty=penalty)
lr_grid_search = LogisticRegression()
lr_grid = GridSearchCV(lr_grid_search, param_grid, cv=5, scoring='f1', verbose=2, n_jobs=-1, refit = True)
lr_grid.fit(X_train_sc, y_train)
lr_grid_preds = lr_grid.predict(X_val_sc)

lr_grid_best_params = lr_grid.best_params_
lr_grid_best_estimator = lr_grid.best_estimator_
lr_grid_best_cm = confusion_matrix(y_val,lr_grid_preds)
lr_grid_best_cr = classification_report(y_val,lr_grid_preds)
print(lr_grid_best_params, lr_grid_best_estimator, lr_grid_best_cm, lr_grid_best_cr)
lr_grid_y_score = lr_grid.predict_proba(X_val)[:, 1]
lr_grid_fpr, lr_grid_tpr, auc_thresholds = roc_curve(y_val, lr_grid_y_score)
roc_auc_lr_grid = auc(lr_grid_fpr, lr_grid_tpr)
print(roc_auc_lr_grid )
# test comparison

# Logistic Regression
scaler = StandardScaler()
X_mid_sc = scaler.fit_transform(X_mid)
X_test_sc = scaler.transform(X_test)

lr = LogisticRegression(C=100)
lr.fit(X_mid_sc, y_mid)
lr_pred = lr.predict(X_test_sc)

lr_ac = lr.score(X_test_sc, y_test)
lr_precision = precision_score(y_test, lr_pred)
lr_recall = recall_score(y_test, lr_pred)
lr_f1 = f1_score(y_test, lr_pred)

lr_y_score = lr.predict_proba(X_test)[:, 1]
lr_fpr, lr_tpr, auc_thresholds = roc_curve(y_test, lr_y_score)
roc_auc_lr = auc(lr_fpr, lr_tpr)

# Random Forst

rf = RandomForestClassifier(criterion='entropy', n_estimators=600)
rf.fit(X_mid, y_mid)
rf_pred = rf.predict(X_test)

rf_ac = rf.score(X_test, y_test)
rf_precision = precision_score(y_test, rf_pred)
rf_recall = recall_score(y_test, rf_pred)
rf_f1 = f1_score(y_test, rf_pred)

rf_y_score = rf.predict_proba(X_test)[:, 1]
rf_fpr, rf_tpr, auc_thresholds = roc_curve(y_test, rf_y_score)
roc_auc_rf = auc(rf_fpr, rf_tpr)
print('Logistic Regression test score: \n Accuracy: %.4f \n Precision: %.4f \n Recall: %.4f \n F1: %.4f \n ROC: %.4f' %
        (lr_ac, 
         lr_precision, 
         lr_recall,
         lr_f1,
         roc_auc_lr)
     )

print('Random Forest test score: \n Accuracy: %.4f \n Precision: %.4f \n Recall: %.4f \n F1: %.4f \n ROC: %.4f' %
        (rf_ac, 
         rf_precision, 
         rf_recall,
         rf_f1,
         roc_auc_rf)
     )<jupyter_output>Logistic Regression test score: 
 Accuracy: 0.7101 
 Precision: 0.7175 
 Recall: 0.8423 
 F1: 0.7749 
 ROC: 0.5029
Random Forest test score: 
 Accuracy: 0.7276 
 Precision: 0.7499 
 Recall: 0.8105 
 F1: 0.7790 
 ROC: 0.7890
{'fraction_non_alphanumeric': 0.11440501043841336, 'fraction_numerical': 0.03507306889352818, 'mean_word_length': 5.303947368421053, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '602944', 'license': 'MIT', 'n_tokens_mistral': 2259, 'n_tokens_neox': 2105, 'n_words': 363, 'path': 'Workflow/6_Model Hyperparameters Tuning.ipynb', 'repo_name': 'crystal-ctrl/classification_project', 'size': '4790'}