<jupyter_start><jupyter_text>Training Sample: train.csv Evaluation Sample: validation_under.csv Method: OOB Output: Best hyperparameters; Pr-curve; ROC AUC  Training Part<jupyter_code>import numpy as np
import pandas as pd
from sklearn.ensemble import RandomForestClassifier
from sklearn.pipeline import Pipeline
from sklearn.model_selection import GridSearchCV
from sklearn.metrics import make_scorer,classification_report, matthews_corrcoef, accuracy_score, average_precision_score, roc_auc_score<jupyter_output><empty_output><jupyter_text>Input data is read and named as the following<jupyter_code>transactions = pd.read_csv('../Data/train.csv')
X_train = transactions.drop(labels='Class', axis=1)
y_train = transactions.loc[:,'Class']<jupyter_output><empty_output><jupyter_text>Tuning parameters<jupyter_code>test = 0
rf = RandomForestClassifier(n_jobs=-1, random_state=1)

if test== 0:
    n_estimators = [75,150,800,1000,1200]
    min_samples_split = [2, 5]
    min_samples_leaf = [1, 5]
else:
    n_estimators = [70]
    min_samples_split = [2]
    min_samples_leaf = [1]

param_grid_rf = {'n_estimators': n_estimators,
                 'min_samples_split': min_samples_split,
                 'min_samples_leaf': min_samples_split,
                 'oob_score': [True]
                }
grid_rf = GridSearchCV(estimator=rf, param_grid=param_grid_rf,cv = 2,
                       n_jobs=-1, pre_dispatch='2*n_jobs', verbose=1, return_train_score=False)
grid_rf.fit(X_train, y_train)<jupyter_output>Fitting 2 folds for each of 20 candidates, totalling 40 fits<jupyter_text>The best score and the estimator<jupyter_code>grid_rf.best_score_
grid_rf.best_params_<jupyter_output><empty_output><jupyter_text>Evaluation Part<jupyter_code>evaluation = pd.read_csv('../Data/validation_under.csv')
X_eval = evaluation.drop(labels='Class', axis=1)
y_eval = evaluation.loc[:,'Class']
def Random_Forest_eval(estimator, X_test, y_test):
    
    y_pred = estimator.predict(X_test)

    print('Classification Report')
    print(classification_report(y_test, y_pred))
    if y_test.nunique() <= 2:
        try:
            y_score = estimator.predict_proba(X_test)[:,1]
        except:
            y_score = estimator.decision_function(X_test)
        print('AUPRC', average_precision_score(y_test, y_score))
        print('AUROC', roc_auc_score(y_test, y_score))
    
Random_Forest_eval(grid_rf, X_eval, y_eval)<jupyter_output>Classification Report
              precision    recall  f1-score   support

           0       0.82      1.00      0.90       173
           1       1.00      0.77      0.87       173

    accuracy                           0.89       346
   macro avg       0.91      0.89      0.89       346
weighted avg       0.91      0.89      0.89       346

AUPRC 0.9553261695540434
AUROC 0.9536569882054194<jupyter_text>Precision Recall Curve<jupyter_code>from sklearn.metrics import precision_recall_curve
precision_rf,recall_rf,thresholds_rf = precision_recall_curve(
    y_eval, grid_rf.predict_proba(X_eval)[:,1])
import matplotlib.pyplot as plt
close_default_rf = np.argmin(np.abs(thresholds_rf * 0.5))
plt.plot(precision_rf,recall_rf,label="rf")
plt.plot(precision_rf[close_default_rf],recall_rf[close_default_rf],'^',c='k',
        label = "threashold 0.5 rf")
plt.xlabel("precision")
plt.ylabel("recall")
plt.legend(loc="best")
plt.title("PR_Curve")<jupyter_output><empty_output><jupyter_text>Area Under the Receiver Operating Characteristic Curve<jupyter_code>from sklearn.metrics import roc_curve
fpr_rf,tpr_rf,th_rf = roc_curve(
    y_eval, grid_rf.predict_proba(X_eval)[:,1])
close_default_rf = np.argmin(np.abs(th_rf * 0.5))
plt.plot(fpr_rf,tpr_rf,label="rf")
plt.plot(fpr_rf[close_default_rf],tpr_rf[close_default_rf],'^',c='k',
        label = "threashold 0.5 rf")
plt.xlabel("False Positive Rate")
plt.ylabel("True Positive Rate")
plt.legend(loc="best")
plt.title("ROC")<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.109979633401222, 'fraction_numerical': 0.03615071283095723, 'mean_word_length': 4.316644113667118, 'pattern_counts': {'":': 0, '<': 28, '<?xml version=': 0, '>': 27, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '7', 'dir': 'jupyter-structured-clean-dedup', 'id': '491252', 'license': 'MIT', 'n_tokens_mistral': 1526, 'n_tokens_neox': 1401, 'n_words': 240, 'path': 'Random_Forest/.ipynb_checkpoints/RF_new_data-checkpoint.ipynb', 'repo_name': 'DeviRule/Ecs171_group-project', 'size': '3928'}