<jupyter_start><jupyter_text>RandomForest<jupyter_code>steps = [('scaler', StandardScaler()),(('rf', RandomForestClassifier(n_estimators=200, max_features=8, max_depth=12)))]
pipeline = Pipeline(steps)
# For RF is not a good option use scaler
random_scaled = pipeline.fit(X_train, y_train)
# y_pred = pipeline.predict(X_test) 
# accuracy_score(y_test, y_pred)
# y_pred_prob = pipeline.predict_proba(X_test)[:,1]
# plotRoc(y_test, y_pred_prob)
# printcfm(y_test, y_pred,title='confusion matrix')<jupyter_output><empty_output><jupyter_text>Positive Predictive Value (PPV)$$Precision=\frac{TP}{TP+FP}$$Sensitivity, Hit Rate, True Positive Rate$$Recall=\frac{TP}{TP+FN}$$Harmonic mean between Precision and Recall$$F1 Score=2 * \frac{Precision * Recall}{Precision + Recall}$$<jupyter_code># print(classification_report(y_test, y_pred))<jupyter_output><empty_output><jupyter_text>Fine-tunning the model. To turn on Fine-tunning:     define ft = 1<jupyter_code>ft = 0<jupyter_output><empty_output><jupyter_text>1 - Grid Search<jupyter_code>if ft == 1 :
    rf = RandomForestClassifier(n_jobs=-1, random_state=42)
    parameters = {'n_estimators' : [400, 500, 600],
                 'min_samples_split': np.arange(2, 5),
                 'max_features': ['auto', 'sqrt', 'log2'],
                 'max_depth' : [4,7,15],
                 'bootstrap': [True,False],
                 'warm_start': [True,False],
                 'criterion' :['entropy']
                 }
    cv = GridSearchCV(rf, param_grid=parameters, verbose=3, n_jobs=-1, cv=5)
                  #"max_depth": np.arange(1, 50),
                  #"max_features": [1, 3, 10],

                  #"min_samples_leaf": np.arange(1, 10),

                  #"criterion": ["gini", "entropy"]
    cv.fit(X_train, y_train);<jupyter_output><empty_output><jupyter_text>[Parallel(n_jobs=-1)]: Done  24 tasks      | elapsed:   17.1s [Parallel(n_jobs=-1)]: Done 120 tasks      | elapsed:  1.3min[Parallel(n_jobs=-1)]: Done 280 tasks      | elapsed:  2.9min[Parallel(n_jobs=-1)]: Done 504 tasks      | elapsed:  5.2min[Parallel(n_jobs=-1)]: Done 792 tasks      | elapsed:  8.2min[Parallel(n_jobs=-1)]: Done 1144 tasks      | elapsed: 11.8min[Parallel(n_jobs=-1)]: Done 1560 tasks      | elapsed: 16.1min[Parallel(n_jobs=-1)]: Done 2040 tasks      | elapsed: 21.0min[Parallel(n_jobs=-1)]: Done 2584 tasks      | elapsed: 26.4min[Parallel(n_jobs=-1)]: Done 3192 tasks      | elapsed: 32.3min[Parallel(n_jobs=-1)]: Done 3864 tasks      | elapsed: 39.0min[Parallel(n_jobs=-1)]: Done 4600 tasks      | elapsed: 46.5min[Parallel(n_jobs=-1)]: Done 5400 out of 5400 | elapsed: 54.9min finished<jupyter_code>if ft == 1:
    print("Best params: ", cv.best_params_,) 
    print("Best Score: %3.3f" %(cv.best_score_))
    y_pred = cv.predict(X_train_scaled)
    final_model =cv.best_estimator_
    print(final_model)<jupyter_output><empty_output><jupyter_text>Best Model Result (11/2019)RandomForestClassifier(bootstrap=False, class_weight=None,            criterion='entropy', max_depth=100, max_features='auto',            max_leaf_nodes=None, min_impurity_decrease=0.0,            min_impurity_split=None, min_samples_leaf=1,            min_samples_split=2, min_weight_fraction_leaf=0.0,            n_estimators=1000, n_jobs=-1, oob_score=False, random_state=42,            verbose=0, warm_start=False)  Best Model Result (11/2018_v2)RandomForestClassifier(bootstrap=False, class_weight=None, criterion='entropy', max_depth=15, max_features='auto', max_leaf_nodes=None, min_impurity_decrease=0.0, min_impurity_split=None, min_samples_leaf=1, min_samples_split=3, min_weight_fraction_leaf=0.0, n_estimators=500, n_jobs=-1, oob_score=False, random_state=42, verbose=0, warm_start=True)  Best Model Result (11/2018)RandomForestClassifier(bootstrap=False, class_weight=None,            criterion='entropy', max_depth=15, max_features='auto',            max_leaf_nodes=None, min_impurity_decrease=0.0,            min_impurity_split=None, min_samples_leaf=1,            min_samples_split=3, min_weight_fraction_leaf=0.0,            n_estimators=500, n_jobs=-1, oob_score=False, random_state=42,            verbose=0, warm_start=True)  Best Model Result (09/2018)RandomForestClassifier(bootstrap=False, class_weight=None,            criterion='entropy', max_depth=7, max_features='auto',            max_leaf_nodes=None, min_impurity_decrease=0.0,            min_impurity_split=None, min_samples_leaf=1,            min_samples_split=2, min_weight_fraction_leaf=0.0,            n_estimators=525, n_jobs=-1, oob_score=False, random_state=42,            verbose=0, warm_start=True)  Regularizating the model   Fill max_depth value<jupyter_code>max_depth=5
final_model = RandomForestClassifier(bootstrap=False, class_weight=None,
            criterion='entropy', max_depth=max_depth, max_features='auto',
            max_leaf_nodes=None, min_impurity_decrease=0.0,
            min_impurity_split=None, min_samples_leaf=1,
            min_samples_split=2, min_weight_fraction_leaf=0.0,
            n_estimators=1000, n_jobs=-1, oob_score=False, random_state=42,
            verbose=0, warm_start=False)<jupyter_output><empty_output><jupyter_text>Predicting the Classes in Trainning Set<jupyter_code>final_model.fit(X_train, y_train)
y_pred = final_model.predict(X_train)
y_pred_prob = final_model.predict_proba(X_train)[:,1]
plotRoc(y_train, y_pred_prob)
auc_train = roc_auc_score(y_train, y_pred_prob)
cv_scores = cross_val_score(final_model, X_train, y_train, cv=5)
print(cv_scores)
printcfm(y_train, y_pred, title='confusion matrix')
print(classification_report(y_train, y_pred))<jupyter_output>precision    recall  f1-score   support

          0       0.96      0.92      0.94        60
          1       0.92      0.97      0.94        60

avg / total       0.94      0.94      0.94       120<jupyter_text>Evaluating the model with Cross-Validation<jupyter_code>y_pred_prob = final_model.predict_proba(X_train)[:,1]
y_scores = cross_val_predict(final_model, X_train, y_train, cv=3, verbose=3, method='predict_proba')
y_train_pred = cross_val_predict(final_model, X_train, y_train, cv=3, verbose=3)
# hack to work around issue #9589 in Scikit-Learn 0.19.0
if y_scores.ndim == 2:
    y_scores = y_scores[:, 1]
# print(y_scores)
# print(np.mean(y_scores))
plotRoc(y_train, y_scores)
auc_cv = roc_auc_score(y_train, y_scores)
printcfm(y_train, y_train_pred, title='confusion matrix')
print(classification_report(y_train, y_train_pred))<jupyter_output>precision    recall  f1-score   support

          0       0.96      0.82      0.88        60
          1       0.84      0.97      0.90        60

avg / total       0.90      0.89      0.89       120<jupyter_text>Evaluating the model with LOO<jupyter_code>loo = LeaveOneOut()
loo.get_n_splits(X_train)
for train, test in loo.split(X_train):
     print("%s %s" % (train, test))
cv=loo
#y_pred_prob = final_model.predict_proba(X_train)[:,1]
y_scores = cross_val_predict(final_model, X_train, y_train, cv=cv, verbose=10, method='predict_proba', n_jobs=-1)
y_train_pred = cross_val_predict(final_model, X_train, y_train, cv=cv, verbose=10)
# hack to work around issue #9589 in Scikit-Learn 0.19.0
if y_scores.ndim == 2:
    y_scores = y_scores[:, 1]
# print(y_scores)
# print(np.mean(y_scores))
plotRoc(y_train, y_scores)
auc_LoO = roc_auc_score(y_train, y_scores)
printcfm(y_train, y_train_pred, title='confusion matrix')
print(classification_report(y_train, y_train_pred))<jupyter_output>precision    recall  f1-score   support

          0       0.94      0.83      0.88        60
          1       0.85      0.95      0.90        60

avg / total       0.90      0.89      0.89       120<jupyter_text>Evaluating the model with Repeated K fold<jupyter_code>def perform_repeated_cv(X, y , model):
    #set random seed for repeatability
    random.seed(1)

    #set the number of repetitions
    n_reps = 45

    # perform repeated cross validation
    accuracy_scores = np.zeros(n_reps)
    precision_scores=  np.zeros(n_reps)
    recall_scores =  np.zeros(n_reps)
    auc_scores =  np.zeros(n_reps)
    #result_pred = pd.DataFrame(index=np.arange(30))
    result_pred = y
    
    ##############################
    tprs = []
    aucs = []
    mean_fpr = np.linspace(0, 1, 100)
    fig = plt.figure(figsize=(20, 10))
    ###############################

    for u in range(n_reps):

        #randomly shuffle the dataset
        indices = np.arange(X.shape[0])
        np.random.shuffle(indices)
#         X = X[indices]
#         y = y[indices] #dataset has been randomly shuffled
        X = X.iloc[indices]
        y = y.iloc[indices] #dataset has been randomly shuffled

        #initialize vector to keep predictions from all folds of the cross-validation
        y_predicted = np.zeros(y.shape)
        probas = np.zeros(y.shape)

        #perform 10-fold cross validation
        kf = KFold(n_splits=4 , random_state=142)
        for train, test in kf.split(X):

            #split the dataset into training and testing
#             X_train = X[train]
#             X_test = X[test]
#             y_train = y[train]
#             y_test = y[test]
            X_train = X.iloc[train]
            X_test = X.iloc[test]
            y_train = y.iloc[train]
            y_test = y.iloc[test]

#             #standardization
#             scaler = preprocessing.StandardScaler().fit(X_train)
#             X_train = scaler.transform(X_train)
#             X_test = scaler.transform(X_test)

            #train model
            clf = model
            clf.fit(X_train, y_train)

            #make predictions on the testing set
            y_predicted[test] = clf.predict(X_test)
#            print(y_predicted[test],y_test,type(y_predicted))
            #y_train_pred_array = np.append(y_train_pred_array,y_train_pred)
#            print(result_pred)



            
            ###############################plot
#             probas_ = clf.predict_proba(X_test)
            probas[test] = clf.predict_proba(X_test)[:, 1]
#             print(probas[test], type(probas), probas.size)
         
#        print(y,y_predicted)
        #result_pred = y
        df_pred = pd.DataFrame(y_predicted, index=y.index,columns=[u])
        result_pred = pd.concat([result_pred, df_pred], axis=1) 
        

                
    
        # Compute ROC curve and area the curve
        fpr, tpr, thresholds = roc_curve(y, probas)
        tprs.append(interp(mean_fpr, fpr, tpr))
        tprs[-1][0] = 0.0
        #roc_auc = auc(fpr, tpr) - Change to obtain AUC by predict proba
#06/11 - 23:26        roc_auc = roc_auc_score(y, y_predicted)
        roc_auc = roc_auc_score(y, probas)
        aucs.append(roc_auc)
        plt.plot(fpr, tpr, lw=1, alpha=0.3,
        label='ROC fold %d (AUC = %0.2f)' % (u, roc_auc))

            
            ################################

        #record scores
        accuracy_scores[u] = accuracy_score(y, y_predicted)
        precision_scores[u] = precision_score(y, y_predicted)
        recall_scores[u]  = recall_score(y, y_predicted)
#06/11 - 18:39        auc_scores[u] = roc_auc_score(y, y_predicted)
        auc_scores[u] = roc_auc_score(y, probas)

    ###############################plot
#    print(result_pred)
    
    plt.plot([0, 1], [0, 1], linestyle='--', lw=2, color='r',
    label='Chance', alpha=.8)

    mean_tpr = np.mean(tprs, axis=0)
    mean_tpr[-1] = 1.0
#     mean_auc = auc(mean_fpr, mean_tpr)
    mean_auc = np.mean(aucs)
    std_auc = np.std(aucs)
    plt.plot(mean_fpr, mean_tpr, color='b',
             label=r'Mean ROC (AUC = %0.2f $\pm$ %0.2f)' % (mean_auc, std_auc),
             lw=2, alpha=.8)


    std_tpr = np.std(tprs, axis=0)
    tprs_upper = np.minimum(mean_tpr + std_tpr, 1)
    tprs_lower = np.maximum(mean_tpr - std_tpr, 0)
    plt.fill_between(mean_fpr, tprs_lower, tprs_upper, color='grey', alpha=.2,
                     label=r'$\pm$ 1 std. dev.')

    plt.xlim([-0.05, 1.05])
    plt.ylim([-0.05, 1.05])
    plt.xlabel('False Positive Rate')
    plt.ylabel('True Positive Rate')
    plt.title('Receiver operating characteristic example')
    #plt.legend(loc="lower right")
    plt.legend(loc='upper center', bbox_to_anchor=(0.5, -0.05),
               fancybox=True, shadow=True, ncol=5)
    plt.show()
    ################################
    
    #return all scores
    return accuracy_scores, precision_scores, recall_scores, auc_scores, result_pred
# classifier = RandomForestClassifier(bootstrap=False, class_weight=None,
#             criterion='entropy', max_depth=6, max_features='auto',
#             max_leaf_nodes=None, min_impurity_decrease=0.0,
#             min_impurity_split=None, min_samples_leaf=1,
#             min_samples_split=2, min_weight_fraction_leaf=0.0,
#             n_estimators=600, n_jobs=-1, oob_score=False, random_state=42,
#             verbose=0, warm_start=False)
# # classifier = RandomForestClassifier(bootstrap=False, class_weight=None,
# #             criterion='entropy', max_depth=6, max_features='auto',
# #             max_leaf_nodes=None, min_impurity_decrease=0.0,
# #             min_impurity_split=None, min_samples_leaf=1,
# #             min_samples_split=2, min_weight_fraction_leaf=0.0,
# #             n_estimators=1000, n_jobs=-1, oob_score=False, random_state=42,
# #             verbose=0, warm_start=True)
accuracy_scores, precision_scores, recall_scores, auc_scores, result_pred = perform_repeated_cv(X_train, y_train, final_model)
print(accuracy_scores, accuracy_scores.size)
print(precision_scores, recall_scores)
print(auc_scores, auc_scores.size)
fig = plt.figure(figsize=(20, 10))
plt.plot(auc_scores, '--o')
plt.legend(loc='lower right')
plt.ylabel('AUC', fontsize=20);
plt.xlabel('Repetições', fontsize=20);
plt.tick_params(axis='both', which='major', labelsize=20);
plt.tick_params(axis='both', which='minor', labelsize=18);
#plt.xlim([0, 18])
#plt.ylim([0.5, 1])
plt.legend(('Acurácia', 'AUC'), loc='lower right', prop={'size': 20})
plt.show()
auc_scores.mean()
auc_scores.std()
print("AUC: %0.2f (+/- %0.2f)" % (np.mean(auc_scores), np.std(auc_scores)))
#result_pred.to_csv('result_kfold_RF.csv', encoding='utf-8')<jupyter_output><empty_output><jupyter_text>Predicting the Classes in Test Set<jupyter_code>final_model.fit(X_train, y_train)
y_pred = final_model.predict(X_test)
y_pred_prob = final_model.predict_proba(X_test)[:,1]
plotRoc(y_test, y_pred_prob)
auc_test = roc_auc_score(y_test, y_pred_prob)
printcfm(y_test, y_pred, title='confusion matrix')
print(classification_report(y_test, y_pred))<jupyter_output>precision    recall  f1-score   support

          0       0.82      0.72      0.77        25
          1       0.50      0.64      0.56        11

avg / total       0.72      0.69      0.70        36<jupyter_text>Varying the Threshold for test set<jupyter_code>predict_mine = np.where(y_pred_prob > .0, 1, 0)
printcfm(y_test, predict_mine, title='confusion matrix')
print(classification_report(y_test, predict_mine))<jupyter_output>precision    recall  f1-score   support

          0       0.00      0.00      0.00        25
          1       0.31      1.00      0.47        11

avg / total       0.09      0.31      0.14        36<jupyter_text>Results<jupyter_code>print("max_depth: ", max_depth)
print("AUC Train: %3.3f" % (auc_train))
print("AUC Repeated k-fold: %0.2f (+/- %0.2f)" % (np.mean(auc_scores), np.std(auc_scores)))
print("AUC LoO: %3.3f" % (auc_LoO))
print("AUC test: %3.3f" % (auc_test))
print("AUC cv: %3.3f" % (auc_cv))
#print("Accuracy Train: %3.2f%%" % (acc_train*100))
#print("Accuracy Test %3.2f%%" % (acc_test*100))<jupyter_output>max_depth:  5
AUC Train: 0.997
AUC Repeated k-fold: 0.94 (+/- 0.02)
AUC LoO: 0.960
AUC test: 0.847
AUC cv: 0.971<jupyter_text>Draft<jupyter_code># X=np.concatenate((X_train),axis=0)
# y=np.append(y_train)
X=X_train
y=y_train
#  validation curve off
vc = 0
if vc == 1:

    print(__doc__)

    param_range = np.arange(1, 800, 20)

    train_scores, test_scores = validation_curve(
        final_model, X, y, param_name="n_estimators", param_range=param_range,
        cv=10, scoring="roc_auc", n_jobs=-1)

    train_scores_mean = np.mean(train_scores, axis=1)
    train_scores_std = np.std(train_scores, axis=1)
    test_scores_mean = np.mean(test_scores, axis=1)
    test_scores_std = np.std(test_scores, axis=1)

    plt.title("Validation Curve with RF")
    plt.xlabel("$\gamma$")
    plt.ylabel("AUC")
    #plt.ylim(0.0, 1.1)
    #plt.xlim(-1, 22)
    lw = 2
    plt.semilogx(param_range, train_scores_mean, label="Training score",
                 color="darkorange", lw=lw)
    plt.fill_between(param_range, train_scores_mean - train_scores_std,
                     train_scores_mean + train_scores_std, alpha=0.2,
                     color="darkorange", lw=lw)
    plt.semilogx(param_range, test_scores_mean, label="Cross-validation score",
                 color="navy", lw=lw)
    plt.fill_between(param_range, test_scores_mean - test_scores_std,
                     test_scores_mean + test_scores_std, alpha=0.2,
                     color="navy", lw=lw)
    plt.legend(loc="best")
    plt.show()<jupyter_output><empty_output><jupyter_text>Export results<jupyter_code>export = 0
rf_df = pd.concat([X_test, y_test], axis=1)  # features and actual
rf_df['Predicted'] = y_pred  # creates a predicted column to the complete_df, now you'll have features, actual, and predicted
rf_df
if export == 1:
    rf_df.to_csv('rf_results.csv', encoding='utf-8')<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.13686799359414323, 'fraction_numerical': 0.039178677648135436, 'mean_word_length': 3.3354822712620877, 'pattern_counts': {'":': 4, '<': 55, '<?xml version=': 0, '>': 56, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 10, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '15', 'dir': 'jupyter-structured-clean-dedup', 'id': '321995', 'license': 'MIT', 'n_tokens_mistral': 6909, 'n_tokens_neox': 6333, 'n_words': 1133, 'path': 'Model-Study/mlModelsRandomForestKFoldLooRepeated.ipynb', 'repo_name': 'giovanimachado/masterOfScience', 'size': '17484'}