<jupyter_start><jupyter_text>Extra Trees Classifier<jupyter_code>## For data handling
import pandas as pd
import numpy as np

## For plotting
import matplotlib.pyplot as plt
import seaborn as sns

## This sets the plot style
## to have a grid on a white background
sns.set_style("darkgrid")
## Import packages
## tree will be used to plot the decisiton tree
from sklearn import tree

## This is the actual out of the box algorithm
from sklearn.ensemble import ExtraTreesClassifier
from sklearn.model_selection import train_test_split, StratifiedKFold
from sklearn.metrics import roc_auc_score,accuracy_score
#import cleaned CSV (no date information)
cmm_pa_clf_read = pd.read_csv("../Data/cmm_pa_clf.csv",index_col = 0)

cmm_pa_clf_data = cmm_pa_clf_read.drop(columns = 'pa_approved').copy()
cmm_pa_clf_target = cmm_pa_clf_read['pa_approved'].copy()
X_train,X_test,y_train,y_test= train_test_split(cmm_pa_clf_data, cmm_pa_clf_target, test_size = 0.2, 
                                             random_state = 10475, shuffle = True,
                                            stratify = cmm_pa_clf_target)

#kfold cross validation for roc_auc, accuracy
kfold = StratifiedKFold(5, shuffle=True, random_state=10475)<jupyter_output><empty_output><jupyter_text>Depths analysisIn what follows we find a good max_depth for extra trees classifier, when the number of estimators is 100.<jupyter_code>depth = range(1,11)
estimators = 100
## Store accuracy and roc_auc scores
cv_accs = np.zeros((5, len(depth)))
cv_aucs = np.zeros((5, len(depth)))


i = 0
for train_index, test_index in  kfold.split(X_train, y_train):
    X_train_train = X_train.iloc[train_index]
    X_holdout = X_train.iloc[test_index]
    y_train_train  =  y_train.iloc[train_index]
    y_holdout = y_train.iloc[test_index]
    
    j = 0
    for d in depth: ## Try different depths
        extra_tree = ExtraTreesClassifier(n_estimators=estimators, max_depth=d, random_state=10475)
        extra_tree.fit(X_train_train, y_train_train)
        
        cv_accs[i,j] = accuracy_score(y_holdout, extra_tree.predict(X_holdout))
        cv_aucs[i,j] = roc_auc_score(y_holdout, extra_tree.predict_proba(X_holdout)[:,1])
        
        j = j + 1
    i = i + 1
## Average accuracy and roc_auc scores
acc_mean = cv_accs.mean(0)
auc_mean = cv_aucs.mean(0)

## This prints the mean of accuracy and roc_auc scores in the cross-validation.
print(acc_mean)
print(auc_mean)

## The following prints out the max_depth with best accuracy/roc_auc score
print("The max_depth with best accuracy score is", np.argmax(acc_mean))
print("The max_depth with best roc_auc score is", np.argmax(auc_mean))
#Plot the accuracy/auc vs depth
plt.figure(figsize=(10,10));
plt.plot(depth,acc_mean,'b')
plt.plot(depth,auc_mean,'r')
plt.legend(["Accuracy","AUC score"])
plt.xlabel("max_depth",fontsize=18)
plt.ylabel("Accuracy/AUC",fontsize=18)
plt.show()<jupyter_output><empty_output><jupyter_text>Estimators analysisWe analyze the behavior of extra trees classifier with different number of estimators.<jupyter_code>depth = 3
## We will try different number of estimators ranging from 1 to 100.
estimators = range(1,151)
## Store accuracy and roc_auc scores
cv_accs = np.zeros((5, len(estimators)))
cv_aucs = np.zeros((5, len(estimators)))

i = 0
for train_index, test_index in  kfold.split(X_train, y_train):
    X_train_train = X_train.iloc[train_index]
    X_holdout = X_train.iloc[test_index]
    y_train_train  =  y_train.iloc[train_index]
    y_holdout = y_train.iloc[test_index]
    
    j = 0 
    for estimator in estimators: ## different number of estimators
        extra_tree = ExtraTreesClassifier(n_estimators=estimator, max_depth=depth, random_state=10475)
        extra_tree.fit(X_train_train, y_train_train)
        
        cv_accs[i,j] = accuracy_score(y_holdout, extra_tree.predict(X_holdout))
        cv_aucs[i,j] = roc_auc_score(y_holdout, extra_tree.predict_proba(X_holdout)[:,1])
        
        j = j + 1
    i = i + 1
## This computes the mean of accuracy and roc_auc scores in the cross-validation.
acc_mean = np.mean(cv_accs,axis=0)
auc_mean = np.mean(cv_aucs,axis=0)

#Plot the accuracy/auc vs number of esimators
plt.figure(figsize=(18,10));
plt.plot(estimators,acc_mean,'b')
plt.plot(estimators,auc_mean,'r')
plt.legend(["Accuracy","AUC score"])
plt.xlabel("number of estimators",fontsize=18)
plt.ylabel("Accuracy/AUC",fontsize=18)
plt.show()

print("The number of estimators achieving best accuracy score is ", np.argmax(acc_mean)+1)
print("The number of estimators achieving best roc_auc score is ", np.argmax(auc_mean)+1)<jupyter_output><empty_output><jupyter_text>Feature importances<jupyter_code>## We fit the extra trees model with number of estimators being 48, and max_depth being 6.
def column(matrix, i):
    return [row[i] for row in matrix]

extra_tree=ExtraTreesClassifier(n_estimators=50,max_depth=9,random_state=10475)
fig=extra_tree.fit(X_train, y_train)
acc_score = accuracy_score(y_train, extra_tree.predict(X_train))
y_pred = column(extra_tree.predict_proba(X_train),1)
auc_score = roc_auc_score(y_train, y_pred)
print(acc_score)
print(auc_score)
## The dataframe storing the feature importances.
score_df = pd.DataFrame({'features': X_train.columns,
                         'feature_importances': extra_tree.feature_importances_})
score_df.sort_values('feature_importances', ascending=False)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.10264471980765674, 'fraction_numerical': 0.01664508969853893, 'mean_word_length': 4.975690607734807, 'pattern_counts': {'":': 0, '<': 17, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '390788', 'license': 'MIT', 'n_tokens_mistral': 2004, 'n_tokens_neox': 1894, 'n_words': 436, 'path': 'Models/Extra Trees.ipynb', 'repo_name': 'huszara2/CoverMyMeds', 'size': '5407'}