<jupyter_start><jupyter_text>Feature Selection  Import required libraries<jupyter_code>import sys, os
import itertools
import pandas as pd
import numpy as np

from sklearn.feature_selection import RFECV, SelectFromModel, f_classif, chi2
from sklearn.ensemble import RandomForestClassifier
from sklearn.preprocessing import StandardScaler
from sklearn.model_selection import GridSearchCV, StratifiedKFold
from sklearn.metrics import roc_curve, auc, confusion_matrix
from sklearn.utils import resample

%matplotlib inline
import seaborn as sns
import matplotlib.pyplot as plt
from mpl_toolkits.mplot3d import Axes3D
def progressBar(value, endvalue, bar_length=50):
    percent = float(value) / endvalue
    arrow = '=' * int(round(percent * bar_length)-1) + '>'
    spaces = ' ' * (bar_length - len(arrow))

    sys.stdout.write("\r[{0}] {1}%".format(arrow + spaces, int(round(percent * 100))))<jupyter_output><empty_output><jupyter_text>Import Data<jupyter_code>data = pd.read_csv(os.path.join(os.path.abspath(os.pardir), 'src/dataset', 'data_101718.csv'), low_memory=False)
data.head()
data.shape<jupyter_output><empty_output><jupyter_text>Data Cleaning  Let's take a look at the missing data, and try to figure our the best way to clean up our data. Drop out the columns that have over 90% missing data points.<jupyter_code>df = data.dropna(thresh=len(data)*.90, axis=1)<jupyter_output><empty_output><jupyter_text>Drop out any rows that are missing more than 50% of the required columns<jupyter_code>df = df.dropna(thresh=df.shape[1]*.5)<jupyter_output><empty_output><jupyter_text>Drop out any rows that are missing the target value<jupyter_code>df = df.dropna(axis=1, subset=df['ca'])<jupyter_output><empty_output><jupyter_text>Fill the rest of missing entries with the mean of each col<jupyter_code>df = df.apply(lambda col: col.fillna(col.mean()))
df.isnull().sum().any()
features = list(df.columns.values)
features.remove('pid')
features.remove('ca')
X = df[features]
y = df.ca.values<jupyter_output><empty_output><jupyter_text>Recursive Feature Elimination Find the optimal number of features<jupyter_code>rfc=RandomForestClassifier(random_state=2018)

selector = RFECV(
    estimator=rfc, cv=StratifiedKFold(5),
    scoring='roc_auc', verbose=1, n_jobs=-1
).fit(X, y)
plt.figure(figsize=(10,6))
plt.plot(range(1, len(selector.grid_scores_) + 1), selector.grid_scores_)
plt.xlabel('Number of features', fontsize=18)
plt.ylabel('Cross validation AUC score', fontsize=18)
plt.xticks(fontsize=14)
plt.yticks(fontsize=14)
plt.title('Optimal Number of Features: {}'.format(selector.n_features_), fontsize=20)
plt.show()
ranks = pd.DataFrame(list(zip(features, selector.ranking_)),columns=['Feature', 'Rank'])
ranks = ranks.sort_values(by='Rank')
ranks.head(60)<jupyter_output><empty_output><jupyter_text>Benchmark Let's balance out the pos and neg samples in the dataset<jupyter_code>df_neg = df[df.ca==0]
df_pos = df[df.ca==1]<jupyter_output><empty_output><jupyter_text>Upsample the pos samples<jupyter_code>df_pos_upsampled = resample(
    df_pos,
    n_samples=len(df_neg),
    replace=True, 
    random_state=2018
)
df_upsampled = pd.concat([df_pos_upsampled, df_neg])<jupyter_output><empty_output><jupyter_text>Downsample the neg samples<jupyter_code>df_neg_downsampled = resample(
    df_neg,
    n_samples=len(df_pos),
    replace=True, 
    random_state=2018
)
df_downsampled = pd.concat([df_pos, df_neg_downsampled])
fig, axes = plt.subplots(figsize=(15,5), nrows=1, ncols=3)
df.ca.value_counts().plot.bar(ax=axes[0], rot=False, title='Original Distribution', color=['g','r'])
df_upsampled.ca.value_counts().plot.bar(ax=axes[1], rot=False, title='w/ Upsampling', color=['r','g'])
df_downsampled.ca.value_counts().plot.bar(ax=axes[2], rot=False, title='w/ Downsampling', color=['r','g'])<jupyter_output><empty_output><jupyter_text>Setup Classifier Normalize Data<jupyter_code>selected_features = ranks.Feature[ranks.Rank == 1].values

# original distribution
#X = StandardScaler().fit_transform(df[selected_features])
#y = df.ca.values

# w/upsampling
#X = StandardScaler().fit_transform(df_upsampled[selected_features])
#y = df_upsampled.ca.values

# w/downsampling
X = StandardScaler().fit_transform(df_downsampled[selected_features])
y = df_downsampled.ca.values<jupyter_output><empty_output><jupyter_text>Specify parameters to test<jupyter_code>rfc=RandomForestClassifier(random_state=2018)
param_grid = { 
    'n_estimators': [100, 300, 500],
    'max_features': ['auto', 'sqrt'],
    'criterion'   : ['gini', 'entropy'],
    'class_weight': ['balanced', 'balanced_subsample'],
}<jupyter_output><empty_output><jupyter_text>Find the best parameters with GridSearchCV for (parallel 5 folds cross-validation)<jupyter_code>%%time
optimizer = GridSearchCV(
    rfc, param_grid=param_grid, cv=5, 
    scoring='roc_auc', verbose=10, n_jobs=-1
).fit(X, y)
optimizer.best_params_<jupyter_output><empty_output><jupyter_text>Run classifier with cross-validation and claculate the accuracy of the model based on True-Positive and False-Positive predictions<jupyter_code>n_splits = 10
cv = StratifiedKFold(n_splits=n_splits, shuffle=True, random_state=2018).split(X, y)
classifier = RandomForestClassifier(**optimizer.best_params_)

tprs, aucs = [], []
mean_fpr = np.linspace(0, 1, 100)
plt.figure(figsize=(10,8))

for itr, (train, test) in enumerate(cv):
    progressBar(itr+1, n_splits) 
    
    # get predictions
    probs = classifier.fit(X[train], y[train]).predict_proba(X[test])
    
    # compute AUC
    fpr, tpr, thresholds = roc_curve(y[test], probs[:, 1])
    tprs.append(np.interp(mean_fpr, fpr, tpr))
    tprs[-1][0] = 0.0
    roc_auc = auc(fpr, tpr)
    aucs.append(roc_auc)
    plt.plot(
        fpr, tpr, lw=1, alpha=0.5,
        label='ROC fold %d (AUC = %0.2f)' % (itr+1, roc_auc)
    )

# plot avg AUC    
mean_tpr = np.mean(tprs, axis=0)
mean_tpr[-1] = 1.0
mean_auc = auc(mean_fpr, mean_tpr)
std_auc = np.std(aucs)
plt.plot(
    mean_fpr, mean_tpr, color='k',
    label=r'Mean ROC (AUC = %0.2f $\pm$ %0.2f)' % (mean_auc, std_auc)
)

# plot std
std_tpr = np.std(tprs, axis=0)
tprs_upper = np.minimum(mean_tpr + std_tpr, 1)
tprs_lower = np.maximum(mean_tpr - std_tpr, 0)
plt.fill_between(
    mean_fpr, tprs_lower, tprs_upper,
    color='grey', alpha=.2, label=r'$\pm$ std.'
)

plt.xlim([-0.05, 1.05])
plt.ylim([-0.05, 1.05])
plt.xlabel('False Positive Rate', fontsize=18)
plt.ylabel('True Positive Rate', fontsize=18)
plt.xticks(fontsize=14)
plt.yticks(fontsize=14)
plt.title('ROC Curve', fontsize=20)
plt.legend(loc="lower right", fontsize=14)
plt.show()
n_splits = 4
cv = StratifiedKFold(n_splits=n_splits, shuffle=True, random_state=2018).split(X, y)
classifier = RandomForestClassifier(**optimizer.best_params_)
classes = df.ca.unique()

fig = plt.figure(figsize=(10,8))
for itr, (train, test) in enumerate(cv):
    progressBar(itr+1, n_splits) 
        
    # get predictions
    y_pred = classifier.fit(X[train], y[train]).predict(X[test])
    
    # compute confusion matrix
    matrix = confusion_matrix(y[test], y_pred)
    
    # plot confusion matrix
    ax = fig.add_subplot(2,2,itr+1)
    ax.imshow(matrix, interpolation='nearest', cmap=plt.cm.GnBu)
    
    for i, j in itertools.product(range(matrix.shape[0]), range(matrix.shape[1])):
        plt.text(
            j, i, format(matrix[i, j], 'd'),
            horizontalalignment="center",
            fontsize=14, color="white" if matrix[i, j] > matrix.max()/2 else "black"
        )
    
    tick_marks = np.arange(len(classes))
    plt.xticks(tick_marks, classes, rotation=45, fontsize=14)
    plt.yticks(tick_marks, classes, fontsize=14)
    plt.ylabel('True label', fontsize=18)
    plt.xlabel('Predicted label', fontsize=18)
    plt.title('Fold %d' % (itr+1), fontsize=20)

plt.tight_layout()<jupyter_output>[=================================================>] 100%
{'fraction_non_alphanumeric': 0.14951011579081308, 'fraction_numerical': 0.023285405267845782, 'mean_word_length': 5.787564766839378, 'pattern_counts': {'":': 0, '<': 56, '<?xml version=': 0, '>': 59, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '14', 'dir': 'jupyter-structured-clean-dedup', 'id': '111947', 'license': 'MIT', 'n_tokens_mistral': 3101, 'n_tokens_neox': 2931, 'n_words': 554, 'path': 'src/notebooks/recursive_feature_elimination.ipynb', 'repo_name': 'thayerAlshaabi/radiomics', 'size': '7859'}