<jupyter_start><jupyter_text>TO LOAD DATA<jupyter_code># TO IMPORT CVS FILES (REGARDING FREQUENCY OF FEATURES)
all_features_list_df_lin = pd.read_csv("training_linear_all_features_list_result.csv",index_col=False)
all_features_count_df_lin = all_features_list_df_lin.stack().value_counts() # it returns a df with the frequency for each features
# TO IMPORT CVS FILE AND CREATE A PD DATAFRAME WITH ONLY THE FIRST n SELECTED FEATURES - TRAINING DATASET

first_n_features_to_select_lin = 5 # choose the value

# load the original dataset
training_dataframe_df_lin = pd.read_csv("training - linear after WEKA CfsSubsetEval.csv",index_col='exam')
size_mapping = {"codeletion":0,"noncodeletion":1} # MAPPING for outcome 
training_dataframe_df_lin["outcome"] = training_dataframe_df_lin["outcome"].map(size_mapping)

training_feature_names_lin = [x[2:-2] for x in [*all_features_count_df_lin.index]]
training_selected_features_lin = training_feature_names_lin[:first_n_features_to_select_lin]
training_New_dataframe_lin = training_dataframe_df_lin[training_selected_features_lin]
training_New_dataframe_lin["outcome"] = training_dataframe_df_lin["outcome"] 
training_dataframe_with_selected_features_df_lin = training_New_dataframe_lin
# TO IMPORT CVS FILE AND CREATE A PD DATAFRAME WITH ONLY THE FIRST n SELECTED FEATURES - TESTING DATASET

first_n_features_to_select_lin = 5 # choose the value

# load the original dataset
testing_dataframe_df_lin = pd.read_csv("testing - linear.csv",index_col='exam', encoding = "ISO-8859-1") # insert the all original dataset
size_mapping = {"codeletion":0,"noncodeletion":1} # MAPPING for outcome 
testing_dataframe_df_lin["outcome"] = testing_dataframe_df_lin["outcome"].map(size_mapping)

testing_feature_names_lin = [x[3:-3] for x in [*all_features_count_df_lin.index]]
testing_selected_features_lin = testing_feature_names_lin[:first_n_features_to_select_lin]
testing_New_dataframe_lin = testing_dataframe_df_lin[testing_selected_features_lin]
testing_New_dataframe_lin["outcome"] = testing_dataframe_df_lin["outcome"]
testing_dataframe_with_selected_features_df_lin = testing_New_dataframe_lin
print ("The chosen features are:", [x[1:-1] for x in [*training_selected_features_lin]])<jupyter_output><empty_output><jupyter_text>Training the model on the training dataset and testing the model on validation dataset<jupyter_code>model_lin = RandomForestClassifier(random_state=1, n_estimators=100) # Choose the model
# To rename dataframes into X_train_lin, Y_train_lin, X_test_lin, Y_test_lin (numpy arrays)

Y_train_lin = training_dataframe_with_selected_features_df_lin['outcome']
X_train_lin = training_dataframe_with_selected_features_df_lin.drop('outcome',axis=1)

Y_test_lin = testing_dataframe_with_selected_features_df_lin['outcome']
X_test_lin = testing_dataframe_with_selected_features_df_lin.drop('outcome',axis=1)
#StandardScaler
ss = StandardScaler() 
X_train_SS_np_lin = ss.fit_transform(X_train_lin) 
X_train_SS_lin=pd.DataFrame(X_train_SS_np_lin, index=X_train_lin.index, columns=X_train_lin.columns)
X_test_SS_np_lin = ss.transform(X_test_lin) 
X_test_SS_lin=pd.DataFrame(X_test_SS_np_lin, index=X_test_lin.index, columns=X_test_lin.columns)

# ADASYN
sm = ADASYN(random_state=1)
X_train_SS_balanced_np_lin, Y_train_balanced_np_lin = sm.fit_sample(X_train_SS_lin, Y_train_lin)
X_train_SS_balanced_lin=pd.DataFrame(X_train_SS_balanced_np_lin, columns=X_train_SS_lin.columns)
Y_train_balanced_lin=pd.DataFrame(Y_train_balanced_np_lin, columns=["outcome"])

# Fitting the model
model_lin.fit (X_train_SS_balanced_lin, Y_train_balanced_lin)

# Compute predictions, probabilities and accuracy
predictions_lin = model_lin.predict(X_test_SS_lin)
probabilities_lin = model_lin.predict_proba(X_test_SS_lin)
accuracy_lin = accuracy_score(Y_test_lin, predictions_lin)

# Compute AUC
fpr_lin, tpr_lin, threshold_lin = roc_curve(Y_test_lin, np.array(probabilities_lin)[:,1])
roc_auc_lin = auc(fpr_lin, tpr_lin)
# Rename the values for bootstrap code and De-Long test
y_true_lin = np.array(Y_test_lin)
y_pred_lin = np.array(predictions_lin)
y_prob_lin = np.array(probabilities_lin)[:,1]
# print Confusion Matrix
print ("Confusion matrix for linear features: \n", confusion_matrix(y_true_lin, y_pred_lin))
# Perform BOOTSTRAP with y_true, predictions, probabilities

n_bootstraps = 10000
rng_seed = 1  # control reproducibility

bootstrapped_acc_lin = []
bootstrapped_auc_lin = []
bootstrapped_sens_lin = []
bootstrapped_spec_lin = []

bootstrapped_tpr_lin = []
bootstrapped_fpr_lin = []
bootstrapped_thr_lin = []

bootstrapped_tprs_lin = []

mean_fpr = np.linspace(0, 1, 100)

rng = np.random.RandomState(rng_seed)

for i in range(n_bootstraps):
    # bootstrap by sampling with replacement on the prediction indices
    indices_0=np.where(y_true_lin == 0)
    indices_1=np.where(y_true_lin == 1)
    
    # 'balanced bootstrapping'
    random_indices_0=rng.choice(indices_0[0],len(indices_0[0]))
    random_indices_1=rng.choice(indices_1[0],len(indices_0[0]))
    random_indices=np.concatenate((random_indices_0,random_indices_1), axis=None)

    acc_lin = accuracy_score(y_true_lin[random_indices], y_pred_lin[random_indices])
    auc_lin = roc_auc_score(y_true_lin[random_indices], y_prob_lin[random_indices])
    sens_lin = recall_score(y_true_lin[random_indices], y_pred_lin[random_indices], pos_label=1)
    spec_lin = recall_score(y_true_lin[random_indices], y_pred_lin[random_indices], pos_label=0)
    
    fpr_lin, tpr_lin, threshold_lin = roc_curve(y_true_lin[random_indices], y_prob_lin[random_indices])
    
    interp_tpr_lin = interp(mean_fpr, fpr_lin, tpr_lin)
    interp_tpr_lin[0] = 0.0
    bootstrapped_tprs_lin.append(interp_tpr_lin)
    
    bootstrapped_acc_lin.append(acc_lin)
    bootstrapped_auc_lin.append(auc_lin)
    bootstrapped_sens_lin.append(sens_lin)
    bootstrapped_spec_lin.append(spec_lin)<jupyter_output><empty_output><jupyter_text>metrics distributions for bootstrapping steps<jupyter_code>plt.figure(figsize=(10, 15))

plt.subplot(2,2,1)
plt.hist(bootstrapped_acc_lin)
plt.title('Acc lin')

plt.subplot(2,2,2)
plt.hist(bootstrapped_auc_lin)
plt.title('AUC lin')

plt.subplot(2,2,3)
plt.hist(bootstrapped_sens_lin)
plt.title('Sens lin')

plt.subplot(2,2,4)
plt.hist(bootstrapped_spec_lin)
plt.title('Spec lin')

plt.show()<jupyter_output><empty_output><jupyter_text>distr normality test (Shapiro-Wilcoxon)<jupyter_code>print ('Acc lin: ', stats.shapiro(bootstrapped_acc_lin))
print ('AUC lin: ', stats.shapiro(bootstrapped_auc_lin))
print ('Sens lin: ', stats.shapiro(bootstrapped_sens_lin))
print ('Spec lin: ', stats.shapiro(bootstrapped_spec_lin))<jupyter_output><empty_output><jupyter_text>p-values are small -> distr is not normal -> estimation should be represented as median (low_percentile, up_percentile)<jupyter_code>print ('Acc lin: {} ({}, {})'.format(np.median(bootstrapped_acc_lin), np.percentile(bootstrapped_acc_lin, 2.5), np.percentile(bootstrapped_acc_lin, 97.5)))
print ('AUC lin: {} ({}, {})'.format(np.median(bootstrapped_auc_lin), np.percentile(bootstrapped_auc_lin, 2.5), np.percentile(bootstrapped_auc_lin, 97.5)))
print ('Sens lin: {} ({}, {})'.format(np.median(bootstrapped_sens_lin), np.percentile(bootstrapped_sens_lin, 2.5), np.percentile(bootstrapped_sens_lin, 97.5)))
print ('Spec lin: {} ({}, {})'.format(np.median(bootstrapped_spec_lin), np.percentile(bootstrapped_spec_lin, 2.5), np.percentile(bootstrapped_spec_lin, 97.5)))<jupyter_output><empty_output><jupyter_text>ROC CURVE AND AUC<jupyter_code># ROC CURVE

fig, ax = plt.subplots(figsize=(10,10))  

plt.title('ROC Validation dataset')

plt.plot([0, 1], [0, 1], linestyle='--', lw=2, color='r', alpha=.8)

mean_tpr_lin = np.median(bootstrapped_tprs_lin, axis=0)  
mean_tpr_lin[-1] = 1.0                
plt.plot(mean_fpr, mean_tpr_lin, color='b', 
        label=r'Median ROC (AUC = %0.2f)' % (np.median(bootstrapped_auc_lin)), 
        lw=2, alpha=.8)

tprs_upper = np.percentile(bootstrapped_tprs_lin, 2.5,  axis = 0)
tprs_lower = np.percentile(bootstrapped_tprs_lin, 97.5, axis = 0)
plt.fill_between(mean_fpr, tprs_lower, tprs_upper, color='grey', alpha=.2, label='95 % CI')

plt.ylabel('True Positive Rate')
plt.xlabel('False Positive Rate')

plt.xlim([0, 1])
plt.ylim([0, 1])

plt.legend(loc="lower right")
plt.show()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.11230010821209571, 'fraction_numerical': 0.015149693399062162, 'mean_word_length': 8.12061403508772, 'pattern_counts': {'":': 4, '<': 25, '<?xml version=': 0, '>': 27, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '6', 'dir': 'jupyter-structured-clean-dedup', 'id': '109922', 'license': 'MIT', 'n_tokens_mistral': 3263, 'n_tokens_neox': 3110, 'n_words': 490, 'path': 'linear interpolation/4 - model training and testing (testing dataset) - linear features.ipynb', 'repo_name': 'roberto-casale/LGG-1p-19q-deletion', 'size': '8317'}