<jupyter_start><jupyter_text>Decision Tree (gini)<jupyter_code>res_Dtree = experiment(DecisionTreeClassifier(max_leaf_nodes=12), X, Y, 10)<jupyter_output>precision    recall  f1-score   support

        0.0       1.00      1.00      1.00      1000
        1.0       0.96      0.94      0.95        90

avg / total       0.99      0.99      0.99      1090<jupyter_text>K-Neirest Neighbors<jupyter_code>res_knn = experiment(neighbors.KNeighborsClassifier(n_neighbors=3, weights='uniform'), X, Y, 10)<jupyter_output>precision    recall  f1-score   support

        0.0       0.99      0.99      0.99      1000
        1.0       0.89      0.88      0.88        90

avg / total       0.98      0.98      0.98      1090<jupyter_text>Support Vector Machines<jupyter_code>res_svc = experiment(svm.SVC(kernel='linear', probability=True), X, Y, 10)<jupyter_output>precision    recall  f1-score   support

        0.0       0.99      0.98      0.98      1000
        1.0       0.83      0.84      0.84        90

avg / total       0.97      0.97      0.97      1090<jupyter_text>Random Forest<jupyter_code>res_Rforest = experiment(RandomForestClassifier(n_estimators=300, 
                                                max_features=8, 
                                                min_samples_leaf=20,
                                                n_jobs=8), X, Y, 10)<jupyter_output>precision    recall  f1-score   support

        0.0       0.98      0.99      0.99      1000
        1.0       0.91      0.77      0.83        90

avg / total       0.97      0.97      0.97      1090<jupyter_text>C45<jupyter_code>res_C45 = experiment(DecisionTreeClassifier(criterion='entropy', min_samples_leaf=15), X, Y, 10)<jupyter_output>precision    recall  f1-score   support

        0.0       0.98      0.99      0.99      1000
        1.0       0.89      0.78      0.83        90

avg / total       0.97      0.97      0.97      1090<jupyter_text>ROC Curve<jupyter_code>plt.figure(figsize=(6, 6))
#plt.figaspect(.8)

fpr = res_Dtree['fpr']
tpr = res_Dtree['tpr']
roc_auc = metrics.auc(fpr, tpr)
plt.plot(fpr, tpr, lw=1.5, color='blue', label='Dtree AUC={:06.4f}'.format(roc_auc))

fpr = res_C45['fpr']
tpr = res_C45['tpr']
roc_auc = metrics.auc(fpr, tpr)
plt.plot(fpr, tpr, lw=1.5, color='red', label='C45 AUC={:06.4f}'.format(roc_auc))

fpr = res_Rforest['fpr']
tpr = res_Rforest['tpr']
roc_auc = metrics.auc(fpr, tpr)
plt.plot(fpr, tpr, lw=1.5, color='green', label='Rforest AUC={:06.4f}'.format(roc_auc))

fpr = res_svc['fpr']
tpr = res_svc['tpr']
roc_auc = metrics.auc(fpr, tpr)
plt.plot(fpr, tpr, lw=1.5, color='black', label='SVM AUC={:06.4f}'.format(roc_auc))

fpr = res_knn['fpr']
tpr = res_knn['tpr']
roc_auc = metrics.auc(fpr, tpr)
plt.plot(fpr, tpr, lw=1.5, color='magenta', label='KNN AUC={:06.4f}'.format(roc_auc))

plt.xlim(0, 0.1)
plt.ylim(0.8, 1)
plt.xlabel('False Positive Rate')
plt.ylabel('True Positive Rate')
plt.legend(loc='lower right')
plt.title('Receiver Operating Characteristic Curve')
plt.figure(figsize=(6, 6))
#plt.figaspect(.8)

pr, rec, thr = res_Dtree['pr_rec_curve']
plt.plot(rec, pr, lw=1.5, color='blue', label='Dtree AUC={:06.4f}'.format(roc_auc))

pr, rec, thr = res_C45['pr_rec_curve']
plt.plot(rec, pr, lw=1.5, color='red', label='C45 AUC={:06.4f}'.format(roc_auc))

pr, rec, thr = res_Rforest['pr_rec_curve']
plt.plot(rec, pr, lw=1.5, color='green', label='Rforest AUC={:06.4f}'.format(roc_auc))

pr, rec, thr = res_svc['pr_rec_curve']
plt.plot(rec, pr, lw=1.5, color='black', label='SVM AUC={:06.4f}'.format(roc_auc))

pr, rec, thr = res_knn['pr_rec_curve']
plt.plot(rec, pr, lw=1.5, color='magenta', label='KNN AUC={:06.4f}'.format(roc_auc))

#plt.xlim(0, 0.2)
plt.ylim(0.5, 1)
plt.xlabel('Recall')
plt.ylabel('Precision')
plt.legend(loc='lower right')
plt.title('Precision-Recall Curve')
# methods = {'C45'    : res_C45, 
#            'DTree'  : res_Dtree,
#            'KNN'    : res_knn, 
#            'RForest': res_Rforest, 
#            'SVC'    : res_svc}
true_labels = {'predictions': res_svc['y_test']}

methods = [res_C45, res_Dtree, res_knn, res_Rforest, res_svc, true_labels]

kappa_agreement = np.empty((len(methods), len(methods)))

for i in range(len(methods)):
    for j in range(len(methods)):
        kappa_agreement[i, j] = metrics.cohen_kappa_score(methods[i]['predictions'],
                                                          methods[j]['predictions'])#, 
                                                          #labels=['star', 'galaxy'])

plot_confusion_matrix(np.round(kappa_agreement, 3), 
                      classes=['C45', 'DTree', 'KNN', 'RForest', 'SVC', 'TrueLabels'],
                      title=r'Cohen $\kappa$ agreement')<jupyter_output>Confusion matrix, without normalization
[[ 1.     0.858  0.794  0.826  0.715  0.814]
 [ 0.858  1.     0.865  0.823  0.777  0.945]
 [ 0.794  0.865  1.     0.797  0.825  0.872]
 [ 0.826  0.823  0.797  1.     0.781  0.818]
 [ 0.715  0.777  0.825  0.781  1.     0.82 ]
 [ 0.814  0.945  0.872  0.818  0.82   1.   ]]<jupyter_text>Testing vs non-visual galaxies<jupyter_code>no_vis = cat[cat['Visual']=='False']
gxs_no_vis = no_vis[['J', 'H', 'Ks', 'J-Ks', 'J-H', 'H-Ks', 'R$_{1/2}$', 'C', '$\\epsilon$', 'n']].to_pandas()

gxs_no_vis.dropna(inplace=True)
strs2 = strs#.sample(5000)
X_test = np.vstack((strs2.as_matrix(), gxs_no_vis.as_matrix()))

Y_test = np.hstack((np.repeat(0, len(strs2)), np.repeat(1, len(gxs_no_vis))))
rforest = RandomForestClassifier(n_estimators=300, max_features=8, min_samples_leaf=20, n_jobs=8)
rforest.fit(X, Y)
y_proba = rforest.predict_proba(X_test)[:,0]
y_predict  = rforest.predict(X_test)

print metrics.classification_report(Y_test, y_predict)
pr, rec, thr = metrics.precision_recall_curve(Y_test, 1.-y_proba)
fpr, tpr, thr = metrics.roc_curve(Y_test, 1.-y_proba)
print "Brier-loss score = {}".format(metrics.brier_score_loss(Y_test, 1.-y_proba))
plt.figure(figsize=(6, 3))
plt.subplot(121)
plt.xlim(0, 0.2)
plt.ylim(0.8, 1.)
plt.plot(fpr, tpr)
plt.subplot(122)
plt.plot(rec, pr)
print metrics.confusion_matrix(Y_test, y_predict)
import itertools
def plot_confusion_matrix(cm, classes,
                          normalize=False,
                          title='Confusion matrix',
                          cmap=plt.cm.Blues):
    """
    This function prints and plots the confusion matrix.
    Normalization can be applied by setting `normalize=True`.
    """
    plt.imshow(cm, interpolation='nearest', cmap=cmap)
    plt.title(title)
    plt.colorbar()
    tick_marks = np.arange(len(classes))
    plt.xticks(tick_marks, classes, rotation=45)
    plt.yticks(tick_marks, classes)

    if normalize:
        cm = cm.astype('float') / cm.sum(axis=1)[:, np.newaxis]
        print("Normalized confusion matrix")
    else:
        print('Confusion matrix, without normalization')

    print(cm)

    thresh = cm.max() / 2.
    for i, j in itertools.product(range(cm.shape[0]), range(cm.shape[1])):
        plt.text(j, i, cm[i, j],
                 horizontalalignment="center",
                 color="white" if cm[i, j] > thresh else "black")

    plt.tight_layout()
    plt.ylabel('True label')
    plt.xlabel('Predicted label')

plot_confusion_matrix(metrics.confusion_matrix(Y_test, y_predict), ['Star', 'Galaxy'], normalize=False)<jupyter_output>Confusion matrix, without normalization
[[39708   292]
 [  300   182]]
{'fraction_non_alphanumeric': 0.15670831070070615, 'fraction_numerical': 0.06830526887561109, 'mean_word_length': 2.7595712098009186, 'pattern_counts': {'":': 0, '<': 22, '<?xml version=': 0, '>': 23, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 20, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '7', 'dir': 'jupyter-structured-clean-dedup', 'id': '556084', 'license': 'BSD-2-Clause', 'n_tokens_mistral': 3192, 'n_tokens_neox': 2872, 'n_words': 523, 'path': 'visuales_vs_all.ipynb', 'repo_name': 'BrunoSanchez/VVV_galaxies', 'size': '7364'}