<jupyter_start><jupyter_text>ROC and AUC<jupyter_code>fpr, tpr, th = roc_curve(true, score)
auc = roc_auc_score(true, score)<jupyter_output><empty_output><jupyter_text>Best accuracy threshold<jupyter_code>a_s_l = []
for t in th[1:]:
    pred = np.array(score.values)
    pred[pred >= t] = 1
    pred[pred < t] = 0
    a_s = accuracy_score(true, pred)
    a_s_l.append(a_s)

m_val = max(a_s_l)
th_m_val = th[np.argmax(a_s_l)]<jupyter_output><empty_output><jupyter_text>Precision-Recall curve<jupyter_code>prec, rec, th_pr = precision_recall_curve(true, score)

th_m_val_index = np.where(th_pr == th_m_val)
print (prec[th_m_val_index], rec[th_m_val_index])

diff2 = np.power((prec[1:]-rec[1:]),2.0)
th_min_diff = th_pr[diff2.argmin()]<jupyter_output>[0.87584918] [0.8595]<jupyter_text>F1-score at max_accuracy threshold<jupyter_code>pred = np.array(score.values)
pred[pred >= th_m_val] = 1
pred[pred < th_m_val] = 0
f1_score(true, pred)<jupyter_output><empty_output><jupyter_text>All diagrams on a single graph<jupyter_code>fig, axs = plt.subplots(2,2,figsize=(12,8))

fpr, tpr, th = roc_curve(true, score)
auc = roc_auc_score(true, score)

ax = axs[0,0]
ax.plot(fpr, tpr, label='AUC: %.3f' % auc)
ax.legend(loc='lower right')
ax.grid()
ax.set_xlabel('fpr')
ax.set_ylabel('tpr (recall)')
ax.set_title('ROC')

ax = axs[0,1]
ax.plot(prec, rec)
ax.grid()
ax.set_xlabel('Precision')
ax.set_ylabel('Recall')
ax.set_title('Precision-Recall')

ax = axs[1,0]
ax.plot(th[1:], a_s_l)
ax.axhline(m_val, label='max acc: %.3f' % m_val, c='g', ls = '--')
ax.axvline(th_m_val, label='th at max acc: %.2f' % th_m_val, c='k', ls='--')
#ax.axvline(th_min_diff, c='r', ls='--', label=r'th $prec \approx recall$: %.2f' % th_min_diff)
ax.legend()
ax.grid()
ax.set_title('Accuracy vs threshold')
ax.set_xlabel('Threshold')
ax.set_ylabel('Accuracy')

ax = axs[1,1]
ax.plot(th_pr[:], prec[1:], label='Precision')
ax.plot(th_pr[:], rec[1:], label='Recall')
ax.axvline(th_min_diff, c='r', ls='--', label=r'th $prec \simeq recall$: %.2f' % th_min_diff)
#ax.axvline(th_m_val, label='th at max acc: %.2f' % th_m_val, c='k', ls='--')
ax.grid()
ax.legend()
ax.set_xlabel('Threshold')
ax.set_title('Precision and Recall vs threshold')

plt.tight_layout()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.19466666666666665, 'fraction_numerical': 0.021777777777777778, 'mean_word_length': 6.553691275167785, 'pattern_counts': {'":': 0, '<': 22, '<?xml version=': 0, '>': 22, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '5', 'dir': 'jupyter-structured-clean-dedup', 'id': '227887', 'license': 'MIT', 'n_tokens_mistral': 1074, 'n_tokens_neox': 1030, 'n_words': 168, 'path': 'notebooks/Evaluation/ROC_Analysis.ipynb', 'repo_name': 'simleo/promort_pipeline', 'size': '2250'}