<jupyter_start><jupyter_text>Reading results of bootsrap simulations<jupyter_code>file_path = 'result_bootstrap_pipeline.txt'

res_dict = {
    'regex': {'f1': [], 'auc': [], 'tn': [], 'tp': [], 'fn': [], 'fp': [], 'fpr': [], 'tpr': []},
    'lgbm': {'f1': [], 'auc': [], 'tn': [], 'tp': [], 'fn': [], 'fp': [], 'fpr': [], 'tpr': []},
    'lgbm + regex': {'f1': [], 'auc': [], 'tn': [], 'tp': [], 'fn': [], 'fp': [], 'fpr': [], 'tpr': []},
    'random_forests': {'f1': [], 'auc': [], 'tn': [], 'tp': [], 'fn': [], 'fp': [], 'fpr': [], 'tpr': []},
    'svm': {'f1': [], 'auc': [], 'tn': [], 'tp': [], 'fn': [], 'fp': [], 'fpr': [], 'tpr': []},
    'logit': {'f1': [], 'auc': [], 'tn': [], 'tp': [], 'fn': [], 'fp': [], 'fpr': [], 'tpr': []}
}
file_path = "result_bootstrap.txt"

with open(file_path, 'r') as file:
    for cnt, line in enumerate(file):
        ss = line.split('\t')
        model = ss[0]
        res_dict[model]['f1'].append(float(ss[1]))
        res_dict[model]['tn'].append(int(ss[2]))
        res_dict[model]['tp'].append(int(ss[3]))
        res_dict[model]['fn'].append(int(ss[4]))
        res_dict[model]['fp'].append(int(ss[5]))
        res_dict[model]['auc'].append(float(ss[6]))
        res_dict[model]['fpr'].append(ss[8])
        res_dict[model]['tpr'].append(ss[10])<jupyter_output><empty_output><jupyter_text>Computing the CI for the metrics of interest<jupyter_code>metrics = ['f1', 'auc', 'tn', 'tp', 'fn', 'fp']
medians = {}
icmin = {}
icmax = {}
for model in res_dict.keys():
    try:
        print(model)
        for metric in metrics:
            print('\t', metric, ': ', np.percentile(res_dict[model][metric], [2.5, 50, 97.5]))
        icmin[model], medians[model],icmax[model] = np.percentile(res_dict[model]['auc'], [2.5, 50, 97.5])
    except:
        print('Missing results for '+model)<jupyter_output><empty_output><jupyter_text>Plotting the median ROC curves<jupyter_code>plt.figure()
lw = 2
ordered_models = [k for k, v in sorted(medians.items(), key=lambda item: item[1], reverse=True)]

for model in ordered_models:
    if 'regex' in model:
        continue
    plt.plot([float(x) for x in fprs[model].split('$')], [float(x) for x in tprs[model].split('$')],
             lw=lw, label='ROC curve {} (area = %0.2f [%0.2f - %0.2f])'.format(model.replace('_', ' ')) % (medians[model], icmin[model], icmax[model]))
plt.plot([float(x) for x in fprs['regex'].split('$')], [float(x) for x in tprs['regex'].split('$')], color = 'grey',
             lw=lw, label='ROC curve {} (area = %0.2f [%0.2f - %0.2f])'.format('regex') % (medians['regex'], icmin['regex'], icmax['regex']))
plt.plot([0, 1], [0, 1], color='black', lw=lw, linestyle='--')
plt.xlim([0.0, 1.0])
plt.ylim([0.0, 1.0])
plt.xlabel('False Positive Rate', fontsize=14)

plt.ylabel('True Positive Rate', fontsize=14)
plt.title('Receiver operating characteristic curves', fontsize=18)
plt.legend(loc="lower right")
plt.savefig('roc_auc', dpi=None, facecolor='w', edgecolor='w')
plt.show()<jupyter_output><empty_output><jupyter_text>Plotting the F1 score with CI<jupyter_code>ci_inf = []
ci_sup = []
meds = []
for model in ordered_models:
    per = np.percentile(res_dict[model]["f1"][:1000], [2.5, 50, 97.5])
    ci_inf.append(per[0])
    meds.append(per[1])
    ci_sup.append(per[2])

ci_inf = np.array(ci_inf)
meds = np.array(meds)
ci_sup = np.array(ci_sup)
# get the range of the confidence interval
legend = [
    'lgbm + regex',
     'lgbm',
     'logit',
     'svm',
     'random forests',
     'regex'
]
yerr = np.c_[meds-ci_inf,ci_sup-meds ].T
bar = plt.bar(
    range(len(ordered_models_wo_lstmrx)),
    meds, yerr=yerr,
    error_kw={"capsize":3, "elinewidth":1, "markeredgewidth":1},
    alpha=0.8,
    align='center',
    color=['C0', 'C0', 'C1', 'C2', 'C3', 'C4', 'C5', 'C6', 'grey']
)
bar[0].set_hatch('/')
plt.xticks(range(len(ordered_models_wo_lstmrx)), legend, fontsize=12, rotation=45)
plt.title('F1 with 95% confidence intervals', fontsize=18)
plt.savefig('f1_ci', dpi=None, facecolor='w', edgecolor='w')
plt.show()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.2530179847253018, 'fraction_numerical': 0.025622074402562207, 'mean_word_length': 4.377483443708609, 'pattern_counts': {'":': 3, '<': 17, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '448250', 'license': 'MIT', 'n_tokens_mistral': 1662, 'n_tokens_neox': 1605, 'n_words': 307, 'path': 'notebooks/bootstrap-analysis.ipynb', 'repo_name': 'louisletinier/MAITAI', 'size': '4059'}