<jupyter_start><jupyter_text>Definitions and Loading<jupyter_code>import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
from autorank import autorank, create_report
from IPython.display import display, HTML
import seaborn as sns
from os import listdir

def show_results(metric, order='ascending'):
    result_df = pd.DataFrame()
    for approach, score in scores.items():
        result_df[approach] = score[metric]

    res = autorank(result_df, order=order, approach='bayesian')

    display(res.rankdf)
    fig, ax = plt.subplots()
    sns.boxplot(data=result_df, ax=ax)
    sns.swarmplot(data=result_df, ax=ax)
    plt.show()
    create_report(res)

scores = {}
for file in listdir('../scores'):
    if file.endswith('.csv'):
        approach = file.split('.')[0]
        cur_scores = pd.read_csv('../scores/'+file)
        cur_scores = cur_scores.sort_values('project')
        scores[approach] = cur_scores<jupyter_output><empty_output><jupyter_text>MCC<jupyter_code>show_results('mcc')<jupyter_output>/home/sherbold/git/promise-challenge/notebooks/venv/lib/python3.8/site-packages/scipy/stats/morestats.py:1678: UserWarning: Input data for shapiro has range zero. The results may not be accurate.
  warnings.warn("Input data for shapiro has range zero. The results "
/home/sherbold/git/promise-challenge/notebooks/venv/lib/python3.8/site-packages/autorank/_util.py:107: RuntimeWarning: invalid value encountered in double_scalars
  return (np.median(x) - np.median(y)) / _pooled_mad(x, y)<jupyter_text>Costs (C=1000)<jupyter_code>show_results('cost_1000', order='descending')<jupyter_output><empty_output><jupyter_text>Costs (C=10000)<jupyter_code>show_results('cost_1000', order='descending')<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.12650257584430452, 'fraction_numerical': 0.01659988551803091, 'mean_word_length': 5.37956204379562, 'pattern_counts': {'":': 0, '<': 16, '<?xml version=': 0, '>': 16, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '192525', 'license': 'Apache-2.0', 'n_tokens_mistral': 647, 'n_tokens_neox': 608, 'n_words': 112, 'path': 'evaluation/Challenge.ipynb', 'repo_name': 'aaghamohammadi/promise-challenge', 'size': '1747'}