<jupyter_start><jupyter_text>Explore all scenario results This notebook reproduces the analysis of Section 4.3.1 to Section 4.3.5 with some additional visualizations. SetupFollow the instructions `README.md` to download and setup the data files.<jupyter_code>evaluation_file = '../data/output/output.csv'
plot_output_path = '../plots/'
%matplotlib inline
import os
import datetime
import pandas as pd
import numpy as np
import seaborn as sns
import matplotlib.pyplot as plt
sns.set(style="whitegrid")
font_scale=2.2
sns.set_context("notebook", font_scale=font_scale)
plt.rcParams['figure.figsize'] = (10,10)
# Set "True" to output tables in tex-style
print_latex = False
def num_init_labeled(x):
    if x['initial_pool_strategy'] == 'Pp':
        return 0.1 * x['num_points']
    elif x['initial_pool_strategy'] == 'Pn':
        return 20
    elif x['initial_pool_strategy'] == 'Pa':
        return x['num_dimensions'] + 5
    else:
        return 0        
qs_types = {
    'data': ['MinimumMarginPQs', 'ExpectedMinimumMarginPQs', 'ExpectedMaximumEntropyPQs', 'MinimumLossPQs'],
    'model': ['HighConfidencePQs', 'DecisionBoundaryPQs', 'NeighborhoodBasedPQs', 'BoundaryNeighborCombinationPQs'],
    'random': ['RandomPQs', 'RandomOutlierPQs']
}
qs_types = {y: x for x in qs_types for y in qs_types[x] }
def pretty_plot(title="Plot", f=sns.catplot, plot_size=(12, 5), pretty_order=None, pretty_hue_order=None, **kwargs):
    if 'hue' in kwargs:
        kwargs['hue_order'] = order[kwargs['hue']] if pretty_hue_order is None else pretty_hue_order
    if 'x' in kwargs and f != sns.scatterplot and f != sns.pointplot:
        kwargs['order'] = order[kwargs['x']] if pretty_order is None else pretty_order
    fig, ax = plt.subplots()
    _ = fig.set_size_inches(plot_size)
    g = f(ax=ax, **kwargs)
    if type(g) == sns.axisgrid.FacetGrid:
        plt.close(g.fig)
        ax.legend(loc='center left', bbox_to_anchor=(1.0, 0.5))
    return fig
def load_csv_file(f):
    df = pd.read_csv(f)
    df.split_strategy = df.split_strategy.apply(str)
    # merge kde_errors
    df.exit_code = df.exit_code.apply(lambda x: 'kde_error' if x in ["KDEException", "PyCall.PyError"] else x)
    df.exit_code = df.exit_code.apply(lambda x: 'missing_label' if x == "MissingLabelTypeException" else x)
    df.exit_code = df.exit_code.apply(lambda x: 'missing_train_data' if x == "InexactError" else x)
    # Add learning scenario
    df['model'] = df['model'].apply(lambda x: 'SVDD' if x == 'VanillaSVDD' else x)
    df['model_type'] = df['model'].apply(lambda x: 'SSAD' if 'SSAD' in x else 'SVDD')
    df['qs_type'] = df['qs'].apply(lambda x: qs_types[x])
    df['learning_scenario'] = df[['split_strategy', 'initial_pool_strategy']].astype(str).apply(lambda x: '-'.join(x), axis=1)
    df['num_init_labeled'] = df.apply(num_init_labeled, axis=1)
    # Remove Parkinson data set because of is size of only 40 observations (not useful with 50 AL iterations)
    df = df[df['data_set'] != 'Parkinson']
    df_success = df.query('exit_code == "success"').reset_index(drop=True)
    return df, df_success
df_full, df_full_success = load_csv_file(evaluation_file)<jupyter_output><empty_output><jupyter_text>Number of experiments<jupyter_code>df_full.id.count()
df_full.groupby('scenario').count()[['id']]
f"Total experiment run time: {str(datetime.timedelta(seconds=np.sum(df_full.total_run_time)))}."<jupyter_output><empty_output><jupyter_text>Experiment exit codes.<jupyter_code>df_full.groupby('exit_code')[['exit_code']].count()<jupyter_output><empty_output><jupyter_text>* ArgumentError: Incompatible initial pools and split strategy, e.g., Pu (all unlabeled) with Si (train on inliers) OR empty training set and rule of thumb scott (-> Gamma is Inf)* early_abort: No more unlabeled observations left (small data sets: Hepatitis and Parkinson)* kde_error: Covariance matrix is singular (attributes vs. observations; \approx 0 threshold;* missing_label: incompatible initial pools and query strategy* missing_train_data: Train on labeled inliers when label pools is all unlabeled* success: everything worked Specify plot order.<jupyter_code>order = {
    'model_type': ['SVDD', 'SSAD'],
    'qs_type': ['data', 'model', 'random'],
    'qs': ['MinimumMarginPQs', 'ExpectedMinimumMarginPQs', 'ExpectedMaximumEntropyPQs', 'MinimumLossPQs', 'HighConfidencePQs', 'DecisionBoundaryPQs', 'NeighborhoodBasedPQs', 'BoundaryNeighborCombinationPQs', 'RandomPQs', 'RandomOutlierPQs'],
    'data_set': ["ALOI", "Annthyroid", "Arrhythmia", "Cardiotocography", "Glass",
             "HeartDisease", "Hepatitis", "Ionosphere", "KDDCup99", "Lymphography",
             "PageBlocks", "Parkinson", "PenDigits", "Pima", "Shuttle", "SpamBase",
             "Stamps", "WBC", "WDBC", "WPBC", "Waveform"],
    'model': ['SVDD', 'SVDDneg', 'SSAD_0.1', 'SSAD_0.5', 'SSAD_1.0'],
    'split_strategy': ['Sh', 'Sf', 'Si'],
    'initial_pool_strategy': ['Pu', 'Pp', 'Pn', 'Pa']
}
pretty_names = {
                'AEQ': {'m_average_end_quality': 'MCC', 'c_average_end_quality': 'kappa', 'auc_average_end_quality': 'AUC', 'pauc_average_end_quality': 'PAUC'},
                'AEQ-M': {'m_average_end_quality': 'MCC-AEQ', 'c_average_end_quality': 'kappa-AEQ', 'auc_average_end_quality': 'AUC-AEQ', 'pauc_average_end_quality': 'PAUC-AEQ'},
                'SQ-M': {'m_start_quality': 'MCC-SQ', 'k_start_quality': 'kappa-SQ', 'auc_start_quality': 'AUC-SQ', 'pauc_start_quality': 'PAUC-SQ'},
                'MCC': {'m_start_quality': 'SQ', 'm_end_quality': 'EQ', 'm_average_end_quality': 'AEQ', 'm_ramp_up': 'RU'},
                'qs': {
                    'MinimumMarginQs': '$\tau_\text{MM}$',
                    'ExpectedMinimumMarginQs': '$\tau_\text{EMM}$',
                    'ExpectedMaximumEntropyQs': '$\tau_\text{EME}$',
                    'MinimumLossQs': '$\tau_\text{ML}$',
                    'HighConfidenceQs': '$\tau_\text{HC}$',
                    'DecisionBoundaryQs': '$\tau_\text{DB}$',
                    'NeighborhoodBasedQs': '$\tau_\text{NB}$',
                    'BoundaryNeighborCombinationQs': '$\tau_\text{BNC}$',
                    'RandomQs': '$\tau_\text{rand}$',
                    'RandomOutlierQs': '$\tau_\text{rand-out}$',
                  },
}<jupyter_output><empty_output><jupyter_text>Analysis<jupyter_code>df_p1, df_p1_success = df_full.query('scenario == "evaluation_part1"'), df_full_success.query('scenario == "evaluation_part1"')<jupyter_output><empty_output><jupyter_text>Data set statistics<jupyter_code>res = df_full.groupby(["data_set"]).max()[["num_points", "num_dimensions"]].reset_index()
print(res.to_latex(index=False)) if print_latex else res<jupyter_output><empty_output><jupyter_text>Comparison of Evaluation Metrics (Section 4.3.1)<jupyter_code>res = df_p1_success[["m_average_end_quality", "c_average_end_quality", "auc_average_end_quality", "pauc_average_end_quality"]].rename(columns=pretty_names['AEQ']).corr('pearson').round(2)
print(res.to_latex()) if print_latex else res<jupyter_output><empty_output><jupyter_text>Inspect correlation of mcc and pauc.<jupyter_code>f = pretty_plot(title=f'PAUC vs. MCC', 
                x="pauc_average_end_quality", 
                y="m_average_end_quality", 
                data=df_p1_success.sample(5000, random_state=0),
                f = sns.scatterplot,
                legend = None)
f.axes[0].set_xlabel("pAUC-AEQ")
f.axes[0].set_ylabel("MCC-AEQ")
f.tight_layout()
t1 = plt.Polygon([[0.52, -0.02], [1.0025, 0.65], [1.0025, -0.02]], alpha=0.25, color=sns.color_palette()[3], joinstyle='round')
plt.gca().add_patch(t1)
f.savefig(f'{plot_output_path}/eval_comp_pauc_vs_mcc_5k.pdf')<jupyter_output><empty_output><jupyter_text>Influence of Kernel Parameter (Section 4.3.2) Inspect the average end quality per gamma initalization strategy.<jupyter_code>res = df_p1_success. \
    groupby(["model", "init_strategy_gamma"]).\
    median()[["m_start_quality", "m_average_end_quality"]].\
    rename(columns=pretty_names['MCC']).\
    reset_index().\
    round(2).\
    pivot(index = 'model', columns='init_strategy_gamma', values=['SQ', 'AEQ']) .\
    reset_index()
res = res.loc[[3, 4, 0, 1, 2], :]
print(res.to_latex(index=False)) if print_latex else res<jupyter_output><empty_output><jupyter_text>Inspect results per data set.<jupyter_code>res = df_p1_success.query('init_strategy_gamma_fitted_value != "nan" and init_strategy_gamma_fitted_value != "inf" and model == "SVDDneg"'). \
    groupby(["data_set", "init_strategy_gamma"]).\
    median()[["m_start_quality", "m_average_end_quality"]].\
    rename(columns=pretty_names['MCC']).\
    reset_index().\
    round(2).\
    pivot(index = 'data_set', columns='init_strategy_gamma', values=['SQ', 'AEQ'])
print(res.to_latex(index=False)) if print_latex else res<jupyter_output><empty_output><jupyter_text>Inspect fitted gamma values<jupyter_code>df_p1_success.query('init_strategy_gamma_fitted_value != "nan" and init_strategy_gamma_fitted_value != "inf"'). \
   groupby(["init_strategy_gamma"]).\
   agg({"init_strategy_gamma_fitted_value": [min, np.median, np.mean, max]})<jupyter_output><empty_output><jupyter_text>Only include results for WangTax in the next sections<jupyter_code>df_p1_wang_success = df_p1_success.query('init_strategy_gamma == "WangTax"')<jupyter_output><empty_output><jupyter_text>Comparison of Split Strategies (Section 4.3.3)<jupyter_code>f = pretty_plot(title=f'Comparison of split strategies', 
                x="split_strategy", 
                y="m_average_end_quality", 
                hue="model",
                data=df_p1_wang_success,
                kind="box")
f.axes[0].set_xlabel('Split strategy');
f.axes[0].set_ylabel('AEQ')
f.tight_layout()
f.savefig(f'{plot_output_path}/eval_comp_split_strategies.pdf')<jupyter_output><empty_output><jupyter_text>Visualize fluctuation with Average Quality Gain and Average Quality Loss by model.<jupyter_code>f = pretty_plot(title=f'Comparison of initial pool strategies', 
            x="m_average_loss", 
            y="m_average_gain", 
            hue="model_type", 
            data=df_p1_wang_success.query('split_strategy == "Sf"'), 
            f=sns.scatterplot)
f.axes[0].set_xlabel('AQL')
f.axes[0].set_ylabel('AQG')<jupyter_output><empty_output><jupyter_text>Comparison of Initial Pool Strategies (Section 4.3.4)<jupyter_code>f = pretty_plot(title=f'Comparison of initial pool strategies', 
            x="initial_pool_strategy", 
            y="m_start_quality", 
            hue="model", 
            data=df_p1_wang_success, 
            kind="box",
            pretty_order=['Pu', 'Pp', 'Pn', 'Pa'],
            pretty_hue_order=['SVDDneg', 'SSAD_0.1'],
            palette=sns.color_palette()[1:])
f.axes[0].set_xlabel('Initial pool strategy')
f.axes[0].set_ylabel('SQ')
f.tight_layout()
f.savefig(f'{plot_output_path}/eval_comp_initial_pool_strategies.pdf')<jupyter_output><empty_output><jupyter_text>Inspect the high value outliers from the plot.<jupyter_code>df_p1_wang_success.query('m_start_quality > 0.8').groupby('data_set').count()<jupyter_output><empty_output><jupyter_text>Compare number of initially labeled observations for Pn and Pp.<jupyter_code>res = df_p1_wang_success.query('data_set in ["ALOI", "WBC"] ' +
                       'and split_strategy in ["Sf"] ' +
                       'and model == "SVDDneg" ' +
                       'and initial_pool_strategy in ["Pp", "Pn"]').\
    groupby(["data_set", "initial_pool_strategy"]).\
    mean()[["num_points", "num_init_labeled", "m_start_quality", "m_average_end_quality"]].\
    rename(columns=pretty_names['MCC']).\
    reset_index().\
    round(2)
res.num_points = res.num_points.round().astype(int)
res.num_init_labeled = res.num_init_labeled.round().astype(int)
res.columns = ['Data set', 'Initial setup', 'Observations', 'Initially labeled', 'SQ', 'AEQ']
print(res.to_latex(index=False)) if print_latex else res<jupyter_output><empty_output><jupyter_text>Comparison of query strategies (Section 4.3.5) Load experiments with 5 resamples of the initial pool.<jupyter_code>df_p1_qs_success = df_full_success.query('scenario == "evaluation_part1_qs" '+
                                         'and init_strategy_gamma == "WangTax"')<jupyter_output><empty_output><jupyter_text>Heptatitis is not part of this evaluation section because of its size of only 60.<jupyter_code>set(df_full.data_set.unique()) - set(df_p1_qs_success.data_set.unique())<jupyter_output><empty_output><jupyter_text>Compute correlation between summary statistics.<jupyter_code>target_metrics = ['m_end_quality', 'm_ramp_up', 'm_quality_range',
       'm_average_end_quality', 'm_average_quality_change', 'm_average_gain',
       'm_average_loss', 'm_learning_stability', 'm_ratio_of_outlier_queries']
corr = df_p1_qs_success[target_metrics]
corr = corr.corr()
cmap = sns.diverging_palette(240, 10, as_cmap=True)
_ = pretty_plot('Correlation Qs', f=sns.heatmap, data=corr, center=0, square=True, cmap=cmap, linewidths=1)
# For other metrics replace this value, e.g., with "m_average_end_quality"
metric = "m_start_quality"
res = df_p1_qs_success.groupby(["data_set", "qs"]).median()[[metric]].reset_index().pivot(index='data_set', columns = 'qs')[[metric]]
res = res.T.reset_index(level=0, drop=True).T
res = res[[x for x in order['qs'] if x in res.columns]].round(decimals=4)
if print_latex:
    res.columns = [pretty_names['qs'][x] for x in res.columns]
res = res.reset_index()
res.columns = ['Data set'] + res.columns[1:].values.tolist()
print(res.to_latex(escape=False, na_rep='-', index=False)) if print_latex else res<jupyter_output><empty_output><jupyter_text>Additional statistics Comparison against upper bound (classifier trained with all ground-truth labels).<jupyter_code>df_sq = df_p1_qs_success.query('qs == "RandomPQs" and init_strategy_gamma == "WangTax" and model == "SVDDneg"')[['file_name', 'initial_pool_resample_version', 'm_start_quality']].reset_index(drop=True)
df_sq.columns = ['data_file', 'seed', 'mcc_sq']
df_baseline_svddneg = pd.read_csv('../data/output/svddneg_baseline.csv')
df_baseline_svddneg['model'] = "SVDDneg"
df_baseline_ssad = pd.read_csv('../data/output/ssad_baseline.csv')
df_baseline_ssad['model'] = "SSAD"
df_baselines = pd.concat([df_baseline_svddneg, df_baseline_ssad]).set_index(['data_file', 'seed'])
df_baselines = df_baselines.join(df_sq.set_index(['data_file', 'seed'])).reset_index()
df_baselines['mcc_qr'] = df_baselines['mcc'] - df_baselines['mcc_sq']
df_baselines.groupby(['data_set']).median()[['mcc_sq', 'mcc', 'mcc_qr']].round(2)<jupyter_output><empty_output><jupyter_text>Query selection time.<jupyter_code>df_p1_qs_success.query('data_set == "PageBlocks"').groupby(['data_set', 'qs']).mean()[['time_qs_mean']]<jupyter_output><empty_output><jupyter_text>Inspect the ratio of outliers queried per model and query startegy.<jupyter_code>f = pretty_plot(title=f'Init strategy vs ratio of outliers', 
                x="model", 
                y="m_average_end_quality",
                hue="qs",
                data=df_p1_qs_success,
                pretty_order=['SVDDneg', 'SSAD_0.1'],
                kind="box")<jupyter_output><empty_output><jupyter_text>Inspect the performance of the two gamma initialization strategies per query strategy for SVDDneg.<jupyter_code>f = pretty_plot(title=f'Init strategy vs ratio of outliers', 
                x="init_strategy_gamma",
                y="m_average_end_quality",
                hue="qs",
                data=df_p1_success.query('model == "SVDDneg"'),
                pretty_order=['Scott', 'WangTax'],
                kind="box",
                plot_size=(20, 8))
f.axes[0].set_xlabel('Gamma initialization')
f.axes[0].set_ylabel('AEQ')
f.tight_layout()<jupyter_output><empty_output><jupyter_text>Reyes summary stats comparison. Active learning metrics proposed in: <NAME> al. 2018. Statistical comparisons of active learning strategies over multiple datasets. Knowledge-Based Systems. 145, (2018), 1–14. DOI:https://doi.org/10.1016/j.knosys.2018.01.033.<jupyter_code>df_p1_qs_success.query('data_set == "WBC"')[['data_set', 'qs', 'm_quality_range', 'm_aulc', 'm_reyes_paulc', 'm_reyes_tpr', 'm_reyes_naulc', 'm_reyes_tnr', 'm_reyes_tp']].groupby('qs').mean()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.15298530406444075, 'fraction_numerical': 0.011498493512882003, 'mean_word_length': 4.818962432915921, 'pattern_counts': {'":': 1, '<': 102, '<?xml version=': 0, '>': 104, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 3, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '25', 'dir': 'jupyter-structured-clean-dedup', 'id': '575017', 'license': 'MIT', 'n_tokens_mistral': 6076, 'n_tokens_neox': 5737, 'n_words': 1010, 'path': 'notebooks/evaluation_part1.ipynb', 'repo_name': 'kit-dbis/ocal-evaluation', 'size': '16269'}