<jupyter_start><jupyter_text>Evaluating ExCeeD with respect to the empirical frequencies of class predictions.<jupyter_code>%matplotlib inline
%reload_ext autoreload
%autoreload 2

import numpy as np
import matplotlib
import matplotlib.pyplot as plt
import pandas as pd
import random

from pyod.models.knn import KNN
from pyod.models.iforest import IForest
from pyod.models.ocsvm import OCSVM

from ExCeeD import *
from evaluate_ExCeeD import *
from scipy.io import arff

def ignore_warnings():
    import warnings
    warnings.simplefilter(action='ignore', category=FutureWarning)
    warnings.simplefilter(action='ignore', category=Warning)

ignore_warnings()<jupyter_output><empty_output><jupyter_text>Evaluating ExCeeD & state-of-the-art methods on a Benchmark Dataset<jupyter_code>data = arff.loadarff('/Benchmark_Datasets/WBC_norm_v02.arff')#WBC_norm_v02.arff
df = pd.DataFrame(data[0])
df['outlier'] = [string.decode("utf-8") for string in df['outlier'].values]
y = np.asarray([1 if string == 'yes' else 0 for string in df['outlier'].values])
X = df[df.columns[:-2]].values #insert the effective features (no response label y)

model = 'KNN' #otherwise 'IForest', 'OCSVM'

L2_error = compute_confidence_error(model, X, y, saveresults = True) #dictionary with the L2 errors

print('L2 error between confidence methods and empirical frequencies:')
L2_error
data = arff.loadarff('/Benchmark_Datasets/Pima_withoutdupl_norm_20_v02.arff')#Pima_withoutdupl_norm_20_v02.arff
df = pd.DataFrame(data[0])
df['outlier'] = [string.decode("utf-8") for string in df['outlier'].values]
y = np.asarray([1 if string == 'yes' else 0 for string in df['outlier'].values])
X = df[df.columns[:-2]].values #insert the effective features (no response label y)

model = 'KNN' #otherwise 'IForest', 'OCSVM'

L2_error = compute_confidence_error(model, X, y, saveresults = False) #dictionary with the L2 errors

print('L2 error between confidence methods and empirical frequencies:')
L2_error
data = arff.loadarff('/Benchmark_Datasets/Ionosphere_withoutdupl_norm.arff')#Ionosphere_withoutdupl_norm.arff
df = pd.DataFrame(data[0])
df['outlier'] = [string.decode("utf-8") for string in df['outlier'].values]
y = np.asarray([1 if string == 'yes' else 0 for string in df['outlier'].values])
X = df[df.columns[:-2]].values #insert the effective features (no response label y)

model = 'KNN' #otherwise 'IForest', 'OCSVM'

L2_error = compute_confidence_error(model, X, y, saveresults = False) #dictionary with the L2 errors

print('L2 error between confidence methods and empirical frequencies:')
L2_error<jupyter_output>L2 error between confidence methods and empirical frequencies:
{'fraction_non_alphanumeric': 0.1160075329566855, 'fraction_numerical': 0.01544256120527307, 'mean_word_length': 6.46067415730337, 'pattern_counts': {'":': 0, '<': 8, '<?xml version=': 0, '>': 8, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '289056', 'license': 'Apache-2.0', 'n_tokens_mistral': 931, 'n_tokens_neox': 887, 'n_words': 244, 'path': 'Experiments/Experiments.ipynb', 'repo_name': 'Lorenzo-Perini/Confidence_AD', 'size': '2655'}