<jupyter_start><jupyter_text>Evaluation of oversamplers with a set of classifiers on one databaseIn this notebook we give an example of optimizing oversamplers and classifiers for given dataset.<jupyter_code>import os.path

from sklearn.neighbors import KNeighborsClassifier
from sklearn.tree import DecisionTreeClassifier

import smote_variants as sv

import imbalanced_databases as imbd
import pandas as pd
import numpy as np
# the evaluation procedure uses a directory for caching

cache_path= os.path.join(os.path.expanduser('~'), 'smote_test')

if not os.path.exists(cache_path):
    os.makedirs(cache_path)
# specifying the dataset to be used

dataset= imbd.load_glass0()
datas = {}
dat = pd.read_csv('bupa-full.csv')
dat.head()
datas['target'] = dat['output_class'].to_numpy()
datas['target']
type(datas['target'])
type(dataset['target'])
dat = dat.drop('output_class', axis=1)
print(dat.head())
datas['data'] = np.array(dat.values.tolist())
datas['data']
print(type(datas['data']))
print(type(dataset['data']))
datas['name'] = 'bupa-full'
datas
# dataset[]
# specifying the classifiers

knn_classifier= KNeighborsClassifier()
dt_classifier= DecisionTreeClassifier()
# executing the evaluation using 5 parallel jobs and at most 35 random but meaningful parameter combinations
# with the 5 quickest oversamplers

results= sv.evaluate_oversamplers(datasets= [datas],
                                    samplers= [sv.SMOTE, sv.SMOTE_IPF],
                                    classifiers= [knn_classifier],
                                    cache_path= cache_path,
                                    n_jobs= 5,
                                    max_samp_par_comb= 35)
# determining oversampler and classifier combination with highest AUC score

highest_auc_score= results['auc'].idxmax()
# print(highest_auc_score)
# The results are arranged in a pandas DataFrame with the following columns:
# db_name - name of the database
# classifier - name of the classifier
# sampler - name of the oversampling technique
# auc - highest auc score with the classifier and oversampler (aggregated over all classifier and oversampler
# parameter combinations)
# brier - highest brier score with the classifier and oversampler (aggregated similarly)
# acc - the highest accuracy score with the classifier and oversampler (aggregated similarly)
# f1 - the highest f1 score with the classifier and oversampler (aggregated similarly)
# p_top20 - the highest p_top20 score with the classifier and oversampler (aggregated similarly)
# gacc - the highest GACC score with the classifier and oversampler (aggregated similarly)
# runtime - average runtime in seconds
# db_size - size of the dataset
# db_n_attr - number of attributes in the dataset
# imbalanced_ratio - the ratio of majority/minority class sizes
# sampler_categories - the categories assigned to the oversampler
# classifier_parameters_auc - the classifier parameters reaching the highest auc score
# classifier_parameters_acc - the classifier parameters reaching the highest acc score
# classifier_parameters_gacc - the classifier parameters reaching the highest gacc score
# classifier_parameters_f1 - the classifier parameters reaching the highest f1 score
# classifier_parameters_p_top20 - the classifier parameters reaching the highest p_top20 score
# classifier_parameters_brier - the classifier parameters reaching the highest brier score
# sampler_parameters_auc - the oversampler parameters reaching the highest auc score
# sampler_parameters_acc - the oversampler parameters reaching the highest acc score
# sampler_parameters_gacc - the oversampler parameters reaching the highest gacc score
# sampler_parameters_f1 - the oversampler parameters reaching the highest f1 score
# sampler_parameters_p_top20 - the oversampler parameters reaching the highest p_top20 score
# sampler_parameters_brier - the oversampler parameters reaching the highest brier score

print(results.columns)
# The results can be processed according to the requirements of the analysis

print(results)
# querying classifier and oversampler parameters with highest AUC score

cl, cl_par, samp, samp_par= results.loc[highest_auc_score][['classifier',
                                                           'classifier_parameters_auc',
                                                           'sampler',
                                                           'sampler_parameters_auc']]
# instantiating oversampler and classifier objects providing the highest AUC score

samp_obj= getattr(sv, samp)(**eval(samp_par))
cl_obj= eval(cl)(**eval(cl_par))
# oversampling the entire dataset and fitting a classifier

X_samp, y_samp= samp_obj.sample(dataset['data'], dataset['target'])
cl_obj.fit(X_samp, y_samp)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.06743455497382199, 'fraction_numerical': 0.005654450261780105, 'mean_word_length': 3.9853862212943634, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 11, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '596519', 'license': 'MIT', 'n_tokens_mistral': 1327, 'n_tokens_neox': 1229, 'n_words': 452, 'path': 'code/smote_borderline_ipf_comparative_study/003_evaluation_one_dataset.ipynb', 'repo_name': 'vs666/SMAI-Project', 'size': '4775'}