<jupyter_start><jupyter_text>Visualize Results: Impute Performance - "Fully Observed" ExperimentThis notebook should answer the questions: *Which imputer works best to re-construct missing values?* Notebook Structure * Application Scenario 1 - Imputation Performance    * Categorical  Columns (Classification)   * Numerical Columns (Regression)   * Heterogenous Columns (Classification and Regression Combined)<jupyter_code>import numpy as np
import matplotlib as mpl
import matplotlib.pyplot as plt
import os
import pandas as pd
import re
import seaborn as sns

from pathlib import Path
from data_imputation_paper.experiment import read_experiment, read_csv_files
from data_imputation_paper.plotting import plot_metric_vs_missing_fraction, plot_rank_vs_fraction_by_type, draw_cat_box_plot

%matplotlib inline

%load_ext autoreload
%autoreload 2<jupyter_output><empty_output><jupyter_text>Settings<jupyter_code>sns.set(style="whitegrid")
sns.set_context('paper', font_scale=1.5)
mpl.rcParams['lines.linewidth'] = '2'
EXPERIMENT = "fully_observed"

EXPERIMENT_PATH = Path(f"../data/experiments/{EXPERIMENT}/")

IMPUTE_RESULT_TYPE = "impute_performance_mean"
FIGURES_PATH = Path("../paper/figures/")

CLF_METRIC = "Categorical Columns"
REG_METRIC = "Numerical Columns"

CONFIDENCE_INTERVAL = 90<jupyter_output><empty_output><jupyter_text>Import the data<jupyter_code>%%time

results = read_csv_files(read_experiment(EXPERIMENT_PATH), read_details=False)
results.head()
impute_results = results[
    (results["result_type"] == IMPUTE_RESULT_TYPE) & 
    (results["metric"].isin(["F1_macro", "MAE"]))
]
assert len(results["strategy"].unique()) == 1
impute_results.drop(["experiment", "train", "strategy", "baseline", "corrupted", "imputed"], axis=1, inplace=True) 
impute_results = impute_results.rename(
    {
        "imputer": "Imputation Method",
        "task": "Task",
        "missing_type": "Missing Type",
        "missing_fraction": "Missing Fraction",
        "column": "Column",
        "test": "Imputation"
    },
    axis = 1
)
rename_imputer_dict = {
    "ModeImputer": "Mean/Mode",
    "KNNImputer": "$k$-NN",
    "ForestImputer": "Random Forest",
    "AutoKerasImputer": "Discriminative DL",
    "VAEImputer": "VAE",
    "GAINImputer": "GAIN"    
}

rename_metric_dict = {
    "F1_macro": CLF_METRIC,
    "MAE": REG_METRIC
}

impute_results = impute_results.replace(rename_imputer_dict)
impute_results = impute_results.replace(rename_metric_dict)

impute_results.head()<jupyter_output><empty_output><jupyter_text>Compute Imputer Ranking<jupyter_code>EXPERIMENTAL_CONDITIONS = ["Task", "Missing Type", "Missing Fraction", "Column", "result_type", "metric"]

clf_row_idx = impute_results["metric"] == CLF_METRIC
reg_row_idx = impute_results["metric"] == REG_METRIC
impute_results.loc[clf_row_idx, "Imputation Rank"] = impute_results[clf_row_idx].groupby(EXPERIMENTAL_CONDITIONS).rank(ascending=False, na_option="bottom", method="min")["Imputation"]
impute_results.loc[reg_row_idx, "Imputation Rank"] = impute_results[reg_row_idx].groupby(EXPERIMENTAL_CONDITIONS).rank(ascending=True, na_option="bottom", method="min")["Imputation"]
impute_results<jupyter_output><empty_output><jupyter_text>Robustness: check which imputers yielded `NaN` values<jupyter_code>for col in impute_results.columns:
    na_sum = impute_results[col].isna().sum()
    if na_sum > 0:
        print("-----" * 10)        
        print(col, na_sum)
        print("-----" * 10)        
        na_idx = impute_results[col].isna()
        print(impute_results.loc[na_idx, "Imputation Method"].value_counts(dropna=False))
        print("\n")<jupyter_output>--------------------------------------------------
Imputation 276
--------------------------------------------------
GAIN                 273
Discriminative DL      3
Name: Imputation Method, dtype: int64<jupyter_text>Application Scenario 1 - Imputation Performance<jupyter_code>impute_results.groupby(["metric"]).size()<jupyter_output><empty_output><jupyter_text>The data is not equally distributed between categorical and numerical imputation tasks, so simply averaging the ranks, as done in the plots below, could be misleading.<jupyter_code>draw_cat_box_plot(
    impute_results,
    "Imputation Rank",
    None,
    FIGURES_PATH,
    "fully_observed_impute_rank_boxplot.eps",
    hue_order=list(rename_imputer_dict.values()),
    row_order=list(rename_metric_dict.values())
)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.13990103463787673, 'fraction_numerical': 0.006072874493927126, 'mean_word_length': 5.083447332421341, 'pattern_counts': {'":': 14, '<': 28, '<?xml version=': 0, '>': 29, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '7', 'dir': 'jupyter-structured-clean-dedup', 'id': '238641', 'license': 'Apache-2.0', 'n_tokens_mistral': 1557, 'n_tokens_neox': 1495, 'n_words': 275, 'path': 'notebooks/results_impute_fully_obs.ipynb', 'repo_name': 'se-jaeger/data-imputation-paper', 'size': '4446'}