<jupyter_start><jupyter_text>Visualize Results: Downstream Performance - "Corrupted" ExperimentThis notebook should answer the questions: *Does imputation lead to better downstream performances?* Notebook Structure * Application Scenario 2 - Downstream Performance     * Categorical  Columns (Classification)   * Numerical Columns (Regression)   * Heterogenous Columns (Classification and Regression Combined)<jupyter_code>import numpy as np
import matplotlib as mpl
import matplotlib.pyplot as plt
import os
import pandas as pd
import re
import seaborn as sns

from pathlib import Path
from data_imputation_paper.experiment import read_experiment, read_csv_files
from data_imputation_paper.plotting import draw_cat_box_plot

%matplotlib inline

%load_ext autoreload
%autoreload 2<jupyter_output><empty_output><jupyter_text>Settings<jupyter_code>sns.set(style="whitegrid")
sns.set_context('paper', font_scale=1.5)
mpl.rcParams['lines.linewidth'] = '2'
EXPERIMENT = "corrupted_fix"

EXPERIMENT_PATH = Path(f"../data/experiments/{EXPERIMENT}/")

CLF_METRIC = "Classification Tasks"
REG_METRIC = "Regression Tasks"

DOWNSTREAM_RESULT_TYPE = "downstream_performance_mean"
IMPUTE_RESULT_TYPE = "impute_performance_mean"

FIGURES_PATH = Path(f"../paper/figures/")<jupyter_output><empty_output><jupyter_text>Import the data<jupyter_code>%%time

results = read_csv_files(read_experiment(EXPERIMENT_PATH), read_details=False)
results.head()
na_impute_results = results[
    (results["result_type"] == IMPUTE_RESULT_TYPE) & 
    (results["metric"].isin(["F1_macro", "RMSE"]))
]
na_impute_results.drop(["baseline", "corrupted", "imputed"], axis=1, inplace=True)
na_impute_results = na_impute_results[na_impute_results.isna().any(axis=1)]
na_impute_results.shape
downstream_results = results[
    (results["result_type"] == DOWNSTREAM_RESULT_TYPE) & 
    (results["metric"].isin(["F1_macro", "RMSE"]))
]

# remove experiments where imputation failed
downstream_results = downstream_results.merge(
    na_impute_results,
    how = "left",
    validate = "one_to_one",
    indicator = True,
    suffixes=("", "_imp"),
    on = ["experiment", "imputer", "task", "missing_type", "missing_fraction", "strategy", "column"]
)
downstream_results = downstream_results[downstream_results["_merge"]=="left_only"]

assert len(results["strategy"].unique()) == 1
downstream_results.drop(["experiment", "strategy", "result_type_imp", "metric_imp", "train", "test", "train_imp", "test_imp", "_merge", "corrupted"], axis=1, inplace=True)

downstream_results = downstream_results.rename(
    {
        "imputer": "Imputation Method",
        "task": "Task",
        "missing_type": "Missing Type",
        "missing_fraction": "Missing Fraction",
        "column": "Column",
        "imputed": "Imputed",
        "baseline": "Baseline"
    },
    axis = 1
)
rename_imputer_dict = {
    "ModeImputer": "Mean/Mode",
    "KNNImputer": "$k$-NN",
    "ForestImputer": "Random Forest",
    "AutoKerasImputer": "Discriminative DL",
    "VAEImputer": "VAE",
    "GAINImputer": "GAIN"    
}

rename_metric_dict = {
    "F1_macro": CLF_METRIC,
    "RMSE": REG_METRIC
}

downstream_results = downstream_results.replace(rename_imputer_dict)
downstream_results = downstream_results.replace(rename_metric_dict)

downstream_results<jupyter_output><empty_output><jupyter_text>Robustness: check which imputers yielded `NaN`values<jupyter_code>for col in downstream_results.columns:
    na_sum = downstream_results[col].isna().sum()
    if na_sum > 0:
        print("-----" * 10)        
        print(col, na_sum)
        print("-----" * 10)        
        na_idx = downstream_results[col].isna()
        print(downstream_results.loc[na_idx, "Imputation Method"].value_counts(dropna=False))
        print("\n")<jupyter_output><empty_output><jupyter_text>Compute Downstream Performance relative to Baseline<jupyter_code>clf_row_idx = downstream_results["metric"] == CLF_METRIC
reg_row_idx = downstream_results["metric"] == REG_METRIC
downstream_results["Improvement"]   = (downstream_results["Imputed"] - downstream_results["Baseline"]  ) / downstream_results["Baseline"]
downstream_results.loc[reg_row_idx, "Improvement"]   = downstream_results.loc[reg_row_idx, "Improvement"]   * -1

downstream_results<jupyter_output><empty_output><jupyter_text>Application Scenario 2 - Downstream Performance<jupyter_code>draw_cat_box_plot(
    downstream_results,
    "Improvement",
    (-0.02, 0.02),
    FIGURES_PATH,
    "corrupted_downstream_boxplot.eps",
    hue_order=list(rename_imputer_dict.values()),
    row_order=list(rename_metric_dict.values())
)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.12832145171743356, 'fraction_numerical': 0.005616763879887665, 'mean_word_length': 5.869436201780416, 'pattern_counts': {'":': 15, '<': 25, '<?xml version=': 0, '>': 26, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '6', 'dir': 'jupyter-structured-clean-dedup', 'id': '331196', 'license': 'Apache-2.0', 'n_tokens_mistral': 1624, 'n_tokens_neox': 1565, 'n_words': 270, 'path': 'notebooks/results_downstream_corrupted.ipynb', 'repo_name': 'se-jaeger/data-imputation-paper', 'size': '4629'}