<jupyter_start><jupyter_text>Create synthetic data and train RandomForest<jupyter_code>p = 0.7
n = 10000
d = 5
mean = np.zeros(d)
# determinitist covariance
cov = p*np.ones(shape=(d, d)) - (p - 1)*np.eye(d)
C = [[]]

# random covariance
# b = np.random.randn(d, d)
# cov = np.dot(b, b.T)

# covariance
coefs = 4*np.random.randn(d)
model_type = 'linear'
k = 10
coefs[k:] = 0

exp = ExperimentsLinear(mean=mean, cov=cov, coefs=coefs, n=n, C=C)
data = exp.data
print('Coefficients of the Linear regression are = {}'.format(coefs))

# model = XGBRegressor()
model = RandomForestRegressor(n_estimators=1)
model.fit(exp.data, exp.y_train)

print('TEST SET MSE = {}'.format(mean_squared_error(model.predict(exp.data_test), exp.y_test)))<jupyter_output>TEST SET MSE = 4.3908717109733235<jupyter_text>Build ACVTREE<jupyter_code>acvtree = ACVTree(model, data)<jupyter_output>100%|██████████| 1/1 [00:02<00:00,  2.62s/it]<jupyter_text>Sampled data for explanations $\mathcal{D}^{Explain}$<jupyter_code>n_gen = 50000
nb = 1000
exp2 = ExperimentsLinear(mean=mean, cov=cov, coefs=coefs, n=n_gen, C=C)
data_gen = np.array(exp.data, dtype=np.float)<jupyter_output><empty_output><jupyter_text>Compute SV with the differents estimators on $\mathcal{D}^{Explain}$<jupyter_code>plug_sv = acvtree.py_shap_values_notoptimized(data_gen[:nb], C=[[]], data=data)
mc_sv = tree_sv_exact(data_gen[:nb], C=[[]], tree=acvtree, mean=mean, cov=cov, N=10000)
# shap_sv = tree_shap(X=data_gen[:nb], data=data, C=[[]], tree=acvtree, value_function=shap_cond_exp)
explainer_observational = shap.TreeExplainer(model, feature_perturbation='observational')
shap_sv = explainer_observational.shap_values(data_gen[:nb])
plug_sv = plug_sv.squeeze()
shap_sv = shap_sv.squeeze()
mc_sv = mc_sv.squeeze()

sample_data = np.concatenate([shap_sv, plug_sv], axis=0)

shap_dataFra = pd.DataFrame(data=sample_data.squeeze())
shap_dataFra['estimators'] = nb * ['Tree SHAP'] + nb * ['Leaf'] 

relative = np.zeros(shape=mc_sv.shape)
for i in range(mc_sv.shape[0]):
    for j in range(mc_sv.shape[1]):
        if np.abs(mc_sv[i, j]) <= 1:
            relative[i, j] = 1
        else:
            relative[i, j] = mc_sv[i, j]

# err_monte_carlo = l1_norm(mc_sv- mc_sv)/mc_sv.shape[1]
# err_plugin = l1_norm((plug_sv - mc_sv)/relative)/mc_sv.shape[1]
# err_shap = l1_norm((shap_sv - mc_sv)/relative)/mc_sv.shape[1]
err_monte_carlo = l1_norm(mc_sv- mc_sv)
err_plugin = l1_norm((plug_sv - mc_sv)/relative)
err_shap = l1_norm((shap_sv - mc_sv)/relative)
err_concat = np.concatenate([err_shap, err_plugin], axis=0)
shap_dataFra['R-MAE'] = err_concat

plot = True
if plot:
    fig, ax = plt.subplots(dpi=150)
    sns.boxplot(data=shap_dataFra, x='estimators', y='R-MAE', palette=palette_colors)
#     plt.title('R-MAE Leaf vs TreeSHAP on 1000 new observations ')
fig, ax = plt.subplots(dpi=150)
sns.barplot(data=shap_dataFra, x='estimators', y='R-MAE', palette=palette_colors)
print('err_shap = {} -- err_plugin = {}'.format(np.mean(err_shap), np.mean(err_plugin)))
from experiments import utils

def return_fi(sv, k=3):
    fi = np.zeros(shape=sv.shape)
    argsort = np.argsort(sv, axis=1)
    for i in range(argsort.shape[0]):
        for j in range(k):
            fi[i, argsort[i, j]] = 1 
    return fi

fi_mc = return_fi(mc_sv)
fi_shap = return_fi(shap_sv)
fi_plug = return_fi(plug_sv)
print('SHAP - TPR - std TPR - FDE - std FDR')
utils.feature_performance_metric(fi_mc, fi_shap)
print('Leaf - TPR - std TPR - FDE - std FDR')
utils.feature_performance_metric(fi_mc, fi_plug)<jupyter_output>Leaf - TPR - std TPR - FDE - std FDR<jupyter_text>Errors on anomaly samples<jupyter_code>from sklearn.ensemble import IsolationForest
clf = IsolationForest(random_state=0).fit(data_gen)

ano_idx = clf.predict(data_gen[:nb])
# data_ano = data_gen[:nb][ano_idx == -1]
# nb_ano = np.sum(ano_idx == -1)

nb_ano = np.sum(ano_idx == -1)
idx = ano_idx==-1

sample_data = np.concatenate([shap_sv[idx], plug_sv[idx]], axis=0)

shap_dataFra = pd.DataFrame(data=sample_data.squeeze())
shap_dataFra['estimators'] = nb_ano * ['Tree SHAP'] + nb_ano * ['Leaf'] 

# err_monte_carlo = l1_norm(mc_sv- mc_sv)/mc_sv.shape[1]
# err_plugin = l1_norm((plug_sv - mc_sv)/relative)/mc_sv.shape[1]
# err_shap = l1_norm((shap_sv - mc_sv)/relative)/mc_sv.shape[1]
err_monte_carlo = l1_norm(mc_sv[idx]- mc_sv[idx])
err_plugin = l1_norm((plug_sv[idx] - mc_sv[idx])/relative[idx])
err_shap = l1_norm((shap_sv[idx] - mc_sv[idx])/relative[idx])
err_concat = np.concatenate([err_shap, err_plugin], axis=0)
shap_dataFra['R-MAE'] = err_concat

plot = True
if plot:
    fig, ax = plt.subplots(dpi=150)
    sns.boxplot(data=shap_dataFra, x='estimators', y='R-MAE')
fig, ax = plt.subplots(dpi=150)
sns.barplot(data=shap_dataFra, x='estimators', y='R-MAE', palette=palette_colors)
print('err_shap = {} -- err_plugin = {}'.format(np.mean(err_shap), np.mean(err_plugin)))
from experiments import utils

def return_fi(sv, k=3):
    fi = np.zeros(shape=sv.shape)
    argsort = np.argsort(sv, axis=1)
    for i in range(argsort.shape[0]):
        for j in range(k):
            fi[i, argsort[i, j]] = 1 
    return fi

fi_mc = return_fi(mc_sv[idx])
fi_shap = return_fi(shap_sv[idx])
fi_plug = return_fi(plug_sv[idx])
print('SHAP - TPR - std TPR - FDE - std FDR')
utils.feature_performance_metric(fi_mc, fi_shap)
print('Leaf - TPR - std TPR - FDE - std FDR')
utils.feature_performance_metric(fi_mc, fi_plug)<jupyter_output>Leaf - TPR - std TPR - FDE - std FDR<jupyter_text>Errors on normal samples<jupyter_code>nb_ano = np.sum(ano_idx == 1)
idx = ano_idx==1

sample_data = np.concatenate([shap_sv[idx], plug_sv[idx]], axis=0)

shap_dataFra = pd.DataFrame(data=sample_data.squeeze())
shap_dataFra['estimators'] = nb_ano * ['Tree SHAP'] + nb_ano * ['Leaf'] 

# err_monte_carlo = l1_norm(mc_sv- mc_sv)/mc_sv.shape[1]
# err_plugin = l1_norm((plug_sv - mc_sv)/relative)/mc_sv.shape[1]
# err_shap = l1_norm((shap_sv - mc_sv)/relative)/mc_sv.shape[1]
err_monte_carlo = l1_norm(mc_sv[idx]- mc_sv[idx])
err_plugin = l1_norm((plug_sv[idx] - mc_sv[idx])/relative[idx])
err_shap = l1_norm((shap_sv[idx] - mc_sv[idx])/relative[idx])
err_concat = np.concatenate([err_shap, err_plugin], axis=0)
shap_dataFra['R-MAE'] = err_concat

plot = True
if plot:
    fig, ax = plt.subplots(dpi=150)
    sns.boxplot(data=shap_dataFra, x='estimators', y='R-MAE', palette=palette_colors)
fig, ax = plt.subplots(dpi=150)
sns.barplot(data=shap_dataFra, x='estimators', y='R-MAE', palette=palette_colors)
print('err_shap = {} -- err_plugin = {}'.format(np.mean(err_shap), np.mean(err_plugin)))
def return_fi(sv, k=3):
    fi = np.zeros(shape=sv.shape)
    argsort = np.argsort(sv, axis=1)
    for i in range(argsort.shape[0]):
        for j in range(k):
            fi[i, argsort[i, j]] = 1 
    return fi
fi_mc = return_fi(mc_sv[idx])
fi_shap = return_fi(shap_sv[idx])
fi_plug = return_fi(plug_sv[idx])
from experiments import utils
print('SHAP - TPR - std TPR - FDE - std FDR')
utils.feature_performance_metric(fi_mc, fi_shap)
print('Leaf - TPR - std TPR - FDE - std FDR')
utils.feature_performance_metric(fi_mc, fi_plug)<jupyter_output>Leaf - TPR - std TPR - FDE - std FDR
{'fraction_non_alphanumeric': 0.1625070106561974, 'fraction_numerical': 0.01962983735277622, 'mean_word_length': 5.279049295774648, 'pattern_counts': {'":': 0, '<': 22, '<?xml version=': 0, '>': 20, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '6', 'dir': 'jupyter-structured-clean-dedup', 'id': '76433', 'license': 'MIT', 'n_tokens_mistral': 3190, 'n_tokens_neox': 3058, 'n_words': 522, 'path': 'notebook/Figure 3 - Comparisons of the differents estimators on continuous variables.ipynb', 'repo_name': 'aistats2022exp/AccurateShapleyValues', 'size': '7132'}