<jupyter_start><jupyter_text>Create synthetic classification dataset  Make classification<jupyter_code>n_features = 4
X, y = make_classification(n_samples=1000, n_informative=2, n_features=n_features, n_redundant=2)
X = preprocessing.normalize(X)
X=pd.DataFrame(data=X)
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.33, random_state=42)<jupyter_output><empty_output><jupyter_text>XGBoost<jupyter_code>X_train.columns = ['0','1','2','3']
X_test.columns = ['0','1','2','3']
xgb_model = xgb.XGBClassifier(use_label_encoder=False, eval_metric='logloss')
xgb_model.fit(X_train, y_train)
xgb_preds = xgb_model.predict(X_test)
print(accuracy_score(y_test, xgb_preds))

print(classification_report(y_test, xgb_preds))<jupyter_output>0.9212121212121213
              precision    recall  f1-score   support

           0       0.92      0.92      0.92       160
           1       0.92      0.92      0.92       170

    accuracy                           0.92       330
   macro avg       0.92      0.92      0.92       330
weighted avg       0.92      0.92      0.92       330<jupyter_text>Permutation importance<jupyter_code>perm = PermutationImportance(xgb_model, random_state=1).fit(X_test, y_test)
perm_importances = perm.feature_importances_
feature_names = [str(i) for i in range(n_features)]
eli5.show_weights(perm, feature_names=feature_names)
gm = GlobalFeatureMetric()
ct = ColumnTransformer([('_INXAI_normal_noise_perturber', NormalNoisePerturber(scale=10),X_test.columns)])

acc_loss = gm.gradual_perturbation(model=xgb_model, X=X_test, y=y_test, column_transformer=ct, importances_orig=perm_importances, resolution=50,  count_per_step=10)
from sklearn.metrics import auc
1-auc(np.linspace(0, 1, 50), acc_loss)<jupyter_output><empty_output><jupyter_text>Check stability<jupyter_code>shape_lips = gm.stability(X_test,generate_per_instance_importances(models=xgb_model, X=X_test, y=y_test, framework='tree_shap') )
lime_lips = gm.stability(X_test,generate_per_instance_importances(models=xgb_model, X=X_test, y=y_test, framework='lime') )
lip_df = pd.DataFrame({'lime':lime_lips, 'shap':shape_lips})
sns.boxplot(x="variable", y="value", data=pd.melt(lip_df))<jupyter_output><empty_output><jupyter_text>Check consistency<jupyter_code>from sklearn.linear_model import LogisticRegression
lr = LogisticRegression()
lr.fit(X_train, y_train)
res = generate_per_instance_importances(models=[xgb_model, lr], X=X_test, y=y_test, framework='lime') 
lime_cons = gm.consistency(res)
cons_df = pd.DataFrame({'consistency':lime_cons})
sns.boxplot(x="variable", y="value", data=pd.melt(cons_df))
lip_df<jupyter_output><empty_output><jupyter_text>Make circles<jupyter_code>X,y = sklearn.datasets.make_moons(n_samples=1000,  noise=0.08)#factor=0.5
X_train, X_test, y_train, y_test = train_test_split(pd.DataFrame(X), y, test_size=0.33, random_state=42)
X_train.columns = ['0','1']
X_test.columns = ['0','1']
X_train.plot(kind='scatter',x='0', y='1',style='.', c=y_train, colormap='viridis')
#xgb_model = xgb.XGBClassifier(use_label_encoder=False, eval_metric='logloss')
from sklearn.svm import SVC
svc = SVC(kernel='poly',probability=True)
svc.fit(X_train, y_train)
svc_preds = svc.predict(X_test)
print(accuracy_score(y_test, svc_preds))

print(classification_report(y_test, svc_preds))
from sklearn.linear_model import LogisticRegression
lr = LogisticRegression()
lr.fit(X_train, y_train)
lr_preds = lr.predict(X_test)
print(accuracy_score(y_test, lr_preds))

print(classification_report(y_test, lr_preds))
fig, ax = plt.subplots()
point = [1.0,-0.25]
ax.plot(point[0], point[1], 'og')
X_test.plot(kind='scatter',x='0', y='1', c=y_test, colormap='viridis', ax=ax)
X_test_appended = X_test.append(pd.DataFrame([point], columns=['0','1']), ignore_index=True)
y_test_appended = np.concatenate([y_test,[1]])<jupyter_output><empty_output><jupyter_text>Stability<jupyter_code>gm = GlobalFeatureMetric()
shap_res=generate_per_instance_importances(models=svc, X=X_test_appended, y=y_test_appended, framework='kernel_shap')
lime_res = generate_per_instance_importances(models=svc, X=X_test_appended, y=y_test_appended, framework='lime')
shape_lips = gm.stability(X_test_appended,shap_res ,epsilon=0.3)

lime_lips = gm.stability(X_test_appended,lime_res,epsilon=0.3 )

lip_df = pd.DataFrame({'lime':lime_lips, 'shap':shape_lips})
sns.boxplot(x="variable", y="value", data=pd.melt(lip_df))<jupyter_output><empty_output><jupyter_text>Consistency per model<jupyter_code>res_lime_con = generate_per_instance_importances(models=[svc, lr], X=X_test_appended, y=y_test_appended, framework='lime') 
res_shap_con = generate_per_instance_importances(models=[svc, lr], X=X_test_appended, y=y_test_appended, framework='kernel_shap') 
lime_cons = gm.consistency(res_lime_con)
shap_cons = gm.consistency(res_shap_con)
cons_df = pd.DataFrame({'lime':lime_cons, 'shap':shap_cons})
sns.boxplot(x="variable", y="value", data=pd.melt(cons_df))<jupyter_output>Using 331 background data samples could cause slower run times. Consider using shap.sample(data, K) or shap.kmeans(data, K) to summarize the background as K samples.<jupyter_text>Area under the ACCLOSS<jupyter_code>import shap
import lime
explainer_shap = shap.KernelExplainer(svc.predict_proba, X_train)
explainer_lime =lime.lime_tabular.LimeTabularExplainer(X_train,feature_names=list(X_train.columns), 
                                                       class_names=['0','1'], discretize_continuous=False)
shap_res_instance = explainer_shap.shap_values(X_test_appended)
explainer_lime.explain_instance(X_test.iloc[0],svc.predict_proba).as_list()
def expl(row):
    e = explainer_lime.explain_instance(row,svc.predict_proba).as_list()
    return [e[0][1],e[1][1]]
shap_res_global = pd.DataFrame(shap_res_instance[0]).apply(abs).mean()
lime_res_global = X_test.apply(lambda x: expl(x), axis=1, result_type='expand').apply(abs).mean()
ct = ColumnTransformer([('_INXAI_normal_noise_perturber', NormalNoisePerturber(scale=10),X_test_appended.columns)])
acc_loss_shap = gm.gradual_perturbation(model=svc, X=X_test_appended, y=y_test_appended, column_transformer=ct, importances_orig=shap_res_global, resolution=50,  count_per_step=10)
acc_loss_lime = gm.gradual_perturbation(model=svc, X=X_test_appended, y=y_test_appended, column_transformer=ct, importances_orig=lime_res_global, resolution=50,  count_per_step=10)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.13306199123356294, 'fraction_numerical': 0.02582968065122104, 'mean_word_length': 7.692517006802721, 'pattern_counts': {'":': 0, '<': 35, '<?xml version=': 0, '>': 35, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 6, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '9', 'dir': 'jupyter-structured-clean-dedup', 'id': '311997', 'license': 'MIT', 'n_tokens_mistral': 2595, 'n_tokens_neox': 2501, 'n_words': 325, 'path': 'examples/global_feature_metrics.ipynb', 'repo_name': 'sbobek/inxai', 'size': '6388'}