<jupyter_start><jupyter_text>Partial Dependence Plots<jupyter_code># !pip install scikit-learn==1.0.1
import sklearn

import pandas as pd
import numpy as np
import seaborn as sns
import matplotlib.pyplot as plt

from sklearn.datasets import fetch_california_housing
from sklearn.model_selection import train_test_split

from sklearn.pipeline import Pipeline
from sklearn.compose import ColumnTransformer
from sklearn.neural_network import MLPRegressor
from sklearn.preprocessing import MinMaxScaler

from sklearn.inspection import PartialDependenceDisplay
print('The scikit-learn version is {}.'.format(sklearn.__version__))<jupyter_output>The scikit-learn version is 1.0.1.<jupyter_text>Collect the dataset<jupyter_code>cal_housing = fetch_california_housing()
print(cal_housing.DESCR)
X = cal_housing.data
y = cal_housing.target
cal_features = cal_housing.feature_names

df = pd.concat((pd.DataFrame(X, columns=cal_features),
                pd.DataFrame({'MedianHouseVal': y})), axis=1)
df.head()<jupyter_output><empty_output><jupyter_text>Train an MLP model<jupyter_code># Create models
mlp_reg = MLPRegressor(hidden_layer_sizes=[30, 20, 10],
                       max_iter=500)

# Create pipeline
transformer = ColumnTransformer([
    ('numerical', MinMaxScaler(feature_range=(-1,1)), cal_features),
])

mlp_pipeline = Pipeline(steps=[
    ('transform', transformer),
    ('model', mlp_reg)
])

# Create dataset
X_train, X_test, y_train, y_test = train_test_split(df[cal_features], y, test_size=0.2)

mlp_pipeline.fit(X_train, y_train)
def compute_rmse(preds, labels):
    return np.sqrt(np.mean((preds - labels)**2))
model_rmse_error = compute_rmse(mlp_pipeline.predict(X_test), y_test)
print(f'Root mean squared error of MLP model: {model_rmse_error}')<jupyter_output>Root mean squared error of MLP model: 0.6288807413078079<jupyter_text>Let's look at the partial dependence plot for the features `MedInc'<jupyter_code>PartialDependenceDisplay.from_estimator(
    mlp_pipeline, X_train, features=['MedInc']
)
plt.title("Partial Dependence plot for 'MedInc'")
plt.show()
for feature in cal_features:
    PartialDependenceDisplay.from_estimator(
        mlp_pipeline, X_train, features=[feature])<jupyter_output><empty_output><jupyter_text>Compare two features interaction<jupyter_code>from scipy.stats import pearsonr
corr_coeff = pearsonr(df['MedInc'], df['AveRooms'])[0]
print(f'Pearson correlation coeff: {corr_coeff}')
sns.kdeplot(x=df['MedInc'])
sns.rugplot(x=df['MedInc'])
plt.title("Feature Distribution for 'MedInc'")
plt.show()
PartialDependenceDisplay.from_estimator(
    mlp_pipeline, X_train, features=['MedInc'])
sns.rugplot(data=X_train, x='MedInc', color='orange')
plt.title("Partial Dependence for 'MedInc'")
plt.ylim(1, 4)
plt.show()
sns.kdeplot(x=df['HouseAge'])
sns.rugplot(x=df['HouseAge'])
plt.title("Feature Distribution for 'HouseAge'")
plt.show()
PartialDependenceDisplay.from_estimator(
    mlp_pipeline, X_train, features=['HouseAge'])
sns.rugplot(data=X_train, x='HouseAge', color='orange')
plt.title("Partial Dependence for 'HouseAge'")
plt.ylim(2, 2.5)
plt.show()
PartialDependenceDisplay.from_estimator(
    mlp_pipeline, X_train, features=[('HouseAge', 'MedInc')])
plt.title('Partial Dependence plot for (HouseAge, MedInc)')
plt.show()<jupyter_output><empty_output><jupyter_text>For Classification TaskWe'll use the heart risk dataset.<jupyter_code>import tensorflow as tf
import pandas as pd

from sklearn.ensemble import GradientBoostingClassifier
from sklearn.preprocessing import StandardScaler
from sklearn.preprocessing import OneHotEncoder
print(f'The tensorflow version is {tf.__version__}.')
csv_file = tf.keras.utils.get_file(
    'heart.csv', 'http://storage.googleapis.com/download.tensorflow.org/data/heart.csv')
df = pd.read_csv(csv_file)
df.head()
df.FastingBS = df.fbs.apply(lambda x: 'T' if x==1 else 'F')
# Categorical variables
cat_vars = ['sex', 'cp', 'fbs', 'restecg', 'exang', 'slope', 'ca', 'thal']

# Numerical Variables
num_vars = ['age', 'trestbps', 'chol', 'thalach', 'oldpeak']

X = df[num_vars + cat_vars]

# Label
y = df['target']
# Create dataset
X_train, X_test, y_train, y_test = \
train_test_split(X, y, test_size=0.2, random_state=42)
# Create models
gb_clf = GradientBoostingClassifier(
    n_estimators=100, learning_rate=1.0,
    max_depth=1, random_state=0
)

# Create pipeline
transformer = ColumnTransformer([
    ('numerical', StandardScaler(), num_vars),
    ('categorical', OneHotEncoder(), cat_vars),
])

clf_pipeline = Pipeline(steps=[
    ('transform', transformer),
    ('classifier', gb_clf)
])

clf_pipeline.fit(X_train, y_train)
clf_pipeline.score(X_test, y_test)
for feature in num_vars:
    PartialDependenceDisplay.from_estimator(clf_pipeline, X_train, features=[feature])
age_pdp = PartialDependenceDisplay.from_estimator(clf_pipeline, X_train, features=['chol'])
age_rug = sns.rugplot(data=X_train, x='chol', color='orange')
plt.xlabel('Cholesterol')
plt.show()
age_pdp = PartialDependenceDisplay.from_estimator(clf_pipeline, X_train, features=['thalach'])
age_rug = sns.rugplot(data=X_train, x='thalach', color='orange')
plt.xlabel("Max Heart Rate")
plt.show()<jupyter_output><empty_output><jupyter_text>Dealing with Categorical Features First for the sex feature<jupyter_code># first for 'M'
X_pdp = X.copy()
X_pdp['Sex'] = ['M']*X_pdp.shape[0]
print(clf_pipeline.predict(X_pdp).mean())
cat_pdp_dict = {}
for cat_var in cat_vars:
    feature_pdp_dict = {}
    feature_vals = X_train[cat_var].unique()
    for feature_val in feature_vals:
        X_pdp = X_train.copy()
        X_pdp[cat_var] = [feature_val]*X_pdp.shape[0]
        feature_pdp_dict[feature_val] = clf_pipeline.predict(X_pdp).mean()
    cat_pdp_dict[cat_var] = feature_pdp_dict
cat_pdp_dict
for cat_var in cat_vars:
    plt.bar(*zip(*sorted(cat_pdp_dict[cat_var].items())))
    plt.title(cat_var)
    plt.show()<jupyter_output><empty_output><jupyter_text>Working with Multi-Class classification<jupyter_code>from sklearn.multiclass import OneVsRestClassifier
from sklearn.neural_network import MLPClassifier
# To download the dataset.
!wget http://www3.dsi.uminho.pt/pcortez/wine/winequality.zip
!unzip winequality.zip
df = pd.read_csv("./winequality/winequality-red.csv", sep=';')
df.head()
df.quality.unique()
numeric = ['fixed acidity', 'volatile acidity', 'citric acid', 'residual sugar',
       'chlorides', 'free sulfur dioxide', 'total sulfur dioxide', 'density',
       'pH', 'sulphates', 'alcohol']

X = df[numeric]
y = df['quality']
multi_clf = OneVsRestClassifier(
    MLPClassifier(
        hidden_layer_sizes=[256, 128, 64, 32],
        max_iter=750)
    ).fit(X, y)

# Create pipeline
transformer = ColumnTransformer([
    ('numerical', StandardScaler(), numeric),
])


multi_clf_pipeline = Pipeline(steps=[
    ('transform', transformer),
    ('classifier', multi_clf)
])
multi_clf_pipeline.fit(X, y)
multi_clf_pipeline.score(X, y)
fig, ((ax3, ax4, ax5), (ax6, ax7, ax8)) = plt.subplots(2, 3, figsize=(18, 10))
fig.suptitle('Multi-class partial dependence')

target3_disp = PartialDependenceDisplay.from_estimator(
    multi_clf_pipeline, X, features=['citric acid'], target=3, ax=ax3)
target4_disp = PartialDependenceDisplay.from_estimator(
    multi_clf_pipeline, X, features=['citric acid'], target=4, ax=ax4)
target5_disp = PartialDependenceDisplay.from_estimator(
    multi_clf_pipeline, X, features=['citric acid'], target=5, ax=ax5)
target6_disp = PartialDependenceDisplay.from_estimator(
    multi_clf_pipeline, X, features=['citric acid'], target=6, ax=ax6)
target7_disp = PartialDependenceDisplay.from_estimator(
    multi_clf_pipeline, X, features=['citric acid'], target=7, ax=ax7)
target8_disp = PartialDependenceDisplay.from_estimator(
    multi_clf_pipeline, X, features=['citric acid'], target=8, ax=ax8)

ax3.set_title("Target: 3")
ax4.set_title("Target: 4")
ax5.set_title("Target: 5")
ax6.set_title("Target: 6")
ax7.set_title("Target: 7")
ax8.set_title("Target: 8")
plt.show()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1345, 'fraction_numerical': 0.01425, 'mean_word_length': 6.605513307984791, 'pattern_counts': {'":': 0, '<': 31, '<?xml version=': 0, '>': 31, 'https://': 0, 'lorem ipsum': 0, 'www.': 1, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 3, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '8', 'dir': 'jupyter-structured-clean-dedup', 'id': '561744', 'license': 'Apache-2.0', 'n_tokens_mistral': 3027, 'n_tokens_neox': 2907, 'n_words': 561, 'path': '03-tabular/pdp_plots.ipynb', 'repo_name': 'munnm/XAI-for-practitioners', 'size': '8000'}