<jupyter_start><jupyter_text>Model Inspection<jupyter_code>#code adapted from https://github.com/thomasjpfan/ml-workshop-intermediate-1-of-2
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import sklearn

import seaborn as sns
sns.set_theme(font_scale=1.2)
plt.rcParams['figure.figsize'] = [12, 8]
plt.rcParams['savefig.bbox'] = 'tight'
plt.rcParams["savefig.dpi"] = 300

sklearn.set_config(display='diagram')<jupyter_output><empty_output><jupyter_text>Load the dataset<jupyter_code># %load solutions/regression_example.py

import pandas as pd
url = 'https://raw.githubusercontent.com/davidrkearney/Kearney_Data_Science/master/_notebooks/df_panel_fix.csv'
df = pd.read_csv(url, error_bad_lines=False)
df

import sklearn
from sklearn.datasets import fetch_openml
from sklearn.model_selection import train_test_split



df.columns

sklearn.set_config(display='diagram')

df=df.dropna()

df.isnull().sum()



X, y = df.drop(['specific', 'Unnamed: 0'], axis = 1), df['specific']


X = X.select_dtypes(include='number')
X


_ = X.hist(figsize=(30, 15), layout=(5, 8))
X.head()
y.head()<jupyter_output><empty_output><jupyter_text>Insert random data for demonstration<jupyter_code>import numpy as np

X = X.assign(ran_num=np.arange(0, X.shape[0]))<jupyter_output><empty_output><jupyter_text>Split dataset<jupyter_code>from sklearn.model_selection import train_test_split

X_train, X_test, y_train, y_test = train_test_split(
    X, y, random_state=42)<jupyter_output><empty_output><jupyter_text>Train linear model<jupyter_code>from sklearn.preprocessing import StandardScaler
from sklearn.linear_model import Ridge
from sklearn.pipeline import Pipeline

ridge = Pipeline([
    ('scale', StandardScaler()),
    ('reg', Ridge())
])
ridge.fit(X_train, y_train)
ridge.score(X_train, y_train)
ridge.score(X_test, y_test)<jupyter_output><empty_output><jupyter_text>Plot coefficientsCoefficients represent the relationship between a feature and the target assuming that all other features remain constant.<jupyter_code>import pandas as pd
import matplotlib.pyplot as plt

def plot_linear_coef(coefs, names, ax=None, sorted=False):
    if ax is None:
        fig, ax = plt.subplots()
    coefs = pd.DataFrame(
       coefs, columns=['Coefficients'],
       index=names
    )
    
    if sorted:
        coefs = coefs.sort_values(by='Coefficients')

    coefs.plot(kind='barh', ax=ax)
    ax.axvline(x=0, color='.5')
    return ax

plot_linear_coef(ridge['reg'].coef_, names=X_train.columns, sorted=True);<jupyter_output><empty_output><jupyter_text>Coefficient variability<jupyter_code>from sklearn.model_selection import cross_validate
from sklearn.model_selection import RepeatedKFold
ridges_cv = cross_validate(
    ridge, X_train, y_train, cv=RepeatedKFold(n_splits=5, n_repeats=5),
    return_estimator=True)
ridges_cv
ridge_coefs = pd.DataFrame(
   [model['reg'].coef_ for model in ridges_cv['estimator']],
   columns=X.columns
)
ridge_coefs.head()<jupyter_output><empty_output><jupyter_text>Plotting the variability of the cofficients<jupyter_code>fig, ax = plt.subplots()
_ = ax.boxplot(ridge_coefs, vert=False, labels=ridge_coefs.columns)
from sklearn.linear_model import Lasso
lasso = Pipeline([
    ('scale', StandardScaler()),
    ('reg', Lasso(alpha=0.06))
])
lasso.fit(X_train, y_train)
fig, (ax1, ax2) = plt.subplots(1, 2, figsize=(18, 9))
plot_linear_coef(lasso['reg'].coef_, names=X_train.columns, sorted=True, ax=ax1);
plot_linear_coef(ridge['reg'].coef_, names=X_train.columns, sorted=True, ax=ax2);
lasso_cvs = cross_validate(
    lasso, X_train, y_train, return_estimator=True, cv=RepeatedKFold(n_splits=5, n_repeats=5)
)
lasso_coefs = pd.DataFrame(
   [model['reg'].coef_ for model in lasso_cvs['estimator']],
   columns=X.columns
)
fig, ax = plt.subplots()
_ = ax.boxplot(lasso_coefs, vert=False, labels=ridge_coefs.columns)
# %load solutions/03-ex01-solutions.py
from sklearn.linear_model import Lasso

lasso = Pipeline([
    ('scale', StandardScaler()),
    ('reg', Lasso(random_state=42, alpha=0.04))
])
lasso.fit(X_train, y_train)

lasso.score(X_test, y_test)

fig, (ax1, ax2) = plt.subplots(1, 2, figsize=(16, 8))
plot_linear_coef(ridge['reg'].coef_, X_train.columns, ax=ax1)
plot_linear_coef(lasso['reg'].coef_, X_train.columns, ax=ax2)<jupyter_output><empty_output><jupyter_text>Random Forest<jupyter_code>from sklearn.ensemble import RandomForestRegressor

rf = RandomForestRegressor(random_state=42)
rf.fit(X_train, y_train)
rf.score(X_train, y_train)
rf.score(X_test, y_test)
def plot_importances(importances, names, ax=None):
    if ax is None:
        fig, ax = plt.subplots()
    indices = np.argsort(importances)
    ax.barh(range(len(importances)), importances[indices])
    ax.set(yticks=range(len(importances)),
           yticklabels=np.array(names)[indices]);
importances = rf.feature_importances_
plot_importances(importances, X_train.columns);<jupyter_output><empty_output><jupyter_text>Pay attention to `ran_num`!  Permutation Feature Importance  Can be used on the test data!<jupyter_code>from sklearn.inspection import permutation_importance

rf_perm_results = permutation_importance(rf, X_test, y_test,
                                        n_repeats=10, n_jobs=-1)
def plot_permutation_importance(perm_results, names, ax=None):
    perm_sorted_idx = perm_results.importances_mean.argsort()
    if ax is None:
        fig, ax = plt.subplots()
    _ = ax.boxplot(perm_results.importances[perm_sorted_idx].T, vert=False,
                   labels=np.array(names)[perm_sorted_idx])
    return ax
_ = plot_permutation_importance(rf_perm_results, X_test.columns)<jupyter_output><empty_output><jupyter_text>Load cancer dataset<jupyter_code># %load solutions/classifier_example.py
from sklearn.preprocessing import StandardScaler
from sklearn.linear_model import LogisticRegression
from sklearn.pipeline import Pipeline
from sklearn.model_selection import train_test_split

url = 'https://raw.githubusercontent.com/davidrkearney/colab-notebooks/main/datasets/strokes_training.csv'
df = pd.read_csv(url, error_bad_lines=False)
df

df=df.dropna()

df.isnull().sum()

df.columns

sklearn.set_config(display='diagram')

X, y = df.drop(['stroke', 'id'], axis = 1), df['stroke']


X = X.select_dtypes(include='number')
X

X_train, X_test, y_train, y_test = train_test_split(
    X, y, stratify=y)


from sklearn.ensemble import RandomForestClassifier
from sklearn.metrics import classification_report

rf = RandomForestClassifier(random_state=42)
rf.fit(X_train, y_train)

rf.score(X_test, y_test)

y_pred = rf.predict(X_test)

print(classification_report(y_test, y_pred))
from sklearn.ensemble import RandomForestClassifier

rf = RandomForestClassifier(random_state=42)
rf.fit(X_train, y_train)
rf.score(X_test, y_test)<jupyter_output><empty_output><jupyter_text>Permutation importance with random forest<jupyter_code>from sklearn.inspection import permutation_importance

rf_result = permutation_importance(rf, X_train, y_train,
                                   n_repeats=10, n_jobs=-1)<jupyter_output><empty_output><jupyter_text>Training data<jupyter_code>_ = plot_permutation_importance(rf_result, X)
from scipy.stats import spearmanr
from scipy.cluster import hierarchy

corr = spearmanr(X_train).correlation
corr_linkage = hierarchy.ward(corr)
corr_linkage
from collections import defaultdict

cluster_ids = hierarchy.fcluster(corr_linkage, 1, criterion='distance')
cluster_id_to_feature_ids = defaultdict(list)
for idx, cluster_id in enumerate(cluster_ids):
    cluster_id_to_feature_ids[cluster_id].append(idx)
selected_features = [v[0] for v in cluster_id_to_feature_ids.values()]

selected_features<jupyter_output><empty_output><jupyter_text>Feature importance with selected features<jupyter_code>from sklearn.inspection import permutation_importance

rf_sel_result = permutation_importance(
    rf_sel, X_test, y_test, n_repeats=10, n_jobs=-1)
features_sel = data.feature_names[selected_features]
_ = plot_permutation_importance(rf_sel_result, features_sel)<jupyter_output><empty_output><jupyter_text>Partial Dependence  Train a HistGradientBostingClassifer<jupyter_code>from sklearn.experimental import enable_hist_gradient_boosting
from sklearn.ensemble import HistGradientBoostingClassifier 
hist = HistGradientBoostingClassifier(random_state=0)
hist.fit(X_train, y_train)
# %load solutions/03-ex03-solutions.py
from sklearn.datasets import load_boston

boston = load_boston()

X, y = boston.data, boston.target

X_train, X_test, y_train, y_test = train_test_split(
    X, y, random_state=0)

from sklearn.ensemble import GradientBoostingRegressor
gb = GradientBoostingRegressor(random_state=0)

gb.fit(X_train, y_train)

gb.score(X_train, y_train)

plot_importances(gb.feature_importances_, boston.feature_names)

gb_perm_results = permutation_importance(gb, X_test, y_test, n_repeats=10, n_jobs=-1)

plot_permutation_importance(gb_perm_results, boston.feature_names)

plot_partial_dependence(gb, X_test, features=["LSTAT", "RM", "DIS", "CRIM"],
                        feature_names=boston.feature_names, n_cols=2)

plot_partial_dependence(gb, X_test, features=[('LSTAT', 'RM')],
                        feature_names=boston.feature_names)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.11298884194561802, 'fraction_numerical': 0.009424764380890478, 'mean_word_length': 6.445161290322581, 'pattern_counts': {'":': 0, '<': 61, '<?xml version=': 0, '>': 61, 'https://': 3, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '15', 'dir': 'jupyter-structured-clean-dedup', 'id': '401536', 'license': 'Apache-2.0', 'n_tokens_mistral': 3446, 'n_tokens_neox': 3313, 'n_words': 582, 'path': '_notebooks/2021-06-03-model-inspection.ipynb', 'repo_name': 'davidrkearney/Kearney_Data_Science', 'size': '9231'}