<jupyter_start><jupyter_text>Model to analyse the chosen ML regressor in terms of how the features contribute to its prediction<jupyter_code>import numpy as np
import pandas as pd
import math
import joblib
import os
import custom_transformers
from importlib import reload
reload(custom_transformers)
from custom_transformers import (
    TransformAdoptionFeatures,
    TransformCensusFeatures, 
    TransformClimateFeatures,
    TransformSoilFeatures,
    TransformEconomicFeatures,
)
# For plotting
%matplotlib inline
import matplotlib as mpl
import matplotlib.pyplot as plt
mpl.rc('axes', labelsize=14)
mpl.rc('xtick', labelsize=12)
mpl.rc('ytick', labelsize=12)
# Where to save the figures
PROJECT_ROOT_DIR = "."
IMAGES_PATH = os.path.join(PROJECT_ROOT_DIR, "images")
os.makedirs(IMAGES_PATH, exist_ok=True)

def save_fig(fig_id, tight_layout=True, fig_extension="jpeg", resolution=200):
    path = os.path.join(IMAGES_PATH, fig_id + "." + fig_extension)
    print("Saving figure", fig_id)
    if tight_layout:
        plt.tight_layout()
    plt.savefig(path, format=fig_extension, dpi=resolution)
 # How to display floats
pd.options.display.float_format = '{:.6f}'.format
np.set_printoptions(precision=10)<jupyter_output><empty_output><jupyter_text>Data ingestion Same procedure as in the notebook for the ML models evaluation<jupyter_code>path_to_dataset = "Municipalities final dataset for analysis.csv"
dataset_original = pd.read_csv(path_to_dataset, index_col=['Municipality', 'Year'])
mask_no_adoption = dataset_original['adoption_in_year'] < 0.0000001
dataset_original = dataset_original.drop(dataset_original.loc[mask_no_adoption].index)
len(dataset_original)
# Remove outliers adoption_in_year
outliers = dataset_original.loc[dataset_original['adoption_in_year'] > 0.12].index
dataset_original = dataset_original.drop(outliers)
print(outliers)
# Remove outliers cumul_adoption_10_y_pr_y_munic
outliers = dataset_original.loc[dataset_original['tot_cumul_adoption_pr_y_munic'] > 0.4].index
dataset_original = dataset_original.drop(outliers)
print(outliers)
dataset_original.shape
dataset = dataset_original.reset_index(drop=True)
dataset.shape<jupyter_output><empty_output><jupyter_text>Copy the data splitting label and features<jupyter_code>dataset_labels = dataset['adoption_in_year'].copy()
dataset = dataset.drop('adoption_in_year', axis=1)
dataset_labels.describe()<jupyter_output><empty_output><jupyter_text>Original features names collection<jupyter_code>features_original = [feat for feat in dataset.columns]
features_adoption = [feat for feat in features_original if 'adoption' in feat]
features_climate = [feat for feat in features_original if '_t_' in feat or '_prec_' in feat]
features_soil = ['CaCO3_mean_munic', 'CN_mean_munic', 'N_mean_munic', 'P_mean_munic', 'pH_mean_munic']
features_economic = ['sbp_payment']

features_not_census = (features_adoption + features_climate + features_soil + features_economic)
features_census = [feat for feat in features_original if feat not in features_not_census]
len(features_census) + len(features_not_census)<jupyter_output><empty_output><jupyter_text>Models analysis<jupyter_code>from sklearn.pipeline import Pipeline
from sklearn.compose import ColumnTransformer
from sklearn.preprocessing import StandardScaler

from sklearn.model_selection import train_test_split
from sklearn.base import clone
from sklearn.model_selection import train_test_split
import shap
from pdpbox import pdp, get_dataset, info_plots
ml_model = joblib.load('./Models to analyse/all_adoption/regressor/nl_svm.pkl')
model = Pipeline([
    ('scaling', StandardScaler()),
    ('estimator', ml_model)
])<jupyter_output><empty_output><jupyter_text>Dataset preparation<jupyter_code>transformation_pipeline = ColumnTransformer([
    ('adoption', TransformAdoptionFeatures(), features_adoption),
    ('census', TransformCensusFeatures(), features_census),
    ('climate', TransformClimateFeatures(), features_climate),
    ('soil', TransformSoilFeatures(), features_soil),
    ('economic', TransformEconomicFeatures(), features_economic)
],
    remainder='drop')
dataset_prepared = transformation_pipeline.fit_transform(dataset)
# Get features names
feat_lists = [features_adoption, features_census, features_climate, features_soil, features_economic]
transformers = [TransformAdoptionFeatures(), TransformCensusFeatures(),
                TransformClimateFeatures(), TransformSoilFeatures(), TransformEconomicFeatures()]

features = []
for f, t in zip(feat_lists, transformers):
    features_ind = t.transform(dataset[f]).columns.to_list()
    features = features + features_ind
len(features)
# Create dictionary to be able to get the index of the feature
features_idx = {}
for i, f in enumerate(features):
    features_idx[f] = i<jupyter_output><empty_output><jupyter_text>Stratified split<jupyter_code>years = dataset_original.index.get_level_values(1).to_list()
train_X, val_X, train_y, val_y = train_test_split(dataset_prepared, dataset_labels, stratify=years, random_state=42)
model.fit(train_X, train_y)<jupyter_output><empty_output><jupyter_text>Permutation importance<jupyter_code>import eli5
from eli5.sklearn import PermutationImportance

perm = PermutationImportance(model, random_state=1).fit(val_X, val_y)
eli5.show_weights(perm, feature_names = features)<jupyter_output><empty_output><jupyter_text>SHAP values<jupyter_code>import pickle
filename = './Model analysis shap values/regressor_all_adoption_shapvalues.sav'
explainer = shap.KernelExplainer(model.predict, shap.kmeans(train_X, 100))
# Calculate shap_values for all of val_X
shap_values = explainer.shap_values(val_X)
# Save shap values
pickle.dump(shap_values, open(filename, 'wb'))
# Load shapvalues
shap_values = pickle.load(open(filename, 'rb'))
# Make plot. Index of [1] is explained in text below.
shap.summary_plot(shap_values, val_X, feature_names=features, max_display=len(features), show=False)
f = plt.gcf()
save_fig(('shap_regressor_all_adoption'), tight_layout=True)
feat_to_plot = 'tot_cumul_adoption_pr_y_munic'
feat_interact = 'tot_cumul_adoption_pr_y_port'
shap.dependence_plot(features_idx[feat_to_plot], shap_values, val_X, feature_names=features,
                     interaction_index=feat_interact, show=False)
save_fig('regressor_cumul_adoption_munic_port', tight_layout=True)
feat_to_plot = 'tot_cumul_adoption_pr_y_munic'
feat_interact = None
shap.dependence_plot(features_idx[feat_to_plot], shap_values, val_X, feature_names=features,
                     interaction_index=feat_interact, show=False)
fig = plt.gcf()
save_fig('regressor_cumul_adoption_munic', tight_layout=True)
feat_to_plot = 'tot_cumul_adoption_pr_y_port'
feat_interact = 'auto'
shap.dependence_plot(features_idx[feat_to_plot], shap_values, val_X, feature_names=features,
                     interaction_index=feat_interact, show=False)
fig = plt.gcf()
save_fig('regressor_cumul_adoption_port_munic', tight_layout=True)
feat_to_plot = 'tot_cumul_adoption_pr_y_port'
feat_interact = None
shap.dependence_plot(features_idx[feat_to_plot], shap_values, val_X, feature_names=features,
                     interaction_index=feat_interact, show=False)
fig = plt.gcf()
save_fig('regressor_cumul_adoption_port', tight_layout=True)
feat_to_plot = 'sbp_payment'
feat_interact = None
shap.dependence_plot(features_idx[feat_to_plot], shap_values, val_X, feature_names=features,
                     interaction_index=feat_interact, show=False)
fig = plt.gcf()
save_fig('regressor_payments', tight_layout=True)
feat_to_plot = 'educ_second_super'
feat_interact = 'auto'
shap.dependence_plot(features_idx[feat_to_plot], shap_values, val_X, feature_names=features,
                     interaction_index=feat_interact, show=False)
fig = plt.gcf()
save_fig('regressor_educ_payments', tight_layout=True)
for feat in features:
    feat_to_plot = feat
    feat_interact = 'auto'
    shap.dependence_plot(features_idx[feat_to_plot], shap_values, val_X, feature_names=features,
                         interaction_index=feat_interact, show=False)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.09566943789552054, 'fraction_numerical': 0.004963394962154114, 'mean_word_length': 7.475289169295478, 'pattern_counts': {'":': 0, '<': 38, '<?xml version=': 0, '>': 39, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '9', 'dir': 'jupyter-structured-clean-dedup', 'id': '454433', 'license': 'MIT', 'n_tokens_mistral': 2744, 'n_tokens_neox': 2633, 'n_words': 498, 'path': 'ml_models/Regression - Model analysis.ipynb', 'repo_name': 'giacrava/sbp-adoption-abm', 'size': '8059'}