<jupyter_start><jupyter_text>Model Download And Interpretability  Preparation Library Installation And Imports<jupyter_code>!pip install xgboost==0.90
!pip install eli5
import os
import pickle
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt

import xgboost
from sklearn.metrics import accuracy_score, plot_confusion_matrix, f1_score

from get_model import get_model_weights
from utils import print_f1_scores, get_cause_names_refinement

import warnings
warnings.filterwarnings('ignore')

cause_names = get_cause_names_refinement()<jupyter_output><empty_output><jupyter_text>Downloading And Loading All ModelsThe code below downloads the models from their S3 buckets.**Important:** You must replace the `s3://` paths below with the actual SageMaker model paths. These models were printed on each training notebook after each model was fitted.<jupyter_code>model_linear_data_path = "s3://sagemaker-us-west-1-170667047098/wildfire/linear-learner-2021-02-07-06-54-15-448/output/model.tar.gz"
dest_linear = "models/model_linear.tar.gz"
method_linear = 'mxnet'

model_mlp_data_path = "s3://sagemaker-us-west-1-170667047098/sagemaker-scikit-learn-2021-02-07-05-50-43-359/output/model.tar.gz"
dest_mlp = "models/model_mlp.tar.gz"
method_mlp = 'joblib'

model_xgb_data_path = "s3://sagemaker-us-west-1-170667047098/wildfire/output/xgboost-210213-0501-001-79f78e61/output/model.tar.gz"
dest_xgb = "models/model_xgb.tar.gz"
method_xgb = 'pickle'

model_mlp_ref_data_path = "s3://sagemaker-us-west-1-170667047098/sagemaker-scikit-learn-2021-02-10-21-32-56-685/output/model.tar.gz"
dest_mlp_ref = "models/model_mlp_ref.tar.gz"
loaded_linear_model = get_model_weights(model_linear_data_path, dest_linear, method_linear)
loaded_mlp_model = get_model_weights(model_mlp_data_path, dest_mlp, method_mlp)
loaded_xgb_model = get_model_weights(model_xgb_data_path, dest_xgb, method_xgb)
loaded_mlp_ref_model = get_model_weights(model_mlp_ref_data_path, dest_mlp_ref, method_mlp)<jupyter_output><empty_output><jupyter_text>Refined MLP Model AnalysisAll the work that follows concerns the refined MLP model with 6 target classes, which was trained on `Refined_mlp_model.ipynb` Test Data Load<jupyter_code>data_dir = 'wildfire_data'
test_x = pd.read_csv(os.path.join(data_dir, 'test_ref.csv'), header=None)
test_y = pd.read_csv(os.path.join(data_dir, 'test_ref_y.csv'), header=None)<jupyter_output><empty_output><jupyter_text>Interpretability<jupyter_code>import eli5<jupyter_output>Using MXNet backend<jupyter_text>Using the `PermutationImportance` class to obtain feature importance weights.<jupyter_code>perm = eli5.sklearn.PermutationImportance(loaded_mlp_ref_model).fit(test_x, test_y)
with open(os.path.join("wildfire_data", "feature_names.pkl"), "rb") as file:
    feature_names = pickle.load(file)
eli5.show_weights(perm, top = test_x.shape[1], feature_names = feature_names)<jupyter_output><empty_output><jupyter_text>Plotting top 20 feature weights<jupyter_code>pd.Series(perm.feature_importances_, index = feature_names).nlargest(20).plot.bar()
plt.title("Top 20 most important features")
plt.show()<jupyter_output><empty_output><jupyter_text>Plotting general target class distribution to compare with TX, NY, and GA distribution<jupyter_code>test_y_labels = test_y[0].map(lambda i: cause_names[i])
test_y_labels.groupby(test_y_labels).size().sort_values(ascending = False).plot.bar()
plt.title("Class distribution on whole test set.")
plt.show()<jupyter_output><empty_output><jupyter_text>TX, NY, and GA target class distribution<jupyter_code>test_y_labels_texas = test_y_labels[test_x[feature_names.index('TX')] == 1]
test_y_labels_texas.groupby(test_y_labels_texas).size().sort_values(ascending = False).plot.bar()
plt.title("Class distribution in Texas.")
plt.show()
test_y_labels_texas = test_y_labels[test_x[feature_names.index('NY')] == 1]
test_y_labels_texas.groupby(test_y_labels_texas).size().sort_values(ascending = False).plot.bar()
plt.title("Class distribution in New York.")
plt.show()
test_y_labels_texas = test_y_labels[test_x[feature_names.index('GA')] == 1]
test_y_labels_texas.groupby(test_y_labels_texas).size().sort_values(ascending = False).plot.bar()
plt.title("Class distribution in Georgia.")
plt.show()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.11243851018973998, 'fraction_numerical': 0.03490278753806512, 'mean_word_length': 8.163090128755364, 'pattern_counts': {'":': 0, '<': 32, '<?xml version=': 0, '>': 32, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 3, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '8', 'dir': 'jupyter-structured-clean-dedup', 'id': '49404', 'license': 'MIT', 'n_tokens_mistral': 1663, 'n_tokens_neox': 1553, 'n_words': 259, 'path': 'src/7_Interpretability.ipynb', 'repo_name': 'mariolys07/WildfireCauses', 'size': '4269'}