<jupyter_start><jupyter_text>Feature importance analysis with ELI5Using ELI5, this script allows you to plot feature importance for models, as well as for individual predictions.<jupyter_code>import os
import sys
from collections import OrderedDict

import pandas as pd
import matplotlib.pyplot as plt
import sklearn
import eli5
import numpy as np
import random

sys.path.append('../../src/utils/')
sys.path.append('../../src/models/')
sys.path.append('../../src/features/')
sys.path.append('../../src/store/')
sys.path.append('../../src/dataproc/')

from persist import load, save
from load_config import *
from SQLConn import SQLConn
from create_data_sample import sample
local_paths_env = load_local_paths('../../src/pipeline/local_paths.yaml')
env = load_psql_env(local_paths_env['pgpass_path'])
prod_config = load_config('../../src/prod/prod_config.yaml', append_static=False)
connection = SQLConn(env)

# pull data from database
connection.open()
X_train, X_test, y_train, y_test = sample(ignition=prod_config, connection=connection,
                                              local_features_path=local_paths_env['store_features'])
connection.close()
# store model and file names
model_names = []
file_names = []

for file in os.listdir(local_paths_env['store_production_models']):
    mod = ''.join([c for c in file if c.isupper()])
    model_names.append(mod)
    file_names.append(file.rstrip('.pkl'))<jupyter_output><empty_output><jupyter_text>Saving top 1000 features for all models<jupyter_code>features_groups = {}

for i, mod in enumerate(model_names):
    if not mod in features_groups.keys():
        print(file_names[i])
        print(mod)
        
        # load model
        model = load(local_paths_env['store_production_models'], file_names[i])
        
        # check if it has a vectorizer - rest of the notebook assumes this is the case
        if hasattr(model, 'vectorizer'):
            clf = model.models[model_names[i].lower()]

            # not all models have the same number of weights - try a high number
            # and decreaase if that fails
            try:
                weights = eli5.explain_weights(clf, vec=model.vectorizer, top=1000, target_names=[0,1])
            except:
                try:
                    weights = eli5.explain_weights(clf, vec=model.vectorizer, target_names=[0,1])

                except:
                    try:
                        weights = eli5.explain_weights(clf, vec=model.vectorizer, top=10, target_names=[0,1])
                    except:
                        weights = eli5.explain_weights(clf, vec=model.vectorizer, top=5, target_names=[0,1])

            features_list = eli5.formatters.as_dataframe.format_as_dataframe(weights)['feature'].tolist()
            features_groups[model_names[i]] = features_list
        else:
            print("This model has no vectorizer.")    

        

        
features_data = pd.DataFrame(dict([ (k,pd.Series(v)) for k,v in features_groups.items() ]))
features_data.to_csv('features_data.csv')
features_data.head()<jupyter_output><empty_output><jupyter_text>Analyzing individual models and predictions<jupyter_code># index determines which group's model is selected
i = 1

# load model
model = load(local_paths_env['store_production_models'], file_names[i])

# check if it has a vectorizer - rest of the notebook assumes this is the case
if hasattr(model, 'vectorizer'):
    clf = model.models[model_names[i].lower()]
else:
    print("This model has no vectorizer.")
clf.coef_[:,75000:]<jupyter_output><empty_output><jupyter_text>Showing weights for one model<jupyter_code># show weights for class (overall)
eli5.explain_weights(clf, vec=model.vectorizer, top=6, target_names=[0,1])
np.count_nonzero(clf.coef_)<jupyter_output><empty_output><jupyter_text>Showing weights for one paper<jupyter_code># select a paper randomly belonging to the class of the model currently loaded
classdata = X_test[y_test[model_names[i].lower()]]
n = random.randint(0, len(classdata))
testpaper = classdata[model.tokens_col].iloc[n]
# evaluate the prediction for a paper
print(model_names[i])
eli5.show_prediction(clf, testpaper, vec=model.vectorizer, target_names=[True, False])<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1140018921475875, 'fraction_numerical': 0.009224219489120151, 'mean_word_length': 3.6936736958934517, 'pattern_counts': {'":': 0, '<': 21, '<?xml version=': 0, '>': 21, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '5', 'dir': 'jupyter-structured-clean-dedup', 'id': '486567', 'license': 'MIT', 'n_tokens_mistral': 1326, 'n_tokens_neox': 1301, 'n_words': 311, 'path': 'analyses/postmodelling_analysis/features_eli5.ipynb', 'repo_name': 'alan-turing-institute/DSSG19-Cochrane-PUBLIC', 'size': '4228'}