<jupyter_start><jupyter_text>Shapely<jupyter_code>%reload_ext autoreload
%autoreload 2
import os
import sys
import time
import numpy as np
from Shapley import ShapNN
from DShap import DShap
import matplotlib.pyplot as plt
import sklearn
from shap_utils import *
%matplotlib inline
MEM_DIR = './'
problem, model = 'classification', 'logistic'
hidden_units = [] # Empty list in the case of logistic regression.
train_size = 500
num_test = 240
num_test
X, y = X_raw[:train_size], y_raw[:train_size]
#X_test, y_test = X_raw[train_size:], y_raw[train_size:]
print(X_train.toarray().shape)
print(y_test.shape)
model = 'logistic'
problem = 'classification'
# num_test = 600
directory = './temp1'
dshap = DShap(X.toarray(), y, X_test.toarray(), y_test, num_test, 
              sources=None, 
              sample_weight=None,
              model_family=model, 
              metric='accuracy',
              overwrite=True,
              directory=directory, seed=0)
dshap.run(100, 0.1, g_run=False)
X, y = X_raw[:train_size], y_raw[:train_size]
# X_test, y_test = X_raw[100:], y_raw[100:]
model = 'logistic'
problem = 'classification'
# num_test = 600
directory = './temp1'
dshap = DShap(X.toarray(), y, X_test.toarray(), y_test, num_test, model_family=model, metric='accuracy',
              directory=directory, seed=1)
dshap.run(100, 0.1)
dshap.sources[0]
X, y = X_raw[:train_size], y_raw[:train_size]
# X_test, y_test = X_raw[100:], y_raw[100:]
model = 'logistic'
problem = 'classification'
# num_test = 600
directory = './temp1'
dshap = DShap(X.toarray(), y, X_test.toarray(), y_test, num_test, model_family=model, metric='accuracy',
              directory=directory, seed=2)
dshap.run(100, 0.1)
dshap.merge_results()
convergence_plots(dshap.marginals_tmc)
convergence_plots(dshap.marginals_g)
#removing high value data point

dshap.performance_plots([dshap.vals_tmc, dshap.vals_g, dshap.vals_loo], num_plot_markers=20,
                       sources=dshap.sources)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.12982456140350876, 'fraction_numerical': 0.02606516290726817, 'mean_word_length': 4.483516483516484, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '491054', 'license': 'MIT', 'n_tokens_mistral': 796, 'n_tokens_neox': 736, 'n_words': 156, 'path': 'Untitled.ipynb', 'repo_name': 'OisheeH30/DataShapleyValue', 'size': '1995'}