<jupyter_start><jupyter_text>Intialization: data loading<jupyter_code>df_for_classification = pd.read_csv("../datasets/spam_for_classification.csv")
df_for_classification.drop(["Unnamed: 0"], axis=1, inplace=True)
for c in df_for_classification.columns:
    df_for_classification[c] = df_for_classification[c].apply(lambda x: np.array(eval(x)) if not isinstance(x, int) else x)<jupyter_output><empty_output><jupyter_text>Train NN on top of spam dataset<jupyter_code>df_for_classification_flat = pd.read_csv("../datasets/spam_for_classification_flat.csv")
df_for_classification_flat.drop(["Unnamed: 0"], axis=1, inplace=True)
X_vectors,y_vectors = np.array(df_for_classification_flat.iloc[:, :-1]), np.array(df_for_classification_flat.iloc[:, -1])

X_train_vectors, X_test_vectors, y_train_vectors, y_test_vectors = \
    sklearn.model_selection.train_test_split(X_vectors, y_vectors, test_size=.2, random_state=42)
clf = MLPClassifier(random_state=1, max_iter=300).fit(X_train_vectors, y_train_vectors)
clf.score(X_test_vectors, y_test_vectors)
preds = df_for_classification_flat.apply(lambda x: clf.predict(x.iloc[:-1].values.reshape(1, -1))[0], axis=1)
df_for_classification_flat['Model_pred'] = preds
df_for_classification['Model_pred'] = preds<jupyter_output><empty_output><jupyter_text>choose inp<jupyter_code>## 998 is just a random example instance
inp_raw = pd.DataFrame(df.iloc[998]).T
inp_raw
inp_vectors = pd.DataFrame(df_for_classification.iloc[998]).T
num_features = len(inp_vectors.columns[:-2])
inp_final_rep = pd.DataFrame(df_for_classification_flat.iloc[998]).T
f_inp = clf.predict(inp_final_rep.iloc[:, :-2])
f_inp<jupyter_output><empty_output><jupyter_text>Ref choice The non-categorical nature of these features necessitates a little more careful translation from user query to selection criteria from dataset<jupyter_code># this is an example of how to use more involved selection criteria, but we do not pursue it below
relevant_1 = df_for_classification[df_for_classification.outcome != inp_vectors.outcome.item()]

chosen_ind = nec_suf.close_or_distant_neighbours(relevant_1, inp_vectors, 'From_glv_vec', like=True, perc=0.1);<jupyter_output><empty_output><jupyter_text>Example: identifying minimal sufficient factors + necessity scores for all references of opposite outcome from inp<jupyter_code>CF_condition_1 = df_for_classification.outcome != inp_vectors.outcome.item()
# CF_condition_1_raw = df_just_str.outcome != inp_raw_just_str.outcome.item()
CF_r2i, CF_i2r, \
CF_i2r_raw_text, refs = nec_suf.suff_nec_pipeline(CF_condition_1, inp_vectors, 
                                        clf, df_for_classification, num_features,
                                        datatype='Text', inp_raw=inp_raw, dataset_raw=df, predict=True)
print("##################")
print("Sufficiency R2I")
print("##################")
CF_df_deg_r2i = nec_suf.deg_nec_suff(CF_r2i, inp_raw, f_inp, clf, num_features, r2i=True, pred_on_fly=False, deg_thresh=0)

sub_df_filtered_r2i = nec_suf.filter_by_degree_and_overalp(CF_df_deg_r2i, degree_thresh=0.65, subset_max_num=10)
print("------------------")
print("necessity score: ", nec_suf.recall_nec_score(CF_r2i, sub_df_filtered_r2i, f_inp))
print("------------------")
sub_df_filtered_r2i
print("##################")
print("Sufficiency I2R")
print("##################")
CF_df_deg_i2r = nec_suf.deg_nec_suff(CF_i2r, inp_raw, f_inp, clf, num_features,
                                     r2i=False, CF_i2r_raw_text=CF_i2r_raw_text, pred_on_fly=False, deg_thresh=0)
sub_df_filtered_i2r = nec_suf.filter_by_degree_and_overalp(CF_df_deg_i2r, degree_thresh=0.9, subset_max_num=10)
print("------------------")
print("necessity score: ", nec_suf.recall_nec_score(CF_i2r, sub_df_filtered_i2r, f_inp, r2i=False))
print("------------------")
sub_df_filtered_i2r<jupyter_output>##################
Sufficiency I2R
##################
------------------
necessity score:  0.3490037719549374
------------------<jupyter_text>SHAP comparison<jupyter_code>CF_condition_string = "dataset.outcome != inp_i.outcome.item()"
all_inp_deg_r2i, all_inp_deg_i2r, all_shaps = \
    cum_shp_plot.cumul_shap_vs_us_multi_input(df_for_classification, clf, CF_condition_string, 
                                    num_features, num_inp=25, datatype='Text', 
                                    df_flattened=df_for_classification_flat)
cum_shp_plot.mean_and_plot(all_inp_deg_r2i, all_shaps, title_name='sufficiency r2i', dataset_name='spam')
cum_shp_plot.mean_and_plot(all_inp_deg_i2r, all_shaps, title_name='sufficiency i2r', dataset_name='spam')<jupyter_output><empty_output><jupyter_text>Adverserial Example<jupyter_code># sample some spam examples, and see what minimal changes to those spam emails would have resulted in non-spam predictions
ref_inds = df[df['outcome']==1].sample(n=7, random_state=42).index
refs = df_for_classification.loc[ref_inds]
refs_present = df.loc[ref_inds]
refs_present
model_weakness_spam.model_gaming(clf, refs, df_for_classification, num_features, num_inp=20, df_flattened=df_for_classification_flat, df_raw=df)<jupyter_output>----------------------
gaming option no.:  0
----------------------
{'fraction_non_alphanumeric': 0.15220345563968163, 'fraction_numerical': 0.0207726655018443, 'mean_word_length': 5.656330749354005, 'pattern_counts': {'":': 0, '<': 27, '<?xml version=': 0, '>': 27, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 6, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '7', 'dir': 'jupyter-structured-clean-dedup', 'id': '281765', 'license': 'CC0-1.0', 'n_tokens_mistral': 1994, 'n_tokens_neox': 1882, 'n_words': 284, 'path': 'notebooks/Spam_experiment.ipynb', 'repo_name': 'limorigu/LENS', 'size': '5151'}