<jupyter_start><jupyter_text>Alibi  1. Counterfactual Prototype<jupyter_code>cat_vars_dict = get_cat_vars_dict(scaled_df, categorical_cols, feature_names, target_name)
cat_vars_dict
cat_feature_names = [ col for col in categorical_cols if col != target_name ] 
cat_vars_idx_info = []

for cat_col in cat_feature_names:
    num_unique_v = len([ col for col in train_df.columns if col.startswith(f"{cat_col}_")])
    first_index = min([ list(train_df.columns).index(col) for col in train_df.columns if col.startswith(f"{cat_col}_")])
    
    cat_vars_idx_info.append({
        "col": cat_col,
        "num_unique_v": num_unique_v,
        "first_index": first_index
    })
cat_vars_idx_info
cat_vars_ohe = {}

for idx_info in cat_vars_idx_info:
    cat_vars_ohe[idx_info['first_index']] = idx_info['num_unique_v']

from alibi_cf import AlibiBinaryPredictWrapper
alibi_wrapped = {
    'dt': AlibiBinaryPredictWrapper(models['dt']),
    'rfc': AlibiBinaryPredictWrapper(models['rfc']),
    'nn': AlibiBinaryPredictWrapper(models['nn']),
}
feature_range = (np.ones((1, len(feature_names))), np.zeros((1, len(feature_names))))
cf_p_dict = {}

for k in alibi_wrapped.keys():
    cf_p_dict[k] = CounterFactualProto(
                                alibi_wrapped[k].predict,
                                example_data.shape,
                                cat_vars=cat_vars_ohe,
                                feature_range=feature_range,
                                max_iterations=500,
                                ohe=True,
                                )

    cf_p_dict[k].fit(X_train)
    
""
num_instances = 5
num_cf_per_instance = 1
results = {}
for k in cf_p_dict.keys():
    results[k] = []
    print(f"Finding counterfactual for {k}")
    for idx, instance in enumerate(X_test[0:num_instances]):
        print(f"instance {idx}")
        example = instance.reshape(1, -1)
        for num_cf in range(num_cf_per_instance):
            print(f"CF {num_cf}")
            start_t = time()
            exp = cf_p_dict[k].explain(example)
            end_t = time ()
            running_time = end_t - start_t

            if k=='nn':
                prediction = target_label_encoder.inverse_transform((models[k].predict(example)[0]> 0.5).astype(int))[0]
            else:
                prediction = target_label_encoder.inverse_transform(models[k].predict(example))[0]

            if (not exp.cf is None) and (len(exp.cf) > 0):
                print("Found CF")
                if k == 'nn':
                    cf = inverse_dummy(pd.DataFrame(exp.cf['X'], columns=ohe_feature_names), cat_to_ohe_cat)
                    cf.loc[0, target_name] = target_label_encoder.inverse_transform([exp.cf['class']])[0]
                else:
                    cf = inverse_dummy(pd.DataFrame(exp.cf['X'], columns=ohe_feature_names), cat_to_ohe_cat)
                    cf.loc[0, target_name] = target_label_encoder.inverse_transform([exp.cf['class']])[0]
                # print(exp.cf)
                # print(cf)
            else:
                print("CF not found")
                cf = None

            input_df = inverse_dummy(pd.DataFrame(example, columns=ohe_feature_names), cat_to_ohe_cat)
            input_df.loc[0, target_name] = prediction

            results[k].append({
                "input": input_df,
                "cf": cf,
                'exp': exp,
                "running_time": running_time,
                "ground_truth": target_label_encoder.inverse_transform([y_test[idx]])[0],
                "prediction": prediction,
            })
all_df = {}

for k in results.keys():

    all_data = []

    for i in range(len(results[k])):
        final_df = pd.DataFrame([{}])

        scaled_input_df = results[k][i]['input'].copy(deep=True)
        origin_columns = [f"origin_input_{col}"  for col in scaled_input_df.columns]
        origin_input_df = scaled_input_df.copy(deep=True)
        scaled_input_df.columns = [f"scaled_input_{col}"  for col in scaled_input_df.columns]

        origin_input_df[numerical_cols] = scaler.inverse_transform(origin_input_df[numerical_cols])
        origin_input_df.columns = origin_columns

        final_df = final_df.join([scaled_input_df, origin_input_df])

        if not results[k][i]['cf'] is None:
            scaled_cf_df = results[k][i]['cf'].copy(deep=True)
            # scaled_cf_df.loc[0, target_name] = target_label_encoder.inverse_transform([scaled_cf_df.loc[0, target_name]])[0]
            origin_cf_columns = [f"origin_cf_{col}"  for col in scaled_cf_df.columns]
            origin_cf_df = scaled_cf_df.copy(deep=True)
            scaled_cf_df.columns = [f"scaled_cf_{col}"  for col in scaled_cf_df.columns]

            origin_cf_df[numerical_cols] = scaler.inverse_transform(origin_cf_df[numerical_cols])
            origin_cf_df.columns = origin_cf_columns

            final_df = final_df.join([scaled_cf_df, origin_cf_df])

        # final_df = final_df.join([scaled_input_df, origin_input_df, scaled_cf_df, origin_cf_df])
        final_df['running_time'] = results[k][i]['running_time']
        final_df['Found'] = "Y" if not results[k][i]['cf'] is None else "N"
        final_df['ground_truth'] = results[k][i]['ground_truth'] 
        final_df['prediction'] = results[k][i]['prediction'] 

        all_data.append(final_df)

    all_df[k] = pd.concat(all_data)
for df_k in all_df.keys():
    all_df[df_k].to_csv(f"./results/proto_compas_{df_k}_result.csv")<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.12228508851980288, 'fraction_numerical': 0.004745391494798321, 'mean_word_length': 2.643617021276596, 'pattern_counts': {'":': 8, '<': 5, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '13354', 'license': 'MIT', 'n_tokens_mistral': 1892, 'n_tokens_neox': 1811, 'n_words': 284, 'path': 'deprecated/cfproto_compas_dataset_test.ipynb', 'repo_name': 'LeonChou5311/Counterfactual-benchmark', 'size': '5479'}