<jupyter_start><jupyter_text>import library<jupyter_code>import sys
sys.path.append('../../')
import pandas as pd
import numpy as np
import os
import math
import matplotlib.pyplot as plt
from pathlib import Path
from sklearn.datasets import load_svmlight_file
import torch
from mlinterpreter.mmd.mmd_critic import Dataset, select_prototypes, select_criticisms
from tqdm import tqdm, tqdm_notebook
from sklearn.preprocessing import StandardScaler
from sklearn import preprocessing
def load_data(path, target):
    df = pd.read_csv(path)
    df_origin = df.copy()
    
    num_cols = []
    cat_cols = []

    for col in df.columns:
        if col in [target]:
            continue
        if (df[col].dtype == 'object'):
            cat_cols.append(col)
        else:
            num_cols.append(col)
    
    for col in num_cols:
        scaler = StandardScaler()
        scaler.fit(df[col].values.reshape(-1, 1))

        df[col] = scaler.transform(df[col].values.reshape(-1, 1))
    
    
    for f in cat_cols:
        lbl = preprocessing.LabelEncoder()
        print(f, df[f].nunique())
        lbl.fit(df[f])
        df[f] = lbl.transform(list(df[f].astype(str)))
    
    not_used = [target]
    used_features = [x for x in df.columns if x not in not_used]
    X = df[used_features]
    y = df[target]
    
    X = torch.tensor(np.array(X), dtype=torch.float)
    y = torch.tensor(np.array(y), dtype=torch.long)
    
    return X, y, df_origin<jupyter_output><empty_output><jupyter_text>parameter setting<jupyter_code>gamma = 0.026

num_prototypes = 32
num_criticisms = 10

kernel_type = 'local'
# kernel_type = 'global'

# regularizer = None
regularizer = 'logdet'
# regularizer = 'iterative'

path = '../../demo_data/adult.csv'
target = 'income'<jupyter_output><empty_output><jupyter_text>model<jupyter_code>X_train, y_train, df_origin = load_data(path, target)

d_train = Dataset(X_train, y_train)
if kernel_type == 'global':
    d_train.compute_rbf_kernel(gamma)
elif kernel_type == 'local':
    d_train.compute_local_rbf_kernel(gamma)
else:
    raise KeyError('kernel_type must be either "global" or "local"')
print('Done.', flush=True)

# Prototypes
if num_prototypes > 0:
    print('Computing prototypes...', end='', flush=True)
    prototype_indices = select_prototypes(d_train.K, num_prototypes)

    prototypes = d_train.X[prototype_indices]
    prototype_labels = d_train.y[prototype_indices]

    sorted_by_y_indices = prototype_labels.argsort()
    prototypes_sorted = prototypes[sorted_by_y_indices]
    prototype_labels = prototype_labels[sorted_by_y_indices]
    print('Done.', flush=True)
    print(prototype_indices.sort()[0].tolist())

    # Criticisms
    if num_criticisms > 0:
        print('Computing criticisms...', end='', flush=True)
        criticism_indices = select_criticisms(d_train.K, prototype_indices, num_criticisms, regularizer)

        criticisms = d_train.X[criticism_indices]
        criticism_labels = d_train.y[criticism_indices]

        sorted_by_y_indices = criticism_labels.argsort()
        criticisms_sorted = criticisms[sorted_by_y_indices]
        criticism_labels = criticism_labels[sorted_by_y_indices]
        print('Done.', flush=True)
        print(criticism_indices.sort()[0].tolist())<jupyter_output>workclass 9
education 16
marital.status 7
occupation 15
relationship 6
race 5
sex 2
native.country 42
Done.
Computing prototypes...Done.
[193, 2147, 3862, 4426, 4908, 5611, 6808, 6899, 7814, 8247, 8320, 9393, 10440, 12257, 12778, 16270, 16414, 16520, 17719, 20478, 20970, 21650, 22147, 22856, 24101, 25749, 26317, 27196, 27840, 28968, 30736, 32004]
Computing criticisms...Done.
[2094, 6446, 8370, 10324, 15777, 23747, 24873, 26398, 26434, 27814]<jupyter_text>prototype<jupyter_code>df_origin.loc[prototype_indices, :]<jupyter_output><empty_output><jupyter_text>criticism<jupyter_code>df_origin.loc[criticism_indices, :]<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.11503067484662577, 'fraction_numerical': 0.05649284253578732, 'mean_word_length': 4.121727748691099, 'pattern_counts': {'":': 0, '<': 20, '<?xml version=': 0, '>': 22, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '5', 'dir': 'jupyter-structured-clean-dedup', 'id': '346728', 'license': 'Apache-2.0', 'n_tokens_mistral': 1561, 'n_tokens_neox': 1381, 'n_words': 282, 'path': 'autox/autox_interpreter/interpreter_demo/prototypes_and_criticisms/MMD_demo.ipynb', 'repo_name': 'fanghy06/AutoX', 'size': '3912'}