<jupyter_start><jupyter_text>Dynamic Feature Acquisition Using Denoising Autoencoders<jupyter_code>import pdb
import math
import os
import time
import copy
import pickle

import numpy as np
import scipy
import scipy.io
import pandas as pd
import sklearn.preprocessing
import sklearn.metrics
import sklearn.cluster
import sklearn.feature_selection
import sklearn.ensemble
import sklearn.svm
from mpl_toolkits.mplot3d import Axes3D
import matplotlib.gridspec
import matplotlib.pyplot as plt
import tensorflow as tf
from IPython import embed

import src.datasets
import src.nn_training<jupyter_output><empty_output><jupyter_text>Settings<jupyter_code>%load_ext autoreload
%autoreload 2

#%pdb
%matplotlib notebook
np.random.seed(1)

os.environ['CUDA_VISIBLE_DEVICES'] = ''

PLOT = False

DIR_SUMMARY = './tf_logs'
DIR_MODEL = os.popen('mktemp -d').read().rstrip()+'/'
os.system('rm -r ' + DIR_SUMMARY)
os.system('rm -r ' + DIR_MODEL)
os.system('mkdir ' + DIR_MODEL)

#synthesized', 'mnist'
DATASET = 'synthesized'
RANDSEL_RUNS = 1
STATIC_ALG = 'MI' #'SVM', 'MI', 'none'
N_BITS = 8
OPTIMIZER = tf.train.AdamOptimizer
LR_BASE_ENC = 0.001
LR_BASE = 0.001


if DATASET == 'synthesized':
    # data generation
    opt = {
    'n_features' : 32,
    'n_clusters' : 16,
    'n_clusterpoints' :1000, 
    'std_clusters' : 0.25,
    'cost-aware' : True,
    }
    # ENCODER
    SIZE_LAYERS_ENCODER = [16,10]
    ITER_EARLYSTOP_ENC = 3
    ITERS_MAX_ENC = 20000
    # NN
    EXPERIMENT = 'real_mask'
    SIZE_HIDDENS = [8,4]# [8,4]
    ITER_EARLYSTOP = 20#5
    ITERS_MAX = 100000
    # MISSING VALUES
    MISSING_PORTION = (1.5,1.5)
    # load data
    ds = src.datasets.Dataset()
    ds.load('synthesized', opt)
    ds.preprocess(normalization='unity', fe_std_threshold=0.0)
    ds_dict = ds.get(order='rand', onehot=True)
    dataset_features = ds_dict['features']
    dataset_targets = ds_dict['targets']
    dataset_mask = ds_dict['mask']    
    dataset_costs = ds_dict['costs']
    

elif DATASET == 'mnist':
    # task
    opt = {'task':'multires'}
    # ENCODER
    SIZE_LAYERS_ENCODER = [64,32]
    ITER_EARLYSTOP_ENC = 10
    ITERS_MAX_ENC = 10000
    # NN
    EXPERIMENT = 'real_mask'
    SIZE_HIDDENS = [16]
    ITER_EARLYSTOP = 10
    ITERS_MAX = ITERS_MAX_ENC
    # MISSING VALUES
    MISSING_PORTION = (3.5,1.5) # NEW (1.5,1.5)
    # load data
    ds = src.datasets.Dataset()
    ds.load('mnist', opt)
    ds.preprocess(normalization='unity', fe_std_threshold=1.0e-3)
    ds_dict = ds.get(order='rand', onehot=True)
    dataset_features = ds_dict['features']
    dataset_targets = ds_dict['targets']
    dataset_mask = ds_dict['mask']
    dataset_costs = ds_dict['costs']<jupyter_output><empty_output><jupyter_text>Network Definition<jupyter_code>def net_mlp(input_features, 
            size_hiddens, size_output, 
            ):
    """
    Network definition
    """
    # create the net input layer
    with tf.name_scope('input'):
        layer_activations = input_features
        
    # hidden layers
    for size_hid in size_hiddens:
        with tf.name_scope('hidden_'+str(size_hid)):
            size_before = int(layer_activations.get_shape()[1])
            weights = tf.Variable(
                tf.truncated_normal(
                    [size_before, size_hid],
                    stddev=1.0 / math.sqrt(float(size_before))),
                name='weights')
            biases = tf.Variable(tf.zeros([size_hid])+0.1, name='biases')
            layer_activations = tf.nn.relu(
                tf.matmul(layer_activations, weights) + biases)
            
    # output layer
    with tf.name_scope('output'):
        size_before = int(layer_activations.get_shape()[1])
        weights = tf.Variable(
            tf.truncated_normal(
                [size_before, size_output],
                stddev=1.0 / math.sqrt(float(size_before))),
            name='weights')
        biases = tf.Variable(tf.zeros([size_output])+0.0, name='biases')
        layer_activations = tf.matmul(layer_activations, weights) + biases
        output_prediction = layer_activations
    
    return output_prediction

def loss_cross_entropy(preds, labels):
    cross_entropy = tf.nn.softmax_cross_entropy_with_logits(
        labels=labels, logits=preds, name='xentropy')
    return tf.reduce_mean(cross_entropy, name='xentropy_mean')<jupyter_output><empty_output><jupyter_text>Autoencoder Definition<jupyter_code>def net_autoencoder_bin(input_features, size_layers_encoder, 
                    input_mask=None, algorithm='nomask'):
    """
    Network definition
    """
    # Build the encoding layers
    # create the net input layer
    with tf.name_scope('AE_input'):
        layer_activations = input_features
    
    # binary layer
    with tf.name_scope('AE_input_bin'):
        layer_encoder_bin = encode_binary(input_features)
        layer_activations = layer_encoder_bin
    # hidden encoder layers
    enc_weights = []
    for size_hid in size_layers_encoder:
        with tf.name_scope('AE_hidden_encoder_'+str(size_hid)):
            size_before = int(layer_activations.get_shape()[1])
            weights = tf.Variable(
                tf.truncated_normal(
                    [size_before, size_hid],
                    stddev=1.0 / math.sqrt(float(size_before))),
                name='weights')
            biases = tf.Variable(tf.zeros([size_hid])+0.1, name='biases')
            layer_activations = tf.nn.relu(#sigmoid(
                tf.matmul(layer_activations, weights) + biases)
            enc_weights.append(weights)
    
    # extract a reference to the encoder
    encoded = layer_activations
    
    # hidden decoder layers
    for (size_hid,weights_hid) in \
        zip(size_layers_encoder[::-1], enc_weights[::-1]):
        with tf.name_scope('AE_hidden_decoder_'+str(size_hid)):
            size_before = int(layer_activations.get_shape()[1])
            size_after = int(weights_hid.get_shape()[0])
            weights = tf.transpose(weights_hid)
            biases = tf.Variable(tf.zeros([size_after])+0.1, name='biases')
            if size_after != int(layer_encoder_bin.get_shape()[1]):
                layer_activations = tf.nn.relu(#sigmoid(
                    tf.matmul(layer_activations, weights) + biases)
            else:
                layer_activations = tf.nn.sigmoid(
                    tf.matmul(layer_activations, weights) + biases, 
                    name='decoder_bin')
                layer_decoder_bin = layer_activations
    
    # the output decimal layer
    #layer_decoder_bin = layer_activations
    layer_activations = decode_binary(layer_decoder_bin)
    
    # extract a reference to the decoder
    decoded = layer_activations
    
    return {
        'encoded': encoded,
        'decoded': decoded,
        'cost_internal' : tf.sqrt(tf.reduce_mean(tf.square(input_features-decoded))), 
        'encoder_bin':layer_encoder_bin,
        'decoder_bin':layer_decoder_bin
            }


def loss_mse(preds, targets):
    mse = tf.sqrt(tf.reduce_mean(tf.square(preds-targets)), 
                  name='cost_mse')
    return tf.reduce_mean(mse, name='cost_mse_mean')


def loss_crossentropy_bin(preds_bin, targets_bin):
    ce_terms = -1.0 * (tf.multiply(targets_bin, tf.log(preds_bin+1.0e-10)) + \
                      tf.multiply(1-targets_bin, tf.log(1-preds_bin+1.0e-10)))
    #ce_terms = -1.0 * (tf.multiply(targets_bin, tf.log(tf.clip_by_value(preds_bin,1e-10,1.0))) + \
    #                  tf.multiply(1-targets_bin, tf.log(tf.clip_by_value(1-preds_bin,1e-10,1.0))))
    ce_costs = decode_binary(ce_terms)
    return tf.reduce_mean(ce_costs, name='cost_crossentropy_bin')


def loss_mse_bin(preds_bin, targets_bin):
    ce_terms = tf.pow(preds_bin-targets_bin, 2)
    ce_costs = decode_binary(ce_terms)
    return tf.reduce_mean(ce_costs, name='cost_mse_bin')


def encode_binary_vect(input_vector, n_bits=N_BITS):
    n_fe = int(input_vector.get_shape()[0])
    # allocate memory
    bitmat = []
    for ind_bit in np.arange(0, n_bits, 1):
        curr_bits = tf.floor(input_vector/2.0**(-ind_bit))
        input_vector -= curr_bits * (2.0**(-ind_bit))
        bitmat.append(tf.reshape(curr_bits,(-1,1)))
    bitmat = tf.concat(bitmat, axis=1)
    # return the result
    encoded_vect = tf.reshape(bitmat,(-1,))
    return encoded_vect


def decode_binary_vect(input_vector, n_bits=N_BITS):
    def decode_word(input_element, n_bits=n_bits):
        base_weights = tf.constant(
            2.0** np.arange(0,-n_bits,-1).astype(np.float32), shape=(n_bits,1))
        curr_element = tf.matmul(tf.reshape(input_element,(1,N_BITS)),base_weights)
        return curr_element
        
    n_fe = int(input_vector.get_shape()[0]) // n_bits
    base_weights = tf.constant(
            2.0** np.arange(0,-n_bits,-1).astype(np.float32), shape=(n_bits,1))
    decoded_vector = tf.matmul(
        tf.stack(tf.split(input_vector, n_fe)), base_weights)
    decoded_vector = tf.stack(decoded_vector, axis=0)
    return decoded_vector


def encode_binary(input_matrix, n_bits=N_BITS):        
    n_fe = int(input_matrix.get_shape()[1])
    encoded_matrix = tf.map_fn(encode_binary_vect, input_matrix, 
                              parallel_iterations=n_bits)
    return encoded_matrix


def decode_binary(input_matrix, n_bits=N_BITS):        
    n_fe = int(input_matrix.get_shape()[1]) // n_bits
    decoded_matrix = tf.map_fn(decode_binary_vect, input_matrix, 
                               parallel_iterations=n_bits)
    decoded_matrix = decoded_matrix[:,:,0]
    return decoded_matrix<jupyter_output><empty_output><jupyter_text>Encoder Training<jupyter_code># create as tf session
sess = tf.Session()

# instantiate the graph and its inputs, etc.
# place-holders
ph_input_features = tf.placeholder("float", [None, dataset_features.shape[1]])
ph_input_features_full = tf.placeholder("float", [None, dataset_features.shape[1]])
ph_input_mask = tf.placeholder("float", [None, dataset_features.shape[1]])
ph_output_targets = tf.placeholder("float", [None, dataset_targets.shape[1]])

# net autoencoder
nn_autoencoder = net_autoencoder_bin(
    ph_input_features, size_layers_encoder=SIZE_LAYERS_ENCODER)

if False:
    nn_autoencoder_cost = loss_mse(preds=nn_autoencoder['decoded'], 
                                   targets=ph_input_features_full)
elif True:
    nn_autoencoder_cost = loss_crossentropy_bin(
        preds_bin=nn_autoencoder['decoder_bin'], 
        targets_bin=encode_binary(ph_input_features_full))
elif False:
    nn_autoencoder_cost = loss_mse_bin(
        preds_bin=nn_autoencoder['decoder_bin'], 
        targets_bin=encode_binary(ph_input_features_full))    


# net predictor
nn_predictor = net_mlp(nn_autoencoder['encoded'], 
                       SIZE_HIDDENS, dataset_targets.shape[1])
nn_predictor_cost = loss_cross_entropy(preds=nn_predictor, labels=ph_output_targets)

# create an optimizer
train_step_ae = OPTIMIZER(learning_rate=LR_BASE_ENC).minimize(nn_autoencoder_cost)

# initial the graph
init = tf.global_variables_initializer()
sess.run(init)

# summaries
tf.summary.scalar('autoencoder_cost', nn_autoencoder_cost)
summary_merged = tf.summary.merge_all()
writer_train = tf.summary.FileWriter(DIR_SUMMARY + '/train',
                                      sess.graph)
# iterate and optimize
iters_cnt = []
costs_trn = []
costs_val = []
costs_tst = []
cost_val_previous = 1.0e12
iter_val_noimprove = 0

for cnt_iter in range(ITERS_MAX_ENC):
    # create feed data
    feed_dict_trn = src.nn_training.feed_data(ph_input_features, ph_input_mask, 
                              ph_output_targets, ph_input_features_full,
                              dataset_features, dataset_targets, 
                              phase='train', experiment=EXPERIMENT,
                              size_batch=128, missing_portion = MISSING_PORTION, 
                              seed=None)
    # do an optimization step
    sess.run(train_step_ae, feed_dict=feed_dict_trn)
    # each N iters calculate the train/validation/test costs
    if cnt_iter%250 == 0:
        summary = sess.run(summary_merged, feed_dict=feed_dict_trn)
        writer_train.add_summary(summary, cnt_iter)
        feed_dict_val = src.nn_training.feed_data(ph_input_features, ph_input_mask, 
                                  ph_output_targets, ph_input_features_full,
                                  dataset_features, dataset_targets, 
                                  phase='validation', experiment=EXPERIMENT, 
                                  size_batch=2048, missing_portion = MISSING_PORTION, 
                                  seed=None)
        feed_dict_tst = src.nn_training.feed_data(ph_input_features, ph_input_mask, 
                                  ph_output_targets, ph_input_features_full,
                                  dataset_features, dataset_targets, 
                                  phase='test', experiment=EXPERIMENT, 
                                  size_batch=2048, missing_portion = MISSING_PORTION, 
                                  seed=None)
        cost_trn = sess.run(nn_autoencoder_cost, feed_dict=feed_dict_trn)
        cost_val = sess.run(nn_autoencoder_cost, feed_dict=feed_dict_val)
        cost_tst = sess.run(nn_autoencoder_cost, feed_dict=feed_dict_tst)
        iters_cnt.append(cnt_iter)
        costs_trn.append(cost_trn)
        costs_val.append(cost_val)
        costs_tst.append(cost_tst)
        print('cnt_iter: ' + str(cnt_iter) + ', cost_trn: ' + str(cost_trn)  + \
              ', cost_val: ' + str(cost_val) + ', cost_tst: ' + str(cost_tst), 
              end='\r')
        
        # check early stop condition
        if ITER_EARLYSTOP_ENC < iter_val_noimprove:
            break
        # if no improvement increase the counter
        if cost_val > cost_val_previous:
            iter_val_noimprove += 1
        else:
            iter_val_noimprove = 0
        cost_val_previous = cost_val

# save the encoder model
saver = tf.train.Saver()
saver.save(sess, DIR_MODEL+'autoencoder')

# report the missing reconstruction errors
err_mis_base = feed_dict_tst[ph_input_features] - \
    feed_dict_tst[ph_input_features_full]
err_mis_base = np.mean(err_mis_base**2.0)
err_mis_rec = feed_dict_tst[ph_input_features_full] - \
    sess.run(nn_autoencoder['decoded'], feed_dict=feed_dict_tst)
err_mis_rec = np.mean(err_mis_rec**2.0)
print('')
print('Error Missing Reconstruction Baseline: ' + str(err_mis_base))
print('Error Missing Reconstruction MissingNet: ' + str(err_mis_rec))
print('Error Reduction Rate: ' + str((err_mis_base-err_mis_rec)/err_mis_base))

plt.figure()
plt.plot(iters_cnt, costs_trn)
plt.plot(iters_cnt, costs_val)
plt.plot(iters_cnt, costs_tst)
plt.legend(['costs_trn','costs_val','costs_tst'])<jupyter_output><empty_output><jupyter_text>Predictor Network Training<jupyter_code>lr_enc = LR_BASE_ENC
lr_mlp = LR_BASE
# create an optimizer
vars_enc = []
vars_mlp = []
for var in tf.trainable_variables():
    if var.name[:2] == 'AE':
        vars_enc.append(var)
    else:
        vars_mlp.append(var)

try:
    train_step_enc = OPTIMIZER(learning_rate=lr_enc).minimize(
        nn_predictor_cost, var_list=vars_enc)
    train_step_mlp = OPTIMIZER(learning_rate=lr_mlp).minimize(
        nn_predictor_cost, var_list=vars_mlp)
except:
    pass
    
if lr_enc != 0.0:
    train_op = tf.group(train_step_enc, train_step_mlp)
else:
    train_op = train_step_mlp


# initialize mlp vars
initmlp_op = tf.variables_initializer(vars_mlp)
sess.run(initmlp_op)

# initialize uninitialized vars
variables = tf.global_variables()
init_flag = sess.run([tf.is_variable_initialized(v) for v in variables])
vars_uninit = [v for v, f in zip(variables, init_flag) if not f]
sess.run(tf.variables_initializer(vars_uninit))

# iterate and optimize
iters_cnt = []
costs_trn = []
costs_val = []
costs_tst = []
cost_val_previous = 1.0e12
iter_val_noimprove = 0

for cnt_iter in range(ITERS_MAX):
    # create feed data
    feed_dict_trn = src.nn_training.feed_data(ph_input_features, ph_input_mask, 
                              ph_output_targets, ph_input_features_full,
                              dataset_features, dataset_targets, 
                              phase='train', experiment=EXPERIMENT,
                              size_batch=128, missing_portion = MISSING_PORTION, 
                              seed=None)
    # do an optimization step
    sess.run(train_op, feed_dict=feed_dict_trn)
    # each N iters calculate the train/validation/test costs
    if cnt_iter%250 == 0:
        feed_dict_val = src.nn_training.feed_data(ph_input_features, ph_input_mask, 
                              ph_output_targets, ph_input_features_full,
                              dataset_features, dataset_targets, 
                              phase='validation', experiment=EXPERIMENT,
                              size_batch=2048, missing_portion = MISSING_PORTION, 
                              seed=None)
        feed_dict_tst = src.nn_training.feed_data(ph_input_features, ph_input_mask, 
                              ph_output_targets, ph_input_features_full,
                              dataset_features, dataset_targets, 
                              phase='test', experiment=EXPERIMENT,
                              size_batch=2048, missing_portion = MISSING_PORTION, 
                              seed=None)
        cost_trn = sess.run(nn_predictor_cost, feed_dict=feed_dict_trn)
        cost_val = sess.run(nn_predictor_cost, feed_dict=feed_dict_val)
        cost_tst = sess.run(nn_predictor_cost, feed_dict=feed_dict_tst)
        iters_cnt.append(cnt_iter)
        costs_trn.append(cost_trn)
        costs_val.append(cost_val)
        costs_tst.append(cost_tst)
        print('cnt_iter: ' + str(cnt_iter) + ', cost_trn: ' + str(cost_trn)  + \
              ', cost_val: ' + str(cost_val) + ', cost_tst: ' + str(cost_tst), 
              end='\r')
        
        # check early stop condition
        if ITER_EARLYSTOP < iter_val_noimprove:
            break
        # if no improvement increase the counter
        if cost_val > cost_val_previous:
            iter_val_noimprove += 1
        else:
            iter_val_noimprove = 0
            cost_val_previous = cost_val

# calculate the test accuracy
preds_tst = sess.run(nn_predictor, feed_dict=feed_dict_tst)
preds_tst = np.argmax(preds_tst, axis=1)
accu_tst = np.mean(
    np.argmax(feed_dict_tst[ph_output_targets], axis=1)==preds_tst)

print('')
print('The Accuracy of Test is: ' + str(accu_tst*100))
plt.figure()
plt.plot(iters_cnt, costs_trn)
plt.plot(iters_cnt, costs_val)
plt.plot(iters_cnt, costs_tst)
plt.legend(['costs_trn','costs_val','costs_tst'])<jupyter_output><empty_output><jupyter_text>Compare Random and Sensitivity-based Feature Selection<jupyter_code># load the trained autoencoder
sess_enc = tf.Session()
saver_enc = tf.train.import_meta_graph(DIR_MODEL+'autoencoder.meta')
saver_enc.restore(sess_enc, tf.train.latest_checkpoint(DIR_MODEL))

# deep copy
feed_dict_trn = src.nn_training.feed_data(ph_input_features, ph_input_mask, 
                      ph_output_targets, ph_input_features_full,
                      dataset_features, dataset_targets, 
                      phase='train', experiment=EXPERIMENT,
                      size_batch=2048, missing_portion = 1.0, 
                      seed=None)
feed_dict_tst = src.nn_training.feed_data(ph_input_features, ph_input_mask, 
                      ph_output_targets, ph_input_features_full,
                      dataset_features, dataset_targets, 
                      phase='test', experiment=EXPERIMENT,
                      size_batch=2048, missing_portion = 1.0, 
                      seed=None)
feed_dict_rand = {k:v.copy() for k,v in feed_dict_tst.items()}
feed_dict_muinfo = {k:v.copy() for k,v in feed_dict_tst.items()}
feed_dict_sense = {k:v.copy() for k,v in feed_dict_tst.items()}

n_tst = feed_dict_rand[ph_input_features_full].shape[0]
n_fe = feed_dict_rand[ph_input_features_full].shape[1]

accus_rand = []
accus_muinfo = []
accus_sense = []
accus_base = []

cost_rand = []
cost_muinfo = []
cost_sense = []
cost_base = []

# calculate the baseline accuracy
preds_tst = sess.run(nn_predictor, feed_dict=feed_dict_tst)
preds_tst = np.argmax(preds_tst, axis=1)
accu_tst = 100*np.mean(
    np.argmax(feed_dict_tst[ph_output_targets], axis=1)==preds_tst)
accus_base.append(accu_tst)
cost_base.append(0.0)

# average performance of random selection
for iter_randsel in range(RANDSEL_RUNS):
    print('Random selection #'+str(iter_randsel), end='\r')
    # var init for each iter
    accus_rand_iter = []
    feed_dict_rand = {k:v.copy() for k,v in feed_dict_tst.items()}
    # initial accuracy
    preds_tst = sess.run(nn_predictor, feed_dict=feed_dict_rand)
    preds_tst = np.argmax(preds_tst, axis=1)
    accu_tst = 100*np.mean(
    np.argmax(feed_dict_tst[ph_output_targets], axis=1)==preds_tst)
    accus_rand_iter.append(accu_tst)
    cost_rand.append(0.0)
    # random sel
    for iter_sel in range(n_fe):
        for ind_tst in range(n_tst):
            # rand sel alg
            # select a sample
            mask_tst = feed_dict_rand[ph_input_mask][ind_tst]
            inds_missing = np.where(mask_tst==0)[0]
            if len(inds_missing) > 0:
                # query for feature
                # rand sel alg
                ind_sel_rand = np.random.choice(inds_missing)
                features_tst = feed_dict_rand[ph_input_features][ind_tst]
                feed_dict_rand[ph_input_features][ind_tst][ind_sel_rand] = \
                    feed_dict_rand[ph_input_features_full][ind_tst][ind_sel_rand]
                feed_dict_rand[ph_input_mask][ind_tst][ind_sel_rand] = 1
        # measure accuracy
        preds_tst = sess.run(nn_predictor, feed_dict=feed_dict_rand)
        preds_tst = np.argmax(preds_tst, axis=1)
        accu_tst = 100*np.mean(
            np.argmax(feed_dict_tst[ph_output_targets], axis=1)==preds_tst)
        accus_rand_iter.append(accu_tst)
        cost = np.sum(feed_dict_rand[ph_input_mask]*dataset_costs, axis=1).mean()
        cost_rand.append(cost)
    accus_rand.append(accus_rand_iter)
    
accus_rand = np.vstack(accus_rand).mean(axis=0)


# performance of static mutual info selection
print('')
print('Static mutual info selection', end='\r')
preds_tst = sess.run(nn_predictor, feed_dict=feed_dict_muinfo)
preds_tst = np.argmax(preds_tst, axis=1)
accu_tst = 100*np.mean(
    np.argmax(feed_dict_tst[ph_output_targets], axis=1)==preds_tst)
accus_muinfo.append(accu_tst)
cost_muinfo.append(0.0)

if STATIC_ALG == 'MI':
    #calculate MI
    musel = sklearn.feature_selection.mutual_info_classif(
       dataset_features[:,:], np.argmax(dataset_targets[:,:],1))
    #musel, pval = sklearn.feature_selection.chi2(
    #   dataset_features-dataset_features.min(0), np.argmax(dataset_targets,1))
    #musel = np.arange(0, len(musel), 1, dtype=np.int)

    #rfe = sklearn.feature_selection.RFE(sklearn.svm.SVR(kernel="linear"), 
    #         1, step=1)
    #rfe.fit(dataset_features[:1000], np.argmax(dataset_targets[:1000],1))
    #musel = 1.0 / rfe.ranking_
elif STATIC_ALG == 'SVM':
    clf = sklearn.svm.SVC(kernel='linear')
    clf.fit(dataset_features[:]/dataset_features.std(0), np.argmax(dataset_targets[:],1))
    musel = np.abs(clf.coef_).sum(0)
elif STATIC_ALG == 'none':
    musel = None
else:
    raise NotImplementedError

# do static feature sel 
for iter_sel in range(n_fe):
    # check if we have to skip
    if musel is None:
        print('')
        print('Static selection: SKIPPED')
        break
    else:
        # normalize musel with cost values
        feature_info = musel.copy()
        musel /= dataset_costs
    print('Static selection using: ' + STATIC_ALG + ', #FE: '+str(iter_sel), end='\r')
    # muinfo sel alg
    for ind_tst in range(n_tst):
        mask_tst = feed_dict_muinfo[ph_input_mask][ind_tst]
        inds_missing = np.where(mask_tst==0)[0]
        features_tst = feed_dict_muinfo[ph_input_features][ind_tst]
        if len(inds_missing) > 0:
            ind_sel_muinfo = inds_missing[np.argmax(musel[inds_missing])]
            feed_dict_muinfo[ph_input_features][ind_tst][ind_sel_muinfo] = \
                feed_dict_muinfo[ph_input_features_full][ind_tst][ind_sel_muinfo]
            feed_dict_muinfo[ph_input_mask][ind_tst][ind_sel_muinfo] = 1
            

    # calculate the muinfo accuracy
    preds_tst = sess.run(nn_predictor, feed_dict=feed_dict_muinfo)
    preds_tst = np.argmax(preds_tst, axis=1)
    accu_tst = 100*np.mean(
       np.argmax(feed_dict_tst[ph_output_targets], axis=1)==preds_tst)
    accus_muinfo.append(accu_tst)
    cost = np.sum(feed_dict_muinfo[ph_input_mask]*dataset_costs, axis=1).mean()
    cost_muinfo.append(cost)
print('')

# performance of sens selection
print('Sensitivity-based selection', end='\r')
preds_tst = sess.run(nn_predictor, feed_dict=feed_dict_sense)
preds_tst = np.argmax(preds_tst, axis=1)
accu_tst = 100*np.mean(
    np.argmax(feed_dict_tst[ph_output_targets], axis=1)==preds_tst)
accus_sense.append(accu_tst)
cost_sense.append(0.0)

# define the grad operation
nn_predictor_prob = tf.nn.softmax(nn_predictor)
op_grad = tf.abs(tf.gradients(nn_predictor_prob[:,0], 
                              nn_autoencoder['encoder_bin'])[0])
for ind_y in range(1,int(nn_predictor_prob.get_shape()[1])):
    op_grad = op_grad + tf.abs(
        tf.gradients(nn_predictor_prob[:,ind_y], 
                     nn_autoencoder['encoder_bin'])[0])

# probability estimation operation
op_prob = nn_autoencoder['decoder_bin']
# ground truth probabilities
ph_prob_full = tf.placeholder(dtype=tf.float64, shape=(None,n_fe))
op_prob_full = encode_binary(ph_prob_full)

"""
# sensitivity estimation operation
#op_sens = tf.multiply(op_grad, op_prob)
#op_sens = 1.0 - tf.abs(0.5 - tf.multiply(op_grad, op_prob))
#op_sens = tf.reduce_sum(tf.reshape(op_sens, (-1,n_fe,8)), axis=2)
"""

#pdb.set_trace()

# do feature sel
ptime_start = time.process_time()
selected_log = np.zeros((n_tst,n_fe), dtype=np.int) - 1 # -1 indicating invalid
for iter_sel in range(n_fe):
    #
    print('Sensitivity-based selection, #FE: '+str(iter_sel), end='\r')
    # calc the sensitivities
    #res_prob = sess.run(op_prob, feed_dict=feed_dict_sense)
    
    #res_prob = sess.run(op_prob_full, 
    #                    feed_dict={ph_prob_full:feed_dict_sense[ph_input_features_full]})
    
    res_prob = sess_enc.run(op_prob, feed_dict=feed_dict_sense)
    
    res_grad = sess.run(op_grad, feed_dict=feed_dict_sense)
    res_sens = res_prob * res_grad
    res_sens = np.sum(np.reshape(res_sens, (-1,n_fe,N_BITS)), axis=2) / \
        dataset_costs
    
    # sense sel alg
    for ind_tst in range(n_tst):
        mask_tst = feed_dict_sense[ph_input_mask][ind_tst]
        inds_missing = np.where(mask_tst==0)[0]
        features_tst = feed_dict_sense[ph_input_features][ind_tst]
        grads_tst = res_sens[ind_tst]
        if len(inds_missing) > 0:
            ind_sel_sense = inds_missing[np.argmax(grads_tst[inds_missing])]
            feed_dict_sense[ph_input_features][ind_tst][ind_sel_sense] = \
                feed_dict_sense[ph_input_features_full][ind_tst][ind_sel_sense]
            feed_dict_sense[ph_input_mask][ind_tst][ind_sel_sense] = 1
            # log it
            selected_log[ind_tst,iter_sel] = ind_sel_sense
            

    # append the test accuracy
    accus_base.append(accus_base[-1])
    cost_base.append(cost_rand[-1])
    # calculate the sens accuracy
    preds_tst = sess.run(nn_predictor, feed_dict=feed_dict_sense)
    preds_tst = np.argmax(preds_tst, axis=1)
    accu_tst = 100*np.mean(
       np.argmax(feed_dict_tst[ph_output_targets], axis=1)==preds_tst)
    accus_sense.append(accu_tst)
    cost = np.sum(feed_dict_sense[ph_input_mask]*dataset_costs, axis=1).mean()
    cost_sense.append(cost)
    
ptime_end = time.process_time()
print('')
print('Dataset :', DATASET)
print('Processing-time per sample: ' + \
      str(1000.0 / n_tst *(ptime_end - ptime_start)) + ' (ms)')

auc_rand = np.sum(np.array(accus_rand) / 100.0) / len(accus_rand)
auc_muinfo = np.sum(np.array(accus_muinfo) / 100.0) / len(accus_muinfo)
auc_sense = np.sum(np.array(accus_sense) / 100.0) / len(accus_sense)
acc_th = accus_sense[-1] * 0.95
ind_th = np.where(np.array(accus_sense) > acc_th)[0][0]
auc_rand_th = np.mean(np.array(accus_rand)[:ind_th] / 100.0) 
auc_muinfo_th = np.mean(np.array(accus_muinfo)[:ind_th] / 100.0) 
auc_sense_th = np.mean(np.array(accus_sense)[:ind_th] / 100.0) 


print('')
print('AUC Rand: ' + str(auc_rand))
print('AUC Static: ' + str(auc_muinfo))
print('AUC Sense: ' + str(auc_sense))
print('')
print('AUC Rand (th): ' + str(auc_rand_th))
print('AUC Static (th): ' + str(auc_muinfo_th))
print('AUC Sense (th): ' + str(auc_sense_th))
print('')
print('Accuracy at 0% : ' + str(accus_sense[0]))
print('Accuracy at 25% : ' + str(accus_sense[int(0.25*len(accus_sense))]))
print('Accuracy at 50% : ' + str(accus_sense[int(0.50*len(accus_sense))]))
print('Accuracy at 75% : ' + str(accus_sense[int(0.75*len(accus_sense))]))
print('Accuracy at 100% : ' + str(accus_sense[-1]))

cost_total = cost_sense[-1]
ind_25 = np.where(cost_sense > cost_total*0.25)[0][0]
ind_50 = np.where(cost_sense > cost_total*0.50)[0][0]
ind_75 = np.where(cost_sense > cost_total*0.75)[0][0]

print('')
print('Accuracy at 0%   total cost: ' + str(accus_sense[0]))
print('Accuracy at 25%  total cost: ' + str(accus_sense[ind_25]))
print('Accuracy at 50%  total cost: ' + str(accus_sense[ind_50]))
print('Accuracy at 75%  total cost: ' + str(accus_sense[ind_75]))
print('Accuracy at 100% total cost: ' + str(accus_sense[-1]))

# plot the results
plt.figure()
plt.plot(accus_base)
plt.plot(accus_rand)
plt.plot(accus_muinfo)
plt.plot(accus_sense)
plt.legend(['Base', 'RandSel', 'Static', 'FACT'])
plt.xlabel('Features Acquired')
plt.ylabel('Accuracy')
plt.savefig('./run_outputs/learn_curve_' + DATASET + '.pdf')

plt.figure()
plt.plot(cost_base, accus_base)
plt.plot(cost_rand, accus_rand)
plt.plot(cost_muinfo, accus_muinfo)
plt.plot(cost_sense, accus_sense)
plt.legend(['Base', 'RandSel', 'Static', 'FACT'])
plt.xlabel('Acquisition Cost')
plt.ylabel('Accuracy')
plt.savefig('./run_outputs/cost_curve_' + DATASET + '.pdf')<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.11465662471509265, 'fraction_numerical': 0.01427014171043504, 'mean_word_length': 3.2205492820298343, 'pattern_counts': {'":': 0, '<': 31, '<?xml version=': 0, '>': 38, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 12, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '7', 'dir': 'jupyter-structured-clean-dedup', 'id': '627055', 'license': 'MIT', 'n_tokens_mistral': 11287, 'n_tokens_neox': 10720, 'n_words': 1672, 'path': 'Demo.ipynb', 'repo_name': 'mkachuee/DynamicFeatureSelection', 'size': '30273'}