<jupyter_start><jupyter_text>Model Validation Testdata set $(x,y)^d$, where $y_i=\{1, 0\}^L$we want to map y into lower space by $$z = [M\cdot y]$$ where M is a multivariant i,i,d Gaussian matrix, and $[]$ is tkaing the sign.Then we train binary classifiers on each bit of $z \in \{0, 1\}^{\hat L}$For each test point, we predict its $\hat z$ and then use kNN to find the nearest k neighbors from $z=[My]$ which is all our lower degree space's mapping. Target:* Test relationship between p@k in $y \in \{0, 1\}^L$ and bit flip probability in $z \in \{0, 1\}^{\hat L}$ space* ...  Data Preprocess<jupyter_code>%matplotlib inline
import math
import os
import data_util
import BMapModel
#from data_util import DataPoint
import matplotlib.pyplot as plt
import numpy as np
import pandas as pd
import seaborn as sns
import faiss
import util
import scipy
# import joblib # version incompatibel with sklearn's joblib and can't load the previous model

from scipy.sparse import save_npz, load_npz
from scipy import sparse
from sklearn.externals import joblib # store classifiers
from sklearn.preprocessing import MultiLabelBinarizer # convert y to {0,1}^L
from sklearn.preprocessing import StandardScaler # normalize features 
from sklearn.feature_extraction import DictVectorizer # extract feature vector to x
from numpy.random import normal # generate transforming matrix
from sklearn.neighbors import KDTree #KDTree for fast kNN search
from sklearn.linear_model import LogisticRegression
from sklearn.svm import LinearSVC
from sklearn.feature_selection import SelectPercentile
from sklearn.feature_selection import VarianceThreshold
from sklearn.metrics import average_precision_score
from joblib import Parallel, delayed # Multitread
from pytictoc import TicToc
from sklearn.ensemble import AdaBoostClassifier
from sklearn.ensemble import RandomForestClassifier
!ls -R ../data
ls ../data/Wiki10/
data_dir = "../data"
model_dir = "../model/model6"

path = '/Delicious'
model_path = model_dir + path
data_path = data_dir + path
num_core = -1
L_hat_ratio = 0.5
L_hat = 100
time = TicToc()
[X_tr, X_te, Y_tr, Y_te] = [load_npz(os.path.join(data_path, '{}.npz'.format(name)))\
                            for name in ['X_tr', 'X_te', 'Y_tr', 'Y_te']]
X_tr.shape, X_te.shape, Y_tr.shape, Y_te.shape<jupyter_output><empty_output><jupyter_text>Step 1: map to $\hat L$ space and kNN search indexWe apply $$\hat L = k \log L$$ where $k$ indicates the sparsity of each label vector $y_i = \{0,1\}^L$. By default we choose k to be the 99.9% maximum sparsity to avoid extreme cases.The data in "Eurlex" contains $L = 5000$ labels, we are trying to map it into $\hat L = 200$ space.<jupyter_code>k = sorted([Y.sum() for Y in Y_tr], reverse=True)[int(X_tr.shape[0]*0.0001)]
k
#L_hat = int(math.ceil(k * math.log(Y_tr.shape[1], 2) * L_hat_ratio))
L_hat
Z_tr = util.map_2_z(Y_tr, L_hat)
Z_te = util.map_2_z(Y_te, L_hat)
nn_index = faiss.index_factory(Z_tr.shape[1], "Flat", faiss.METRIC_L2)   # build the index
nn_index.add(Z_tr.astype('float32'))<jupyter_output><empty_output><jupyter_text>Experiment 3: Find the import bits in Z to protect* randomly find 10 bits in Z to flip and then use kNN to evaluate p@k score, record group index.* plot the distribution of p@k to see if $\exists$ someone that needs protection a lot.<jupyter_code>def flip_ind(message, index):
    '''
    Flip the the index bits in every entry in message
    input: 
        message: 2D array
        index: 1D array
    return:
        2D array with same size of message
    '''
    fliped = message.copy()
    fliped[:, index] = 1-message[:, index]
    return fliped
def evalutate_z(z_pred, vote, pk):
    '''
    evaluate precision socre of Y recovered by z_pred
    '''
    dist, ind = nn_index.search(z_pred.astype('float32'), vote)
    Y_pred =  [np.sum([Y_tr[indij]/float(distij*distij+0.01) \
                       for indij, distij in zip(indi, disti)]) \
               for indi, disti in zip(ind,dist)]
    return util.precision_at_k(Y_te, Y_pred)
def simulation_flip_labels(bits=10, vote=30, pk=1):
    index = np.random.randint(low=0, high=Z_te.shape[1], size=10)
    z_pred = flip_ind(Z_te, index)
    return index, evalutate_z(z_pred, vote, pk)
pk=1
result = Parallel(n_jobs=num_core)\
                    (delayed(simulation_flip_labels)(bits=10, vote=30, pk=pk) for _ in range(10000))
index, precision = zip(*result)
index = np.array(index)
precision = np.array(precision)
sns.distplot(precision)
plt.xlabel('precision @{}'.format(pk))
plt.ylabel('density')
plt.title('density of p@{} when we flip 10 entries'.format(pk))<jupyter_output><empty_output><jupyter_text>Experiment 2: Simulate noisy model to see the change w.p.t p@k on y* randomly flip every bit in Z_tr w/ prob p and then use kNN to test p@1, simulation 1000 times for every point* randomly flip every "1" w/ prob p1, and every "0" w/ p0 in Z_tr, ...<jupyter_code>def flip_bits(message, p0, p1):
    '''
    randomly flip every "1" w/ prob p1, and every "0" w/ p0
    '''
    def flip(bit):
        if bit==1 and np.random.rand()<p1:
            bit = 0
        if bit==0 and np.random.rand()<p0:
            bit=1
        return bit
    np.random.seed(0)
    return np.apply_along_axis(lambda bits: np.array([flip(bit) for bit in bits]), 0, message)

def simulation_p_at_k(p0, p1, vote=20, pk=1, iteration=10):
    precision = []
    for _ in range(iteration):
        z_pred = flip_bits(Z_te, p0, p1)
        precision.append(evalutate_z(z_pred, vote, pk))
    return np.mean(precision)
p_at_k = Parallel(n_jobs=num_core)\
    (delayed(simulation_p_at_k)(j, i) for j in np.arange(0, 0.6, 0.1) for i in np.arange(0, 0.6, 0.1))
p_at_k_2d = np.array(p_at_k).reshape(6, 6)
x_grid = np.arange(0, 0.6, 0.1)
x_grid, y_grid = np.meshgrid(x_grid, x_grid)
fig = plt.figure()
ax = fig.gca(projection='3d')
ax.plot_surface(x_grid, y_grid, p_at_k_2d)
ax.view_init(30, 70)
plt.xlabel('p1')
plt.ylabel('p0')
plt.title('bit flip prob vs p@1')
p_at_k_1d = Parallel(n_jobs=num_core)\
    (delayed(simulation_p_at_k)(j, j, pk=1) for j in np.arange(0, 0.6, 0.1))
p_at_ks = []
for pk in np.arange(1,6,2):
    p_at_ks.append(Parallel(n_jobs=num_core)\
        (delayed(simulation_p_at_k)(j, j, pk=pk) for j in np.arange(0, 0.6, 0.1)))
for i in range(len(p_at_ks)):
    plt.plot(p_at_ks[i], label='p@{}'.format(2*i+1))
plt.legend()
plt.title('{}'.format(data_path))
plt.ylabel('p@k')
plt.xlabel('bit flip probability')
plt.plot(p_at_k_1d)
plt.title('{}'.format(data_path))
plt.ylabel('p@1')
plt.xlabel('bit flip probability')<jupyter_output><empty_output><jupyter_text>Experiment 1: test Binary Eraseral Model* Use Random Forest classifier to predict a probability* Threshold the probability into 0, 1, * (miss).* calculate erasure rate and bit flip probability<jupyter_code>model = BMapModel.BM_Predictor(Y_tr.shape[1], L_hat=1, index=nn_index, 
                               Y_tr=Y_tr, model_path=model_path)
z_pred_p = model.predict_prob_z(X_te)
z_pred = model.predict_z(X_te)
erasure_rate = []
bit_flip_rate = []
for margin in np.arange(0.1, 0.5, 0.01):
    z_threshold = np.apply_along_axis(lambda row: [0 if x < margin else 1 if x > 1-margin else np.nan for x in row], 
                                  axis=0, arr=z_pred_p)
    erasure_rate.append(np.isnan(z_threshold).sum()/float(z_threshold.shape[0] * z_threshold.shape[1]))
    bit_flip_rate.append(1-((z_threshold == Z_te).sum()/
                         float(z_threshold.shape[0] * z_threshold.shape[1]-np.isnan(z_threshold).sum())))
plt.plot(np.arange(0.1, 0.5, 0.01), erasure_rate, label='erasure rate')
plt.plot(np.arange(0.1, 0.5, 0.01), bit_flip_rate, label='bit flip rate in remain bits')
plt.xlabel('the noise margin')
plt.ylabel('probability')
plt.title('Binary Erasure Channel of {}'.format(data_path))
plt.legend()
z_pred_t = np.apply_along_axis(lambda row: [0 if x <= 0.5 else 1 if x > 0.5 else np.nan for x in row], 
                    axis=0, arr=z_pred_p)
1-(Z_te == z_pred_t).sum()/float(z_threshold.shape[0] * z_threshold.shape[1])<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1295767655454884, 'fraction_numerical': 0.02519548218940052, 'mean_word_length': 4.096774193548387, 'pattern_counts': {'":': 0, '<': 25, '<?xml version=': 0, '>': 23, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '5', 'dir': 'jupyter-structured-clean-dedup', 'id': '622898', 'license': 'MIT', 'n_tokens_mistral': 3116, 'n_tokens_neox': 2957, 'n_words': 834, 'path': 'src/Model Validation Test (BMap+kNN+classifiers).ipynb', 'repo_name': 'Horacehxw/Multi-label', 'size': '8057'}