<jupyter_start><jupyter_text>In this kernel, I implement vectorized PDF caculation (without for loop) to get their correlation matrix. This is helpful to study feature grouping.credits to @sibmike https://www.kaggle.com/sibmike/are-vars-mixed-up-time-intervals **Functions**<jupyter_code>def logloss(y,yp):
    yp = np.clip(yp,1e-5,1-1e-5)
    return -y*np.log(yp)-(1-y)*np.log(1-yp)
    
def reverse(tr,te):
    reverse_list = [0,1,2,3,4,5,6,7,8,11,15,16,18,19,
                22,24,25,26,27,41,29,
                32,35,37,40,48,49,47,
                55,51,52,53,60,61,62,103,65,66,67,69,
                70,71,74,78,79,
                82,84,89,90,91,94,95,96,97,99,
                105,106,110,111,112,118,119,125,128,
                130,133,134,135,137,138,
                140,144,145,147,151,155,157,159,
                161,162,163,164,167,168,
                170,171,173,175,176,179,
                180,181,184,185,187,189,
                190,191,195,196,199]
    reverse_list = ['var_%d'%i for i in reverse_list]
    for col in reverse_list:
        tr[col] = tr[col]*(-1)
        te[col] = te[col]*(-1)
    return tr,te

def scale(tr,te):
    for col in tr.columns:
        if col.startswith('var_'):
            mean,std = tr[col].mean(),tr[col].std()
            tr[col] = (tr[col]-mean)/std
            te[col] = (te[col]-mean)/std
    return tr,te

def getp_vec_sum(x,x_sort,y,std,c=0.5):
    # x is sorted
    left = x - std/c
    right = x + std/c
    p_left = np.searchsorted(x_sort,left)
    p_right = np.searchsorted(x_sort,right)
    p_right[p_right>=y.shape[0]] = y.shape[0]-1
    p_left[p_left>=y.shape[0]] = y.shape[0]-1
    return (y[p_right]-y[p_left])

def get_pdf(tr,col,x_query=None,smooth=3):
    std = tr[col].std()
    df = tr.groupby(col).agg({'target':['sum','count']})
    cols = ['sum_y','count_y']
    df.columns = cols
    df = df.reset_index()
    df = df.sort_values(col)
    y,c = cols
    
    df[y] = df[y].cumsum()
    df[c] = df[c].cumsum()
    
    if x_query is None:
        rmin,rmax,res = -5.0, 5.0, 501
        x_query = np.linspace(rmin,rmax,res)
    
    dg = pd.DataFrame()
    tm = getp_vec_sum(x_query,df[col].values,df[y].values,std,c=smooth)
    cm = getp_vec_sum(x_query,df[col].values,df[c].values,std,c=smooth)+1
    dg['res'] = tm/cm
    dg.loc[cm<500,'res'] = 0.1
    return dg['res'].values

def get_pdfs(tr):
    y = []
    for i in range(200):
        name = 'var_%d'%i
        res = get_pdf(tr,name)
        y.append(res)
    return np.vstack(y)

def print_corr(corr_mat,col,bar=0.97):
    print(col)
    cols = corr_mat.loc[corr_mat[col]>bar,col].index.values
    cols_ = ['var_%s'%(i.split('_')[-1]) for i in cols]
    print(cols)
    return cols<jupyter_output><empty_output><jupyter_text>**load data & group vars**<jupyter_code>%%time
path = '../input/'
tr = pd.read_csv('%s/train.csv'%path)
te = pd.read_csv('%s/test.csv'%path)
%%time
tr,te = reverse(tr,te)
tr,te = scale(tr,te)
%%time
prob = get_pdf(tr,'var_0')
plt.plot(prob)
%%time
pdfs = get_pdfs(tr)
%%time
df_pdf = pd.DataFrame(pdfs.T,columns=['var_prob_%d'%i for i in range(200)])
corr_mat = df_pdf.corr(method='pearson')
corr_mat.head()
plt.figure(figsize=(15,10))
sns.heatmap(corr_mat, cmap='RdBu_r', center=0.0) 
plt.title('PDF Correlations',fontsize=16)
plt.show()<jupyter_output><empty_output><jupyter_text>**We can group features using this correlation matrix. For example, var_0 and var_2's pdfs is 0.97+ correlated. We can confirm it using the figure below.**<jupyter_code>plt.figure(figsize=(10,5))
plt.plot(pdfs[0],color='b',label='var_0')
plt.plot(pdfs[2],color='r',label='var_2')
plt.legend(loc='upper right')<jupyter_output><empty_output><jupyter_text>**We can find the group of a var using the following functions.**<jupyter_code>cols = print_corr(corr_mat,'var_prob_12')
corr_mat.loc[cols,cols]<jupyter_output>var_prob_12
['var_prob_12' 'var_prob_75' 'var_prob_81' 'var_prob_83' 'var_prob_89'
 'var_prob_95' 'var_prob_99' 'var_prob_113' 'var_prob_122' 'var_prob_139'
 'var_prob_154' 'var_prob_190']
{'fraction_non_alphanumeric': 0.1783596837944664, 'fraction_numerical': 0.08275691699604742, 'mean_word_length': 3.5139353400222966, 'pattern_counts': {'":': 0, '<': 17, '<?xml version=': 0, '>': 19, 'https://': 1, 'lorem ipsum': 0, 'www.': 1, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '503723', 'license': 'MIT', 'n_tokens_mistral': 1915, 'n_tokens_neox': 1694, 'n_words': 262, 'path': '12 customer prediction/fast-pdf-calculation-with-correlation-matrix.ipynb', 'repo_name': 'MLVPRASAD/KaggleProjects', 'size': '4048'}