<jupyter_start><jupyter_text>by: <NAME> Date: 3 Feb 2019  Importing Libaraies<jupyter_code>from matplotlib import pyplot as plt
from scipy.spatial.distance import pdist, squareform
from scipy import exp
from scipy.linalg import eigh

import pandas as pd
import numpy as np
from numpy import linalg as LA

import sklearn
from sklearn import metrics
from sklearn import linear_model as lm
from sklearn import svm
from sklearn.model_selection import cross_val_score as cvs
from sklearn.decomposition import KernelPCA
from sklearn.model_selection import cross_val_score
from sklearn.gaussian_process import GaussianProcessClassifier
from sklearn.gaussian_process.kernels import RBF
from sklearn.model_selection import cross_validate
from sklearn.model_selection import train_test_split
from sklearn.datasets import make_classification
from sklearn.ensemble import ExtraTreesClassifier
from sklearn.linear_model import LogisticRegression
from sklearn.naive_bayes import GaussianNB
from sklearn.ensemble import RandomForestClassifier, VotingClassifier<jupyter_output><empty_output><jupyter_text>Read Data from text file and convert it to numpy arraies<jupyter_code>xtrain=pd.read_table('Data/xtrain.txt',header=None)
xtrain=np.array(xtrain)
xtrain=xtrain.T[1:,]# remove the first colume

ytrain = pd.read_csv('Data/ytrain.txt',header=None)
ytrain=np.array(ytrain)

xtest=pd.read_table('Data/xtest.txt',header=None)
xtest=np.array(xtest)
xtest=xtest.T[1:,]# remove the first colume

print('xtrain shape',xtrain.shape)
print('ytrain shape',ytrain.shape)
print('xtest  shape',xtest.shape)<jupyter_output>xtrain shape (184, 4654)
ytrain shape (184, 1)
xtest  shape (92, 4654)<jupyter_text>plot labels to show that this data is unbalanced<jupyter_code># plt.hist(ytrain,label='Unbalanced Data',histtype='barstacked',orientation='horizontal')
plt.bar([-1,1],[np.count_nonzero(ytrain==1),np.count_nonzero(ytrain==-1)],color=['black', 'red'])
plt.show()<jupyter_output><empty_output><jupyter_text>Creat Balanced Data<jupyter_code>def balance_data (xtrain,ytrain): 
    x_y=np.zeros((184,4655))
    neg_index=0
    for i in range (ytrain.shape[0]):
        if ytrain[i,0]==1:
            x_y[i]=np.concatenate((xtrain[i+1],ytrain[i]))
        else:
            while neg_index ==66: 
                break
            else:    
                neg_index+=1
                x_y[i]=np.concatenate((xtrain[i+1],ytrain[i]))
    x_y=x_y[~np.all(x_y == 0, axis=1)]
    xtrain_balance=x_y[:,0:4654]
    ytrain_balance=x_y[:,4654:]
    return xtrain_balance,ytrain_balance
xtrain_balance,ytrain_balance=balance_data (xtrain,ytrain)<jupyter_output><empty_output><jupyter_text>Kernel PCA<jupyter_code>def stepwise_kpca(X, gamma=0.0001, n_components=123):
    sq_dists = pdist(X, 'sqeuclidean')
    mat_sq_dists = squareform(sq_dists)
    K = exp(-gamma * mat_sq_dists)
    N = K.shape[0]
    one_n = np.ones((N,N)) / N
    eigvals, eigvecs = eigh(K)
    X_pc = np.column_stack((eigvecs[:,-i] for i in range(1,n_components+1)))
    return X_pc<jupyter_output><empty_output><jupyter_text>Features Selection  Cross Validation Single GP<jupyter_code>def cross_validation_GP (xtrain,ytrain,number_exp=5):
    acc=[]
    score=[]
    under_curve=[]
    for j in range (number_exp):
        rand_nu=np.random.randint(0,100)
        X_train, X_test, y_train, y_test = train_test_split(xtrain, ytrain, test_size=0.2, random_state=rand_nu)
        gp = GaussianProcessClassifier(kernel = 1.0 * RBF(1.0),random_state=0)#.fit(xtrain, ytrain.reshape(-1,))
        gp.fit(X_train, y_train.reshape(-1,))
        acc.append(gp.score(X_test,y_test)*100)
#         prob=gp.predict(X_test)
        prob=gp.predict_proba(X_test)
        ror= metrics.roc_auc_score(y_test,prob[:,1])
        under_curve.append(ror)
    print('total accuercy form {} experiments = {:.2f} %     std= {:.2f}'.format(number_exp,np.mean(acc),np.std(acc)))
    print('the total area under the curve = {:.2f}      std= {:.2f}'.format(np.mean(under_curve),np.std(under_curve)))
    return np.mean(under_curve)<jupyter_output><empty_output><jupyter_text>Cross Validation voting from 3 GPs<jupyter_code>def cross_validation_multi_GP (xtrain,ytrain,number_exp=5, number_GP=3):
    acc=[]
    score=[]
    under_curve=[]
    for j in range (number_exp):
        rand_nu=np.random.randint(0,100)
        X_train, X_test, y_train, y_test = train_test_split(xtrain, ytrain, test_size=0.2, random_state=rand_nu)
#         for g in range (number_GP):
        gp1= GaussianProcessClassifier(kernel = 1.0 * RBF(1.0),random_state=0)
        gp1.fit(X_train, y_train.reshape(-1,))
        gp2= GaussianProcessClassifier(kernel = 1.0 * RBF(1.0),random_state=0)
        gp2.fit(X_train, y_train.reshape(-1,))
        gp3= GaussianProcessClassifier(kernel = 1.0 * RBF(1.0),random_state=0)
        gp3.fit(X_train, y_train.reshape(-1,))
        vote=VotingClassifier(estimators=[('lr', gp1), ('rf', gp2), ('gnb', gp3)], voting='soft')
        vote.fit(X_train, y_train.reshape(-1,))
        acc.append(vote.score(X_test,y_test)*100)
#         prob=vote.predict(X_test)
        prob=vote.predict_proba(X_test)
        ror= metrics.roc_auc_score(y_test,prob[:,1])
        under_curve.append(ror)
    print('total accuercy form {} experiments = {:.2f} %     std= {:.2f}'.format(number_exp,np.mean(acc),np.std(acc)))
    print('the total area under the curve = {:.2f}      std= {:.2f}'.format(np.mean(under_curve),np.std(under_curve)))
    return np.mean(under_curve)<jupyter_output><empty_output><jupyter_text>Test   Xtrain, ytrain<jupyter_code>print('Single GP')
cross_validation_GP(xtrain,ytrain)
print('Multi GPs')
cross_validation_multi_GP(xtrain,ytrain)<jupyter_output>Single GP
total accuercy form 5 experiments = 61.62 %     std= 6.26
the total area under the curve = 0.50      std= 0.00
Multi GPs
total accuercy form 5 experiments = 62.16 %     std= 6.62
the total area under the curve = 0.50      std= 0.00<jupyter_text>PCA<jupyter_code>X_pc_unbalance= stepwise_kpca(xtrain,gamma=0.00001,n_components=99)
cross_validation_GP(X_pc_unbalance,ytrain)
cross_validation_multi_GP(X_pc_unbalance,ytrain)<jupyter_output>total accuercy form 5 experiments = 63.78 %     std= 5.82
the total area under the curve = 0.62      std= 0.08
total accuercy form 5 experiments = 65.95 %     std= 10.62
the total area under the curve = 0.59      std= 0.07
{'fraction_non_alphanumeric': 0.12610201511335012, 'fraction_numerical': 0.031171284634760706, 'mean_word_length': 4.749321266968326, 'pattern_counts': {'":': 0, '<': 35, '<?xml version=': 0, '>': 35, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '9', 'dir': 'jupyter-structured-clean-dedup', 'id': '454483', 'license': 'MIT', 'n_tokens_mistral': 2494, 'n_tokens_neox': 2340, 'n_words': 438, 'path': 'Gaussian_Process.ipynb', 'repo_name': 'montaserFath/Kernal-Methods-project', 'size': '6369'}