<jupyter_start><jupyter_text>Test Classifiers and Regressors on train / test data set<jupyter_code>import numpy as np
import math
import pandas as pd
from sklearn.metrics import classification_report
from sklearn.model_selection import cross_val_score,cross_val_predict, train_test_split
from sklearn.neighbors import KNeighborsClassifier
from sklearn.svm import SVC,SVR,LinearSVC
from sklearn.tree import DecisionTreeClassifier
from sklearn.ensemble import RandomForestClassifier, AdaBoostClassifier
from sklearn.naive_bayes import GaussianNB
from sklearn.discriminant_analysis import LinearDiscriminantAnalysis
from sklearn.discriminant_analysis import QuadraticDiscriminantAnalysis
from sklearn.metrics import accuracy_score,roc_curve, auc, get_scorer, roc_auc_score
from sklearn.preprocessing import label_binarize
from sklearn.multiclass import OneVsRestClassifier
from scipy.stats import spearmanr, pearsonr
from matplotlib import pyplot 
from sklearn import linear_model,neighbors,tree,gaussian_process,ensemble,neural_network, manifold,mixture<jupyter_output><empty_output><jupyter_text>Parameters<jupyter_code>import sys
sys.version
sys.version_info
rel = "./"
train_data_file = rel+"features.csv"
test_data_file = rel+"features/STS/2012-en-test.csv"
cv_fold = 10

#train_reference_data_file = rel +"dataset/STS/2012-en-train.csv"
#test_reference_data_file = rel +"dataset/STS/2012-en-test.csv"

threshold = 0.7<jupyter_output><empty_output><jupyter_text>Classifiers to test<jupyter_code>classifiers = [
    #['SVC-linear',SVC(kernel="linear", C=0.025, probability = True)],
    #['SVC-rbf',SVC(gamma=2, C=1, probability = True,decision_function_shape ='ovr')],
    #['KNN-3',KNeighborsClassifier(3)],
    #['KNN-5',KNeighborsClassifier(5)],
    #['KNN-7',KNeighborsClassifier(7)],
    #['DTC-5',DecisionTreeClassifier(max_depth=5)],
    #['DTC-10',DecisionTreeClassifier(max_depth=10)],
    #['DTC-15',DecisionTreeClassifier(max_depth=15)],
    ['RFC-5-10',RandomForestClassifier(max_depth=5, n_estimators=10)],
    ['RFC-10-10',RandomForestClassifier(max_depth=10, n_estimators=10)],
    ['RFC-15-10',RandomForestClassifier(max_depth=15, n_estimators=10)],
    ['RFC-15-20',RandomForestClassifier(max_depth=15, n_estimators=20)],
    ['RFC-15-30',RandomForestClassifier(max_depth=15, n_estimators=30)],
    ['RFC-25',RandomForestClassifier(max_depth=25, n_estimators=50)],
    ['RFC-30',RandomForestClassifier(max_depth=30, n_estimators=50)],
    ['RFC-45',RandomForestClassifier(max_depth=45, n_estimators=50)],
    #['RFC',RandomForestClassifier()],
    #['ADA',AdaBoostClassifier()],
    #['GNB',GaussianNB()],
    #['LDA',LinearDiscriminantAnalysis()],
    #['QDA',QuadraticDiscriminantAnalysis()]
]<jupyter_output><empty_output><jupyter_text>Regressors to test<jupyter_code>regressors = [
    #SVR(kernel="linear", C=0.025),
    #SVR(gamma=2, C=1),
    #mixture.GaussianMixture(1),
    #mixture.GaussianMixture(2),
    #mixture.GaussianMixture(3),
    linear_model.LinearRegression(),
    #linear_model.Ridge(alpha = .5),
    #linear_model.ElasticNet(),
    #linear_model.BayesianRidge(),
    ensemble.RandomForestRegressor(),
    ensemble.RandomForestRegressor(max_depth=15, n_estimators=30),
    ensemble.GradientBoostingRegressor(),
    ensemble.ExtraTreesRegressor(),
    ensemble.BaggingRegressor(),
    ensemble.AdaBoostRegressor(),
    gaussian_process.GaussianProcessRegressor(),
    linear_model.HuberRegressor(),
    linear_model.PassiveAggressiveRegressor(),
    linear_model.RANSACRegressor(),
    linear_model.TheilSenRegressor(),
    linear_model.SGDRegressor(),
    neighbors.KNeighborsRegressor(),
    tree.DecisionTreeRegressor(),
    tree.ExtraTreeRegressor(),
    neural_network.MLPRegressor(activation='logistic',solver='lbfgs',max_iter=200)
    
]<jupyter_output><empty_output><jupyter_text>Load data<jupyter_code>import csv, json

import io
import csv

def parse(data_file, sep):
    d = pd.read_csv(data_file, delimiter=',', header=None).as_matrix()
    pairs = d[:,0:2]
    x = d[:,2:-1]
    y = d[:,-1]
    return x,y,pairs
def multiclass(y_class):
    y = np.zeros([len(y_class),len(np.unique(y_class))])
    for idx, val in enumerate(y_class):
        y[idx,int(val)]=1
    return y
x_train,y_train,pairs_train = parse(train_data_file,sep=',')
x_train.shape
y_train_class = (y_train >= threshold).astype(int)
y_train_mul = multiclass(y_train_class)
x_test,y_test,pairs_test  = parse(test_data_file,sep=',')
x_test.shape
y_test_class = (y_test >= threshold).astype(int)
y_test_mul = multiclass(y_test_class)
y_test_class
pd.DataFrame(x_train).head()
pd.DataFrame(x_test).head()<jupyter_output><empty_output><jupyter_text>Represent data<jupyter_code>### Train
mds = manifold.MDS(n_components=2)
x_train_2d = mds.fit_transform(x_train)
y_train_color = ['r' if bool(v) else 'g' for v in y_train_class]  
pyplot.scatter(x_train_2d[:,0],x_train_2d[:,1],c=y_train_color)
pyplot.show()<jupyter_output><empty_output><jupyter_text>Select features<jupyter_code>from sklearn.datasets import load_boston
from sklearn.feature_selection import SelectFromModel
from sklearn.linear_model import LassoCV

# Load the boston dataset.
boston = load_boston()
X, y = boston['data'], boston['target']

# We use the base estimator LassoCV since the L1 norm promotes sparsity of features.
clf = LassoCV()

# Set a minimum threshold of 0.25
sfm = SelectFromModel(clf, threshold=0.25)
sfm.fit(x_train, y_train)
n_features = sfm.transform(x_train).shape[1]

# Reset the threshold till the number of features equals two.
# Note that the attribute can be set directly instead of repeatedly
# fitting the metatransformer.
while n_features > 10:
    sfm.threshold += 0.1
    X_transform = sfm.transform(x_train)
    n_features = X_transform.shape[1]
    

x_train = sfm.transform(x_train)
x_test = sfm.transform(x_test)<jupyter_output><empty_output><jupyter_text>Roc curve generator<jupyter_code>def plot_roc_curve(y_test, y_score):
    # Compute ROC curve and ROC area for each class
    fpr = dict()
    tpr = dict()
    roc_auc = dict()
    # Compute micro-average ROC curve and ROC area
    fpr["micro"], tpr["micro"], _ = roc_curve(y_test.ravel(), y_score.ravel())
    roc_auc["micro"] = auc(fpr["micro"], tpr["micro"])
    pyplot.figure()
    lw = 2
    pyplot.plot(fpr["micro"],tpr["micro"], color='darkorange', lw=lw, label='ROC curve (area = %0.2f)' % roc_auc["micro"])
    pyplot.plot([0, 1], [0, 1], color='navy', lw=lw, linestyle='--')
    pyplot.xlim([0.0, 1.0])
    pyplot.ylim([0.0, 1.05])
    pyplot.xlabel('False Positive Rate')
    pyplot.ylabel('True Positive Rate')
    pyplot.title('Receiver operating characteristic')
    pyplot.legend(loc="lower right")
    pyplot.show()<jupyter_output><empty_output><jupyter_text>Score classifiers<jupyter_code>import sklearn.pipeline as pipeline
from sklearn.feature_selection import SelectFromModel
from sklearn.ensemble import BaggingClassifier
classifiers_scores = []

# iterate over classifiers
#for clf in classifiers:
#    y_score = clf.fit(x_train, y_train_class).predict(x_test)
#    sc = get_scorer('f1')(clf,x_test,y_test_class)
#    classifiers_scores.append(sc)
    
    
    
classifiers_scores = []
classifiers_regr = []
# iterate over classifiers
for name,clf in classifiers:
    #clf =  ensemble.VotingClassifier(classifiers,voting='soft')
    y_score_clf_test = clf.fit(x_train, y_train_class).predict_proba(x_test)
    y_score_clf_train = clf.predict_proba(x_train)
    #score_max = f1_score(y_train_class, y_score_clf[:,1]>0.5)
    score_max = pearsonr(y_test,y_score_clf_test[:,1])[0]
    regr_max = None
    for regr in regressors: 
        x_train_ext = np.append(x_train, y_score_clf_train, 1)
        x_test_ext  = np.append(x_test, y_score_clf_test, 1)
        y_score = regr.fit(x_train_ext,y_train).predict(x_test_ext)
        #score = f1_score(y_train_class, y_score>0.5)
        score = pearsonr(y_test,y_score)[0]
        if score > score_max:
            score_max = score
            regr_max = regr
    classifiers_scores.append(score_max)
    classifiers_regr.append(regr_max)
pd.DataFrame({"Classifier":[name for name,classifier in classifiers],\
              "Pearson":classifiers_scores},\
             columns=["Classifier","Pearson"]\
            )<jupyter_output><empty_output><jupyter_text>Evaluate features<jupyter_code>import sklearn.metrics as metrics

clas_threshold = 0.6
c_eval = pd.DataFrame(index=range(0,x_train.shape[1]), columns=['F1','Pearson','Precision','Recall','Accuracy'])
for i in range(0,x_train.shape[1]):
    y_score_cl = (x_test[:,i] > clas_threshold).astype(int)
    c_eval['F1'][i] = metrics.f1_score(y_test_class, y_score_cl)
    c_eval['Pearson'][i] = pearsonr(y_test,x_test[:,i])[0]
    c_eval['Precision'][i] = metrics.precision_score(y_test_class, y_score_cl)
    c_eval['Recall'][i] = metrics.recall_score(y_test_class, y_score_cl)
    c_eval['Accuracy'][i] = metrics.accuracy_score(y_test_class, y_score_cl)
c_eval.T
c_eval.T[0]<jupyter_output><empty_output><jupyter_text>The best classifier<jupyter_code>import sklearn.metrics as metrics

max_cls = np.where(classifiers_scores == max(classifiers_scores))[0][0]
clf = classifiers[max_cls][1]
regr = classifiers_regr[max_cls]
print(clf)
y_score_clf = clf.predict_proba(x_test)
y_score_clf_train = clf.predict_proba(x_train)
if regr is not None:
    print(regr)
    x_train_ext = np.append(x_train, y_score_clf_train, 1)
    x_test_ext  = np.append(x_test, y_score_clf, 1)
    y_score = regr.fit(x_train_ext,y_train).predict(x_test_ext)
else:
    print('No regression')
    y_score = y_score_clf[:,1]
print("Pearson:\t%0.3f"%pearsonr(y_test,y_score)[0])
print("F1:\t\t%0.3f"%metrics.f1_score(y_test_class,y_score>0.5))
print("Precision:\t%0.3f"%metrics.precision_score(y_test_class,y_score>0.5))
print("Recall:\t\t%0.3f"%metrics.recall_score(y_test_class,y_score>0.5))
print("Accuracy:\t%0.3f"%metrics.accuracy_score(y_test_class,y_score>0.5))


print(classification_report(y_test_class, y_score>0.5, target_names=['False', 'True']))
plot_roc_curve(y_test_class,y_score)
clf = classifiers[np.where(classifiers_scores == max(classifiers_scores))[0][0]][1] 
print(clf)
y_score = clf.predict_proba(x_test)
print("Pearson:\t%0.3f"    %pearsonr(y_test,y_score[:,1])[0])
print("F1:\t\t%0.3f"       %get_scorer('f1')(clf,x_test,y_test_class))
print("Accuracy:\t%0.3f"   %get_scorer('accuracy')(clf,x_test,y_test_class))
print("Precision:\t%0.3f"  %get_scorer('precision')(clf,x_test,y_test_class))
print("Recall:\t\t%0.3f"   %get_scorer('recall')(clf,x_test,y_test_class))
plot_roc_curve(y_test_class,y_score[:,1])
print(classification_report(y_test_class, y_score[:,1]>0.5, target_names=['False', 'True']))
y_score_class = clf.predict(x_test)
from sklearn.externals import joblib
joblib.dump(clf,type(clf).__name__ +'_'+str(threshold)+'.pkl')
out_regr_err = 'class.res.csv'
columns = ['Plag','Truth','Classification','Diff','Str1','Str2']
pd.DataFrame(np.array([y_test,y_test_class,y_score_class,y_test_class-y_score_class,pairs_test[:,0],pairs_test[:,1]]).T,columns=columns).to_csv(out_regr_err)<jupyter_output><empty_output><jupyter_text>Score regressors<jupyter_code>regressors_scores = []
for rgs in regressors:
    y_score = rgs.fit(x_train, y_train).predict(x_test)
    regressors_scores.append(pearsonr(y_test,y_score)[0])
pd.DataFrame({"Regressor":[type(regressor).__name__ for regressor in regressors],\
              "Pearson":regressors_scores},\
             columns=["Regressor","Pearson"])<jupyter_output><empty_output><jupyter_text>The best regressor<jupyter_code>m_r = np.where(regressors_scores == max(regressors_scores))[0][0]
rgs = regressors[m_r] 
print(rgs)
y_score = rgs.predict(x_test)
y_score_bin = (y_score >=threshold).astype(int)
plot_roc_curve((y_test_class),(y_score))

print("Spearman:\t%0.3f"%spearmanr(y_test,y_score).correlation)
print("Pearson:\t%0.3f" %pearsonr(y_test,y_score)[0])
print("Accuracy:\t%0.3f" % (accuracy_score(y_test_class, y_score_bin)))
print(classification_report(y_test_class, y_score_bin, target_names=['False', 'True']))
out_regr_err = 'regr.res.csv'
columns = ['Truth','Score','Diff','Abs_Diff','Str1','Str2']
pd.DataFrame(np.array([y_test,y_score,y_test-y_score,abs(y_test-y_score),pairs_test[:,0],pairs_test[:,1]]).T,columns=columns).to_csv(out_regr_err)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.14407398393769774, 'fraction_numerical': 0.019793948243692707, 'mean_word_length': 6.56319018404908, 'pattern_counts': {'":': 4, '<': 53, '<?xml version=': 0, '>': 67, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '13', 'dir': 'jupyter-structured-clean-dedup', 'id': '505839', 'license': 'Apache-2.0', 'n_tokens_mistral': 4974, 'n_tokens_neox': 4759, 'n_words': 643, 'path': 'classify.ipynb', 'repo_name': 'piotrsobecki/ISMIS17', 'size': '12327'}