<jupyter_start><jupyter_text>NIMH Project - Machine Learning - Testing<jupyter_code>import os

import re
import nltk
import numpy as np
import pandas as pd
import pickle as pkl
from sklearn.feature_extraction.text import CountVectorizer
from sklearn.svm import SVC
from sklearn.model_selection import KFold
from sklearn.metrics import roc_curve, auc
from nltk.corpus import stopwords
import matplotlib.pyplot as plt
from sklearn.feature_extraction.text import TfidfVectorizer
import string
from sklearn.ensemble import RandomForestClassifier,ExtraTreesClassifier
from nltk.stem.wordnet import WordNetLemmatizer

import math
import chardet
os.chdir('Z:\project')
#%%

#data = pd.read_csv("./results/pat_visit_score_match_1127.csv")
data = pd.read_csv("./results/train_1885.csv")
data.head()
data.shape

from sklearn.utils import shuffle
data = shuffle(data)
data.head()
labelm=data[data.columns[9]].tolist()
labeln=data[data.columns[10]].tolist()
labelm2=data[data.columns[11]].tolist()
labeln2=data[data.columns[12]].tolist()
# 
arr_m1 = np.array(labelm)
arr_n1 = np.array(labeln)
arr_m2 = np.array(labelm2)
arr_n2 = np.array(labeln2)
sum(arr_m1 == arr_n1)/len(arr_m1), sum(arr_m2 == arr_n2)/len(arr_m2)
#%%
corpusList=data[data.columns[4]].tolist()
corpusList2=data[data.columns[7]].tolist()
len(corpusList)
len(corpusList2)
#%%
def tokenize(text):
    tokens = [word for sent in nltk.sent_tokenize(text) for word in nltk.word_tokenize(sent)]
    filtered_tokens = [ token for token in tokens if re.search('(^[a-zA-Z]+$)', token) ]
    a=[]
    for i in filtered_tokens:
        a.append(WordNetLemmatizer().lemmatize(i,'v'))
    return a
    #return filtered_tokens


cachedStopWords = stopwords.words("english")
#%%
cv = TfidfVectorizer(lowercase=True,
                     ngram_range=(1, 4), preprocessor=None, stop_words='english',
                     strip_accents=None, tokenizer=tokenize, vocabulary=None)
X1 = cv.fit_transform(corpusList)
lexicon = cv.get_feature_names()
#print (lexicon)
print()

#%%
x=X1  ## for select fetures 
print(x.shape)

pkl.dump( x, open( "./results/tfidf_1885.pickle", "wb" ) )
pkl.dump( lexicon, open( "./results/lexicon_1885.pickle", "wb" ) )
#x1 = pkl.load( open( "./results/tfidf_4801395.pickle", "rb" ) )
#lexicon1 = pkl.load( open( "./results/lexicon_4801395.pickle", "rb" ) )
#%%
cv2 = TfidfVectorizer(lowercase=True,
                     ngram_range=(1, 4), preprocessor=None, stop_words='english',
                     strip_accents=None, tokenizer=tokenize, vocabulary=None)
X2 = cv2.fit_transform(corpusList2)
lexicon2 = cv2.get_feature_names()
#print (lexicon)
print()

#%%
x2=X2  ## for select fetures 
print(x2.shape)

pkl.dump( x2, open( "./results/tfidf_1885_2.pickle", "wb" ) )
pkl.dump( lexicon2, open( "./results/lexicon_1885_2.pickle", "wb" ) )
#x1 = pkl.load( open( "./results/tfidf_4801395.pickle", "rb" ) )
#lexicon1 = pkl.load( open( "./results/lexicon_4801395.pickle", "rb" ) )
type(x)


Ym = np.array(labelm)
Yn = np.array(labeln)
Ym2 = np.array(labelm2)
Yn2 = np.array(labeln2)
pkl.dump( Ym, open( "./results/ym_1885.pickle", "wb" ) )
pkl.dump( Yn, open( "./results/yn_1885.pickle", "wb" ) )
pkl.dump( Ym2, open( "./results/ym_1885_2.pickle", "wb" ) )
pkl.dump( Yn2, open( "./results/yn_1885_2.pickle", "wb" ) )
#print(Y)  ## class level
"""#%%
#test
cvt = TfidfVectorizer(lowercase=True,
                     ngram_range=(1, 3), preprocessor=None, stop_words='english',
                     strip_accents=None, tokenizer=tokenize, vocabulary=None)

X1t = cvt.fit_transform(corpusList_test)
print(X1t.shape)
print()
lexicon_test = cvt.get_feature_names()
#print (lexicon)
print()

#%%
xt=X1t  ## for select fetures 
print(xt.shape)

Ym = np.array(labels_testm)
Yn = np.array(labels_testn)"""
print(Ym[0:10], Ym[480:490], Ym[1865:])  ## class level
print(Yn[0:10], Yn[480:490], Yn[1865:])  ## class level
print(Ym2[0:10], Ym2[480:490], Ym2[1865:])  ## class level
print(Yn2[0:10], Yn2[480:490], Yn2[1865:])  ## class level
#%%
from sklearn.feature_selection import SelectKBest
from sklearn.feature_selection import chi2

#print(x.shape)
X_new = SelectKBest(chi2, k=800).fit_transform(x, Yn)   # select 800 features

#%%
X=X_new        # make unque name for next cell 
print(X.shape)
#Y
pkl.dump( X, open( "./results/tfidf_1885_800.pickle", "wb" ) )
X_new2 = SelectKBest(chi2, k=800).fit_transform(x2, Yn2)
pkl.dump( X_new2, open( "./results/tfidf_1885_2_800.pickle", "wb" ) )<jupyter_output><empty_output><jupyter_text>Start Here<jupyter_code>X_new = pkl.load( open( "./results/tfidf_1885_800.pickle", "rb" ) )
Ym = pkl.load( open( "./results/ym_1885.pickle", "rb" ) )
Yn = pkl.load( open( "./results/yn_1885.pickle", "rb" ) )
X = X_new[:1000]
X2 = X_new[1000:]
Y = Yn[:1000]
Y2 = Yn[1000:]
Ym1 = Ym[:1000]
Ym2 = Ym[1000:]
""""X2_new = SelectKBest(chi2, k=600).fit_transform(xt, Ym)   # select 600 features
X2_new.shape

#%%
X2=X2_new        # make unque name for next cell 
X2_new.shape""""
#%%
# ExtraTreesClassifier
model=0
results = []
cont = []
kf = KFold(n_splits=10, shuffle=True)
trees = [10, 30, 60, 100, 300, 500]
for t in range(len(trees)):
    fold = 1
    truth = []
    rf_prediction = []
    print("model ", t, ": ExtraTreesClassifier = " + str(trees[t]))
    test_count = 0
    rf = ExtraTreesClassifier(bootstrap=False,
           criterion='entropy', max_depth=15, max_features=0.9,
           max_leaf_nodes=None, min_impurity_decrease=1e-045,
           min_samples_leaf=1, min_samples_split=3,
           min_weight_fraction_leaf=0.0, n_estimators=trees[t], n_jobs=-1,
           oob_score=False, random_state=None, verbose=0, warm_start=False)
    for train_idx, test_idx in kf.split(X):
        trainX = X[train_idx]
        trainY = Y[train_idx]
        testX = X[test_idx]
        testY = Y[test_idx]
        truth.append(testY)
        rf.fit(trainX, trainY)
        Y_hat = rf.predict(testX)
        rf_prediction.append(Y_hat)
        print("   Fold %d accuracy: %.2f %%" % (fold, ((np.sum(Y_hat == testY)/len(testY)) * 100.0)))                        
        fold += 1
    truth = np.concatenate(truth, axis=0)    
    rf_prediction = np.concatenate(rf_prediction, axis=0)
    test_results = np.sum(rf_prediction == truth)/len(truth)
    print("     Overall test accuracy: %.2f %%" % (test_results * 100))  
    rf = rf.fit(X, Y)
    Y_hat = rf.predict(X)
    train_results = np.sum(Y_hat == Y)/len(Y)
    print("     Overall training accuracy: %.2f %%" % (train_results * 100.0))  
    results.append([train_results, test_results])   
    cont.append([truth, rf_prediction])
rf = ExtraTreesClassifier(bootstrap=False,
       criterion='entropy', max_depth=15, max_features=0.9,
       max_leaf_nodes=None, min_impurity_decrease=1e-045,
       min_samples_leaf=1, min_samples_split=3,
       min_weight_fraction_leaf=0.0, n_estimators=60, n_jobs=-1,      #trees[t] = 100
       oob_score=False, random_state=None, verbose=0, warm_start=False)
train_num = 1
for i in range(train_num):
    rf.fit(X, Y)
    Y2_hat = rf.predict(X2)
    vm = np.sum(Y2_hat == Ym2)/len(Ym2)
    vn = np.sum(Y2_hat == Y2)/len(Y2)
    print(np.sum(Y2_hat))
    print("vm ", vm)
    print("vn ",vn)
    
precision = np.sum(Y2_hat + Y2 == 2)/np.sum(Y2_hat)
recall = np.sum(Y2_hat + Y2 == 2)/np.sum(Y2)
f1 = 2*precision*recall/(precision+recall)
print("nlp")
print("precision: ", precision)
print("recall: ", recall)
print("F1: ", f1)

precision0 = np.sum(Y2_hat + Y2 == 0)/(len(Y2_hat)-np.sum(Y2_hat))
recall0 = np.sum(Y2_hat + Y2 == 0)/(len(Y2)-np.sum(Y2))
f10 = 2*precision*recall/(precision+recall)
print("nlp0")
print("precision: ", precision0)
print("recall: ", recall0)
print("F1: ", f10)

print("ave")
print("precision: ", (precision+precision0)/2)
print("recall: ", (recall+recall0)/2)
print("F1: ", (f1+f10)/2)
print(X2.shape)
print(len(Y2_hat))
#%%
model = input("Enter Best Model: ")
best = cont[int(model)]
rf_ct = pd.crosstab(best[1], best[0], margins=True)
rf_ct.columns = ["no", "social", "Total"]
rf_ct.index = ["no", "social", "Total"]
print()
print("ExtraTrees")
print(rf_ct)
print()
#%%
from sklearn.metrics import confusion_matrix
from sklearn.metrics import recall_score,f1_score,precision_score,accuracy_score,classification_report
y_true =cont[int(model)][0]
y_pred = cont[int(model)][1]
print(confusion_matrix(y_true, y_pred))
print(f1_score(y_true, y_pred) )
print(recall_score(y_true, y_pred),   )
print(precision_score(y_true, y_pred))
print(accuracy_score(y_true, y_pred))

print(classification_report(y_true, y_pred))
#%%

Sens = rf_ct.iloc[1][1]/rf_ct.iloc[2][1]
Spec = rf_ct.iloc[0][0]/rf_ct.iloc[2][0]
PPV = rf_ct.iloc[1][1]/rf_ct.iloc[1][2]
NPV = rf_ct.iloc[0][0]/rf_ct.iloc[0][2]
ACC = (rf_ct.iloc[0][0] + rf_ct.iloc[1][1]) / rf_ct.iloc[2][2]
print("ExtraTrees: Sensitivity: %.5f Specificity: %.5f PPV: %.5f NPV: %.5f Accuracy: %.5f" % (Sens, Spec, PPV, NPV, ACC))
#%%
rf_fpr, rf_tpr, rf_thresholds = roc_curve(best[0], best[1], pos_label=1)
roc_auc = auc(rf_fpr, rf_tpr)
plt.figure(1)
plt.plot([0, 1], [0, 1], 'k--')
plt.plot(rf_fpr, rf_tpr, label='ExtraTrees')
plt.xlabel('False positive rate (1 - Specificity)')
plt.ylabel('True Positive Rate (Sensitivity)')
plt.title('ROC curve')
plt.legend(loc='best')
plt.show()
print("AUC = ", roc_auc)
#%%
# SVM
model = 0
cont = []
results = pd.DataFrame(columns=('name', 'accuracy'))
kf = KFold(n_splits=10, shuffle=True)
C = [1,2,4,6,8,10]
gamma = [1,2]  
for g in range(len(gamma)):
    acc = []
    name = "SVM RBF Gamma=%.3f" % (gamma[g])     
    for c in range(len(C)):
        fold = 1
        truth = []
        svm_prediction = []
        print("model %d: SVM RBF Gamma=%.3f C=%.2f" % (model, gamma[g], C[c]))        
        test_count = 0
        svm = SVC(C=C[c], kernel='rbf', gamma=gamma[g])
        for train_idx, test_idx in kf.split(X):
            trainX = X[train_idx]
            trainY = Y[train_idx]
            testX = X[test_idx]
            testY = Y[test_idx]
            truth.append(testY)
            svm.fit(trainX, trainY)
            Y_hat = svm.predict(testX)
            svm_prediction.append(Y_hat)
            print("   Fold %d accuracy: %.2f %%" % (fold, ((np.sum(Y_hat == testY)/len(testY)) * 100.0)))                        
            fold += 1
        truth = np.concatenate(truth, axis=0)    
        svm_prediction = np.concatenate(svm_prediction, axis=0)
        test_results = np.sum(svm_prediction == truth)/len(truth)
        print("     Overall test accuracy: %.2f %%" % (test_results * 100))  
        svm = svm.fit(X, Y)
        Y_hat = svm.predict(X)
        train_results = np.sum(Y_hat == Y)/len(Y)
        print("     Overall training accuracy: %.2f %%" % (train_results * 100.0))  
        acc.append([train_results, test_results])   
        cont.append([truth, svm_prediction])
        model += 1
    results = results.append({'name': name, 'accuracy' : acc}, ignore_index=True)
svm = SVC(C=10, kernel='rbf', gamma=2) #79.2%
#svm = SVC(C=2, kernel='rbf', gamma=128) #79.6
#svm = SVC(C=2, kernel='rbf', gamma=128) 
train_num = 1
for i in range(train_num):
    svm.fit(X, Y)
    Y2_hat = svm.predict(X2)
    vm = np.sum(Y2_hat == Ym2)/len(Ym2)
    vn = np.sum(Y2_hat == Y2)/len(Y2)
    print(np.sum(Y2_hat))
    print("vm ", vm)
    print("vn ",vn)
    
precision = np.sum(Y2_hat + Y2 == 2)/np.sum(Y2_hat)
recall = np.sum(Y2_hat + Y2 == 2)/np.sum(Y2)
f1 = 2*precision*recall/(precision+recall)
print("nlp")
print("precision: ", precision)
print("recall: ", recall)
print("F1: ", f1)

precision0 = np.sum(Y2_hat + Y2 == 0)/(len(Y2_hat)-np.sum(Y2_hat))
recall0 = np.sum(Y2_hat + Y2 == 0)/(len(Y2)-np.sum(Y2))
f10 = 2*precision*recall/(precision+recall)
print("nlp0")
print("precision: ", precision0)
print("recall: ", recall0)
print("F1: ", f10)

print("ave")
print("precision: ", (precision+precision0)/2)
print("recall: ", (recall+recall0)/2)
print("F1: ", (f1+f10)/2)
#%%

model = input("Enter Best Model: ")
best = cont[int(model)]
rf_ct = pd.crosstab(best[1], best[0], margins=True)
rf_ct.columns = ["no", "social", "Total"]
rf_ct.index = ["no", "social", "Total"]
print()
print("SVM")
print(rf_ct)
print()
#%%

Sens = rf_ct.iloc[1][1]/rf_ct.iloc[2][1]
Spec = rf_ct.iloc[0][0]/rf_ct.iloc[2][0]
PPV = rf_ct.iloc[1][1]/rf_ct.iloc[1][2]
NPV = rf_ct.iloc[0][0]/rf_ct.iloc[0][2]
ACC = (rf_ct.iloc[0][0] + rf_ct.iloc[1][1]) / rf_ct.iloc[2][2]
print("Random Forest: Sensitivity: %.5f Specificity: %.5f PPV: %.5f NPV: %.5f Accuracy: %.5f" % (Sens, Spec, PPV, NPV, ACC))
#%%
from sklearn.metrics import confusion_matrix
from sklearn.metrics import recall_score,f1_score,precision_score,accuracy_score,classification_report
y_true =cont[int(model)][0]
y_pred = cont[int(model)][1]
print(confusion_matrix(y_true, y_pred))
print(f1_score(y_true, y_pred) )

print(recall_score(y_true, y_pred),   )
print(precision_score(y_true, y_pred))
print(accuracy_score(y_true, y_pred))

print(classification_report(y_true, y_pred))
#%%
# RandomForest
model=0
results = []
cont = []
kf = KFold(n_splits=10, shuffle=True)
trees = [5, 10, 50, 100, 200, 300, 400, 500]
for t in range(len(trees)):
    fold = 1
    truth = []
    rf_prediction = []
    print("model ", t, ": Random Forest trees = " + str(trees[t]))
    test_count = 0
    rf = RandomForestClassifier(n_estimators=trees[t], criterion='entropy', n_jobs=-1, )
    for train_idx, test_idx in kf.split(X):
        trainX = X[train_idx]
        trainY = Y[train_idx]
        testX = X[test_idx]
        testY = Y[test_idx]
        truth.append(testY)
        rf.fit(trainX, trainY)
        Y_hat = rf.predict(testX)
        rf_prediction.append(Y_hat)
        print("   Fold %d accuracy: %.2f %%" % (fold, ((np.sum(Y_hat == testY)/len(testY)) * 100.0)))                        
        fold += 1
    truth = np.concatenate(truth, axis=0)    
    rf_prediction = np.concatenate(rf_prediction, axis=0)
    test_results = np.sum(rf_prediction == truth)/len(truth)
    print("     Overall test accuracy: %.2f %%" % (test_results * 100))  
    rf = rf.fit(X, Y)
    Y_hat = rf.predict(X)
    train_results = np.sum(Y_hat == Y)/len(Y)
    print("     Overall training accuracy: %.2f %%" % (train_results * 100.0))  
    results.append([train_results, test_results])   
    cont.append([truth, rf_prediction])
rf = RandomForestClassifier(n_estimators=400, criterion='entropy', n_jobs=-1, )
train_num = 1
for i in range(train_num):
    rf.fit(X, Y)
    Y2_hat = rf.predict(X2)
    vm = np.sum(Y2_hat == Ym2)/len(Ym2)
    vn = np.sum(Y2_hat == Y2)/len(Y2)
    print(np.sum(Y2_hat))
    print("vm ", vm)
    print("vn ",vn)
    
precision = np.sum(Y2_hat + Y2 == 2)/np.sum(Y2_hat)
recall = np.sum(Y2_hat + Y2 == 2)/np.sum(Y2)
f1 = 2*precision*recall/(precision+recall)
print("nlp")
print("precision: ", precision)
print("recall: ", recall)
print("F1: ", f1)

precision0 = np.sum(Y2_hat + Y2 == 0)/(len(Y2_hat)-np.sum(Y2_hat))
recall0 = np.sum(Y2_hat + Y2 == 0)/(len(Y2)-np.sum(Y2))
f10 = 2*precision*recall/(precision+recall)
print("nlp0")
print("precision: ", precision0)
print("recall: ", recall0)
print("F1: ", f10)

print("ave")
print("precision: ", (precision+precision0)/2)
print("recall: ", (recall+recall0)/2)
print("F1: ", (f1+f10)/2)
#%%
model = input("Enter Best Model: ")
best = cont[int(model)]
rf_ct = pd.crosstab(best[1], best[0], margins=True)
rf_ct.columns = ["no", "social", "Total"]
rf_ct.index = ["no", "social", "Total"]
print()
print("Random Forest")
print(rf_ct)
print()
#%%

Sens = rf_ct.iloc[1][1]/rf_ct.iloc[2][1]
Spec = rf_ct.iloc[0][0]/rf_ct.iloc[2][0]
PPV = rf_ct.iloc[1][1]/rf_ct.iloc[1][2]
NPV = rf_ct.iloc[0][0]/rf_ct.iloc[0][2]
ACC = (rf_ct.iloc[0][0] + rf_ct.iloc[1][1]) / rf_ct.iloc[2][2]
print("Random Forest: Sensitivity: %.5f Specificity: %.5f PPV: %.5f NPV: %.5f Accuracy: %.5f" % (Sens, Spec, PPV, NPV, ACC))
#%%
from sklearn.metrics import confusion_matrix
from sklearn.metrics import recall_score,f1_score,precision_score,accuracy_score,classification_report
y_true =cont[int(model)][0]
y_pred = cont[int(model)][1]
print(confusion_matrix(y_true, y_pred))
print(f1_score(y_true, y_pred) )



print(recall_score(y_true, y_pred),   )
print(precision_score(y_true, y_pred))
print(accuracy_score(y_true, y_pred))

print(classification_report(y_true, y_pred))
#%%
from sklearn.utils import shuffle
X, Y = shuffle(X, Y)
split = int(len(Y) * 4/5)
trainX = X[:split, :]
trainY = Y[:split]
testX = X[split:, :]
testY = Y[split:]

#%%
from sklearn.neural_network import MLPClassifier

mlp = MLPClassifier(hidden_layer_sizes=(50,), max_iter=1000, alpha=1e-4,
                    solver='adam', verbose=False, tol=1e-4, random_state=1)
#mlp = MLPClassifier(hidden_layer_sizes=(50,), max_iter=10, alpha=1e-4,
#                    solver='sgd', verbose=10, tol=1e-4, random_state=1,
#                    learning_rate_init=.1)

mlp.fit(trainX, trainY)
print("Training set score: %f" % mlp.score(trainX, trainY))
print("Test set score: %f" % mlp.score(testX, testY))
from sklearn.utils import shuffle
X, Y = shuffle(X, Y)
from sklearn.neural_network import MLPClassifier
mlp = MLPClassifier(hidden_layer_sizes=(100,),max_iter=500, alpha=1e-4)
#mlp = MLPClassifier(hidden_layer_sizes=(100,),max_iter=1000, alpha=1e-4) 67.5%
mlp.fit(X,Y)

Y2_hat = mlp.predict(X2)
vm = np.sum(Y2_hat == Ym2)/len(Ym2)
vn = np.sum(Y2_hat == Y2)/len(Y2)
print(np.sum(Y2_hat))
print("vm ", vm)
print("vn ",vn)
from sklearn.utils import shuffle
X, Y = shuffle(X, Y)
from sklearn.neural_network import MLPClassifier
mlp = MLPClassifier(hidden_layer_sizes=(300,),max_iter=500, alpha=1e-4)
#mlp = MLPClassifier(hidden_layer_sizes=(100,),max_iter=1000, alpha=1e-4) 67.5%
mlp.fit(X,Y)

Y2_hat = mlp.predict(X2)
vm = np.sum(Y2_hat == Ym2)/len(Ym2)
vn = np.sum(Y2_hat == Y2)/len(Y2)
print(np.sum(Y2_hat))
print("vm ", vm)
print("vn ",vn)<jupyter_output>402
vm  0.5073446327683616
vn  0.7548022598870057
{'fraction_non_alphanumeric': 0.17744740532959327, 'fraction_numerical': 0.04645161290322581, 'mean_word_length': 4.206191588785047, 'pattern_counts': {'":': 2, '<': 8, '<?xml version=': 0, '>': 8, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '96407', 'license': 'Apache-2.0', 'n_tokens_mistral': 7890, 'n_tokens_neox': 7375, 'n_words': 1313, 'path': 'code/.ipynb_checkpoints/NIMH_test_nlp_1127-checkpoint.ipynb', 'repo_name': 'phzpan/ml_project', 'size': '17825'}