<jupyter_start><jupyter_text>Make Classifiers  This script runs a nested cross-validation within the training data and saves the classifiers  import modules<jupyter_code>%matplotlib inline
import os

import numpy as np
import pandas as pd

import matplotlib.pylab as plt
import seaborn as sns

from sklearn.svm import SVC
from sklearn import preprocessing, model_selection, metrics

import pickle<jupyter_output><empty_output><jupyter_text>get data as an example, here we load a dataset for one type of roi size:<jupyter_code>df = pd.read_csv('../data/interim/csv/roiLaterality_pat0001_a.csv',index_col=[0],header=[0,1])
df.tail()<jupyter_output><empty_output><jupyter_text>when we load from csv, the index and columns are strings by default, but to be able to sort and plot with the thresholds as numbers, we convert the column names to floats<jupyter_code>def changeDf(df):
    
    idx1 = df.columns.get_level_values(0).astype(float)
    idx2 = df.columns.get_level_values(1)

    mIdx = pd.MultiIndex.from_arrays([idx1,idx2])

    df.columns = mIdx
    df.sort_index(axis=1)

    return df
df = changeDf(df)
df.columns[:5]
df.tail()
df.loc[:,-3.0]<jupyter_output><empty_output><jupyter_text>get labels into integer form Initialize a Labeler, which allows to switch from string labels (e.g. "left" to integer labels e.g. 0 and back).<jupyter_code>myLabeler = pickle.load( open('../models/myLabeler.p','rb') )<jupyter_output><empty_output><jupyter_text>divide into training and test data<jupyter_code>my_train = pickle.load( open('../models/my_nest.p','rb'))<jupyter_output><empty_output><jupyter_text>get all data<jupyter_code>csvList = ['../data/interim/csv/%s'%x for x in os.listdir('../data/interim/csv/') if x.startswith('roiLaterality_pat')]
csvList.sort()
csvList[-5:]
def makeDf(csvList,trainOrTest):
    df = pd.DataFrame()
    for pat in csvList:
        for lab in trainOrTest:
            if lab[-1] in pat:
                thisDf = pd.DataFrame([pat],index=[[lab[0]],[lab[1]]])
                df = pd.concat([df,thisDf])
    df.columns = ['csv']
    return df
dfTrain = makeDf(csvList,my_train)
dfTrain.tail()
dfTrain.shape
def getAll(dfTrain):
    
    bigDf = pd.DataFrame()
    
    for i in dfTrain.index:
        
        csv = dfTrain.loc[i]['csv']
        df = changeDf( pd.read_csv(csv,index_col=[0],header=[0,1]) )
        thisDf = pd.DataFrame( df.unstack() ).T
        thisDf.index = [[i[0]],[i[1]]]
        bigDf = pd.concat([bigDf,thisDf])
        
    bigDf = bigDf.reorder_levels([1,2,0],axis=1)
    bigDf.sort_index(inplace=True)

    return bigDf
df = getAll(dfTrain)
df.tail()
df.shape<jupyter_output><empty_output><jupyter_text>get features and labels into arrays<jupyter_code>def makeXy(roiSize,thresh,dims,df=df,myLabeler=myLabeler):
    
    if dims==1:
        # select the data
        thisDf = pd.DataFrame( df['LI'][roiSize][thresh] )
    elif dims==2:
        diffDf = pd.DataFrame( df['L-R'][roiSize][thresh] )
        addDf = pd.DataFrame( df['L+R'][roiSize][thresh] )
        thisDf = pd.concat([diffDf,addDf],axis=1 )
    # make int labels
    labels = thisDf.index.get_level_values(0)#[ thisDf.index.levels[0][x] for x in thisDf.index.labels[0] ]
    y = myLabeler.fit_transform(labels)

    # put features into matrix
    X = np.array( thisDf ).reshape(-1,thisDf.shape[-1])
    
    return X,y
X,y = makeXy(50,3.0,dims=2)
X
X,y =  makeXy(50,3.0,dims=1)
zip(X[::20],y[::20])<jupyter_output><empty_output><jupyter_text>split the nest set into training and validation set<jupyter_code>def dropNans(X,y):
    # get rid of nans for (because they throw an error when we try to fit the classifier)
    
    # this works also for multi-feature X (each observation row has multiple columns/features)
    noNanTrain = np.where(np.isnan(X).sum(axis=1)==False)
    
    # X and y need to be changed, otherwise the order of real and predicted labels
    # will not match
    X = X[noNanTrain[0]]
    y = y[noNanTrain[0]]
    return X,y
def dropInc(X,y,myLabeler=myLabeler):

    # here, we use the labels (y) to find and exclude inconclusive cases
    noInc = np.where(y!=myLabeler.transform(['inconclusive'])[-1])
    
    # both X and y need to be changed
    X = X[noInc[0]]
    y = y[noInc[0]]
    
    return X,y
def makeSplit(X,y,rdState=1,drop=True):
    
    thisSplit = model_selection.StratifiedShuffleSplit(n_splits=1,test_size=0.5,random_state=rdState)
    
    for train_index,validate_index in thisSplit.split(X,y):
        X_train,X_validate = X[train_index],X[validate_index]
        y_train,y_validate = y[train_index],y[validate_index]

    # for training, we cannot use nans, but for testing, we can later hard-code that
    # any nan should be classified as inconclusive
    X_train,y_train = dropNans(X_train,y_train)
    
    if drop:
        X_train,y_train = dropInc(X_train,y_train)
        X_validate,y_validate = dropInc(X_validate,y_validate)

    return X_train,X_validate,y_train,y_validate
X_train,X_validate,y_train,y_validate = makeSplit(X,y,rdState=1,drop=False)
print X_train.shape<jupyter_output>(257, 1)<jupyter_text>now both x and y are divided into two halves<jupyter_code>print X_train.shape,X_validate.shape,y_train.shape,y_validate.shape
X_train[-5:]<jupyter_output><empty_output><jupyter_text>fit the classifier to the training data<jupyter_code>def makeFit(X_train,y_train):
    
    scaler = preprocessing.StandardScaler()
    X_train_std = scaler.fit_transform(X_train)

    clf = SVC(kernel='linear', C=1.0, probability=True,random_state=0)
    clf.fit(X_train_std,y_train)
    
    return clf,scaler
clf,scaler = makeFit(X_train,y_train)<jupyter_output><empty_output><jupyter_text>use the classifier to predict the left-out validation data<jupyter_code>def makeYPreds(X_validate,clf,scaler,myLabeler=myLabeler):
    
    y_pred = []

    for x in X_validate:
        
        # when there is no LI, we consider the case to be inconclusive
        if np.isnan(x).any():
            pred = myLabeler.transform(['inconclusive'])[-1]
            
        # otherwise we apply the acutal classifier
        else:
            scaledX = scaler.transform( x.reshape(1,-1) )
            pred = clf.predict(scaledX)[-1]
            
        y_pred.append(pred)
    
    return np.array(y_pred)
y_pred = makeYPreds(X_validate,clf,scaler)
print y_pred<jupyter_output>[2 2 2 2 2 2 2 2 2 0 2 2 2 2 2 2 2 2 2 2 2 3 2 2 2 2 2 0 2 2 0 2 2 2 2 2 3
 2 2 2 2 0 2 2 0 2 1 2 2 3 2 2 2 2 2 2 2 2 2 2 2 2 2 0 2 2 2 2 3 2 2 2 0 2
 2 2 2 2 0 2 1 0 2 2 2 3 2 0 2 2 2 2 2 2 2 2 2 3 2 2 2 2 2 2 2 2 2 2 2 2 3
 2 2 2 2 2 0 0 2 2 2 2 0 0 3 2 2 2 3 2 2 2 2 2 0 2 2 0 3 2 2 2 0 2 2 2 2 2
 2 2 2 2 2 2 2 2 2 2 0 1 2 2 2 2 2 2 0 2 2 0 2 0 2 0 0 3 2 2 0 0 2 2 2 2 2
 2 2 2 2 2 2 2 3 2 2 2 2 0 2 2 3 0 2 3 1 2 0 2 3 2 2 3 2 0 2 2 2 2 2 2 2 2
 2 2 2 2 2 2 2 2 2 2 3 2 2 0 2 2 2 2 2 2 0 0 2 2 2 2 2 1 0 2 2 2 0 2 2 2 2
 2 2]<jupyter_text>get confusion matrix from prediction of testing data<jupyter_code>def makeCm(X_validate,y_validate,y_pred,clf,scaler,myLabeler=myLabeler):

    # predictions of left-out data
    y_pred = makeYPreds(X_validate,clf,scaler)
    # confusion matrix -- input y and y_pred -- result: true in rows, pred in columns
    cm = metrics.confusion_matrix(y_validate,y_pred)

    # put into df
    cmDf = pd.DataFrame(cm)

    # get the naming right
    try:
        theseLabels = myLabeler.inverse_transform( np.unique(y_validate) )
        cmDf.index = theseLabels
        cmDf.columns = theseLabels
    except:
        theseLabels = myLabeler.classes_
        cmDf.index = theseLabels
        cmDf.columns = theseLabels       

    # make better order
    theseLabels = ['left','bilateral','right','inconclusive']
    cmDf = cmDf.reindex(index=theseLabels)
    cmDf = cmDf.reindex(columns=theseLabels)
    
    cmDf.index = [['True label']*cmDf.shape[0],theseLabels]
    cmDf.columns = [['Predicted label']*cmDf.shape[1],theseLabels]

    return cmDf
cmDf = makeCm(X_validate,y_validate,y_pred,clf,scaler)
cmDf<jupyter_output><empty_output><jupyter_text>scale confusion matrix by total number of cases<jupyter_code>def makeScaleCm(cmDf):
    return ( cmDf.astype(float).T/cmDf.sum(axis=1) ).T
cmScaleDf = makeScaleCm(cmDf)
cmScaleDf<jupyter_output><empty_output><jupyter_text>sanity check: true labels for each class should add up to one<jupyter_code>cmScaleDf.T.sum()<jupyter_output><empty_output><jupyter_text>get diagonal (correct answers only)<jupyter_code>def makeDiagCm(cmDf,idx):
    
    # get diagonal
    diagDf = pd.DataFrame(np.diagonal(cmDf)).T
    diagDf.index=idx
    diagDf.columns=cmDf.columns
    
    return diagDf
diagDf = makeDiagCm(cmDf,[['diag'],[0]])
diagDf<jupyter_output><empty_output><jupyter_text>get accuracy from diagonal<jupyter_code>def getAcc(cmDf,drop=True):
    
    # get absolute count of hits (diagonal)
    n_diag = makeDiagCm(cmDf,[['total'],['total']]).sum(axis=1).values[-1]

    # get absolute count of all test data
    n_total = cmDf.sum().sum()

    # get proportion
    prop = n_diag/float(n_total)

    
    return pd.DataFrame([prop],index=['acc']).T
getAcc(cmDf,drop=True)<jupyter_output><empty_output><jupyter_text>Train in  nested way<jupyter_code>def getClfs(roiSize,dims,n_iter=100,drop=True):

    clfDict = {}
    accDict = {}

    #for threshold in df.columns:
    for threshold in np.arange(-5,15,0.2):
        # get the LIs for a threshold
        X,y = makeXy(roiSize,round(threshold,2),dims)
        # split into train and test
        
        thisAcc = []
        
        for r in range(n_iter):
            
            X_train,X_validate,y_train,y_validate = makeSplit(X,y,rdState=r,drop=drop)

            # train the classifer
            clf,scaler = makeFit(X_train,y_train)
            # make the predictions
            y_pred = makeYPreds(X_validate,clf,scaler)
            # get the confusion matrix
            cmDf = makeCm(X_validate,y_validate,y_pred,clf,scaler)
            # get the diagonal (correct responses)
            diagDf = makeDiagCm(cmDf,[['diag'],[0]])    
        
            thisAcc.append( getAcc(cmDf).iloc[0].acc )
        
        # store the classifier and the scaler, but trained on the whole data,
        # because we will later apply it to the left-out dataset
        X,y = dropNans(X,y)
        
        if drop:
            X,y = dropInc(X,y)
            
        clf,scaler = makeFit(X,y)
        clfDict[round(threshold,1)] = {'clf':clf,'scaler':scaler}
        
        # store the average performance for the left-out data using the nested-cv results above
        accDict[round(threshold,1)] = np.mean(thisAcc)

    return clfDict,accDict<jupyter_output><empty_output><jupyter_text>Example for one ROI size (df)<jupyter_code>clfDict,accDict = getClfs(50,1,n_iter=1,drop=True)<jupyter_output><empty_output><jupyter_text>Example of a stored classifer-scaler pair<jupyter_code>print clfDict[clfDict.keys()[-1]]<jupyter_output>{'clf': SVC(C=1.0, cache_size=200, class_weight=None, coef0=0.0,
  decision_function_shape=None, degree=3, gamma='auto', kernel='linear',
  max_iter=-1, probability=True, random_state=0, shrinking=True, tol=0.001,
  verbose=False), 'scaler': StandardScaler(copy=True, with_mean=True, with_std=True)}<jupyter_text>Example of the accuracies for all thresholds of one ROI size:<jupyter_code>print accDict<jupyter_output>{0.0: 0.87037037037037035, 1.0: 0.89351851851851849, 2.0: 0.89814814814814814, 3.0: 0.91203703703703709, 4.0: 0.90277777777777779, 5.0: 0.89814814814814814, 6.0: 0.875, 7.0: 0.87037037037037035, 8.0: 0.84259259259259256, 9.0: 0.82407407407407407, 10.0: 0.79629629629629628, 11.0: 0.7407407407407407, 12.0: 0.6157407407407407, 13.0: 0.52314814814814814, 14.0: 0.44444444444444442, 12.8: 0.55555555555555558, -2.2: 0.82870370370370372, 8.2: 0.83333333333333337, 0.2: 0.87962962962962965, 4.8: 0.90277777777777779, -4.4: 0.82407407407407407, 3.8: 0.90740740740740744, 7.4: 0.85648148148148151, -4.8: 0.81944444444444442, 12.6: 0.56944444444444442, 13.4: 0.47222222222222221, 0.4: 0.87962962962962965, 8.8: 0.82407407407407407, 0.6: 0.875, 9.4: 0.81481481481481477, 3.2: 0.91203703703703709, 1.6: 0.89814814814814814, -1.4: 0.82407407407407407, 0.8: 0.8842592592592593, 6.2: 0.86111111111111116, -4.6: 0.81944444444444442, 3.6: 0.91666666666666663, 1.2: 0.89814814814814814, -1.2: 0.83796296296296291, 13[...]<jupyter_text>Do this for all ROI sizes<jupyter_code>def getAllClfs(dims,drop,n_iter=100):
    
    clfDict = {}
    accDict = {}
    
    for pc in range(0,96,5):
        #print pc
        # this does everything to get and evaluate the classifiers
        thisClfDict,thisAccDict = getClfs(pc,dims,n_iter=n_iter,drop=drop)
        clfDict[pc] = thisClfDict
        accDict[pc] = thisAccDict
        
    return clfDict,accDict<jupyter_output><empty_output><jupyter_text>Example:<jupyter_code>clfDict,accDict = getAllClfs(dims=1,drop=True,n_iter=2)
print clfDict.keys()
print clfDict[0].keys()<jupyter_output>[0.0, 1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0, 10.0, 11.0, 12.0, 13.0, 14.0, 12.8, -2.2, 8.2, 0.2, 4.8, -4.4, 3.8, 7.4, -4.8, 12.6, 13.4, 0.4, 8.8, 0.6, 9.4, 3.2, 1.6, -1.4, 0.8, 6.2, -4.6, 3.6, 1.2, -1.2, 13.8, 12.4, 3.4, 13.2, 2.2, 10.4, 13.6, 5.8, 8.6, 6.4, 7.8, 2.4, 5.4, -4.2, 10.6, 4.6, 5.6, 9.8, 11.6, 11.8, -3.8, 12.2, -1.8, 4.4, 1.4, 2.6, 14.2, 5.2, -2.4, -2.6, -3.4, -0.2, 9.6, 14.8, 1.8, -2.8, 11.2, 2.8, 6.6, 6.8, 7.6, 10.2, -3.2, -1.6, 7.2, -3.6, -0.4, 9.2, 8.4, 10.8, 11.4, 4.2, 14.6, -0.8, -0.6, 14.4, -1.0, -5.0, -4.0, -3.0, -2.0]<jupyter_text>show all accuracies<jupyter_code>accDf = pd.DataFrame(accDict).T
sns.heatmap( accDf, cmap='RdBu_r')<jupyter_output><empty_output><jupyter_text>Run everything for each of the 4 versions & save<jupyter_code>from datetime import datetime
for thisDim in [1,2]:
    for thisDrop in [True,False]:
        print thisDim,thisDrop, datetime.now()
        # fit the classifier
        clfDict,accDict = getAllClfs(dims=thisDim,drop=thisDrop,n_iter=100)
        
        # save for later re-use
        dropStr = ['full','drop'][thisDrop]
        dimStr = ['1d','2d'][thisDim-1]
        
        pickle.dump(clfDict, open('../models/clf_%s_%s.p'%(dimStr,dropStr),'wb'))
        pickle.dump(accDict, open('../models/acc_%s_%s.p'%(dimStr,dropStr),'wb'))<jupyter_output>1 True 2019-01-23 08:57:30.577788
1 False 2019-01-23 10:53:03.327269
2 True 2019-01-23 13:19:15.983233
2 False 2019-01-23 15:22:15.653904
{'fraction_non_alphanumeric': 0.13115665161514414, 'fraction_numerical': 0.10163251128864188, 'mean_word_length': 3.466645982004344, 'pattern_counts': {'":': 0, '<': 91, '<?xml version=': 0, '>': 91, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 11, 'word_list_counts': {'cursed_substrings.json': 43, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '24', 'dir': 'jupyter-structured-clean-dedup', 'id': '25431', 'license': 'MIT', 'n_tokens_mistral': 6705, 'n_tokens_neox': 5714, 'n_words': 1316, 'path': 'notebooks/06-mw-train-classifier.ipynb', 'repo_name': 'mwegrzyn/laterality-index-deconstruction', 'size': '14395'}