<jupyter_start><jupyter_text>In this notebook, we follow the pilot test noted in the [previous notebook](https://github.com/tingsyo/dlgridsat/blob/main/notebook/09_GLM_with_feature_vectors.ipynb), and wrap up the process with:- [cross validation](https://scikit-learn.org/stable/modules/cross_validation.html),- more detailed [evaluation metrics](https://en.wikipedia.org/wiki/Confusion_matrix), and- looping through event-types and feature-vector-types.<jupyter_code>import numpy as np
import pandas as pd
import os, re, logging

logging.basicConfig(level=logging.DEBUG)

# Parameters
FV_NAMES = ['PCA', 'CAE', 'CVAE', 'Pretrained-BigEarth', 'Pretrained-ImageNet']
FV_FILES = ['fv_pca.zip', 'fv_cae.zip', 'fv_cvae.zip', 'fv_ptbe.zip', 'fv_ptin.zip']
EVENT_NAMES = []
EVENT_FILE = 'tad_filtered.csv'
CV_FOLD = 10
DATAPATH = '../data/'


# Read events
def read_events(file, verbose=0):
    events = pd.read_csv(file, index_col=0)
    if verbose>0:
        print(events.shape)
        for c in events.columns:
            print(c + '\t counts: ' + str(events[c].sum()) + 
                         '\t prob:' + str(events[c].sum()/events.shape[0]))
    return(events)


# Function to give report for binary classifications
def evaluate_binary(yt, yp, id=None, ythresh=1.):
    from sklearn.metrics import confusion_matrix
    ytb = (yt>=ythresh)*1
    ypb = (yp>=ythresh)*1
    # Derive metrics
    output = {'id':id}
    TN, FP, FN, TP = confusion_matrix(ytb, ypb).ravel()
    output['true_positive'] = np.round(TP,2)
    output['false_positive'] = np.round(FP,2)
    output['false_negative'] = np.round(FN,2)
    output['true_negative'] = np.round(TN,2)
    output['sensitivity'] = np.round(TP/(TP+FN),2)
    output['specificity'] = np.round(TN/(FP+TN),2)
    output['prevalence'] = np.round((TP+FN)/(FN+TP+FP+TN),8)
    output['ppv'] = np.round(TP/(TP+FP),4)
    output['npv'] = np.round(TN/(TN+FN),4)
    output['fpr'] = np.round(FP/(FP+TN),4)
    output['threat_score'] = np.round(TP/(TP+FP+FN),4)
    output['accuracy'] = np.round((TP+TN)/(FN+TP+FP+TN),4)
    output['balanced_accuracy'] = np.round((TP/(TP+FN) + TN/(FP+TN))*0.5,4)
    output['F1'] = np.round(2*TP/(2*TP+FP+FN),4)
    output['MCC'] = np.round((TP*TN-FP*FN)/np.sqrt((TP+FP)*(TP+FN)*(TN+FP)*(TN+FN)),4)
    output['informedness'] = np.round(output['sensitivity'] + output['specificity'] - 1,4)
    output['markedness'] = np.round(output['ppv'] + output['npv'] -1,4)
    return(output)

# Test
events = read_events(DATAPATH+EVENT_FILE, verbose=1)
fv = pd.read_csv(DATAPATH+FV_FILES[0], compression='zip', index_col=0)
print(events.shape)
print(fv.shape)
# Clean up NAs
def clean_nans(x, y):
    # Merge x and y by index
    data = pd.merge(y, x, left_index=True, right_index=True)
    nrow_original = data.shape[0]
    # Drop NaNs
    data = data.dropna(axis=0, how='any')
    nrow_dropna = data.shape[0]
    # Retrieve original index as dates and reset
    dates = data.index
    data = data.reset_index(drop=True)
    print('Dropping rows containing NaNs: ' + str(nrow_original - nrow_dropna))
    y = data.iloc[:,0]
    x = data.iloc[:,1:]
    return((x, y, dates))

# Create cross validation folds
def create_cv_folds(x, y, kfold=5, randseed=123):
    from sklearn.model_selection import StratifiedKFold, KFold
    # clean up nans
    x, y, dates = clean_nans(x, y)
    # Create CV
    skf = StratifiedKFold(n_splits=kfold, shuffle=True, random_state=randseed)
    splits = skf.split(x, y)
    # 
    return((splits, x, y, dates))

# Test
splits, x, y, dates = create_cv_folds(x=fv, y=events['HRD'])

print('Class distributions (0,1) in CV folds:')
for train, test in splits:
    print('train -  {}   |   test -  {}'
          .format(np.bincount(y[train]), np.bincount(y[test])))<jupyter_output>Dropping rows containing NaNs: 13
Class distributions (0,1) in CV folds:
train -  [826 332]   |   test -  [206  84]
train -  [825 333]   |   test -  [207  83]
train -  [825 333]   |   test -  [207  83]
train -  [826 333]   |   test -  [206  83]
train -  [826 333]   |   test -  [206  83]<jupyter_text>Thus, we can create CV folds and iterate through them.The package `scikit-learn` provides a faster way to perform such task with [`sklearn.model_selection.cross_val_predict`](https://scikit-learn.org/stable/modules/generated/sklearn.model_selection.cross_val_predict.html) and [`sklearn.model_selection.cross_validate`](https://scikit-learn.org/stable/modules/generated/sklearn.model_selection.cross_validate.htmlsklearn.model_selection.cross_validate). We can directly get the predictions with CV with this function, and then get evaluations.<jupyter_code># Evaluate one FV-Event pair
def evaluate_fv_event_pair(fv, event, fv_id=None, kfold=5, randseed=123):
    from sklearn.model_selection import StratifiedKFold
    from sklearn.linear_model import LogisticRegression
    from sklearn.model_selection import cross_val_predict, cross_validate
    # Create CV folds
    x, y, dates = clean_nans(x=fv, y=events['HRD'])
    skf = StratifiedKFold(n_splits=kfold, shuffle=True, random_state=randseed)
    clf = LogisticRegression(random_state=0, max_iter=1000)
    # 
    y_pred = cross_val_predict(clf, x, y, cv=skf)
    eval_all = evaluate_binary(y, y_pred, id=fv_id)
    print(eval_all)
    # 
    eval_cv = cross_validate(clf, x, y, cv=skf, return_train_score=True,
                             scoring=['accuracy', 'recall', 'precision', 'roc_auc', 'f1'])
    print(eval_cv)
    return(eval_all, eval_cv)


# test
eval_all, eval_cv = evaluate_fv_event_pair(fv, events['HRD'], fv_id='PCA-HRD', kfold=10)
for k in eval_cv.keys():
    print(k+'_mean'+ '\t' + str(np.round(np.mean(eval_cv[k]),4)))
    print(k+'_std'+ '\t' + str(np.round(np.std(eval_cv[k]),4)))<jupyter_output>fit_time_mean	0.2057
fit_time_std	0.0112
score_time_mean	0.0234
score_time_std	0.0007
test_accuracy_mean	0.6754
test_accuracy_std	0.0202
train_accuracy_mean	1.0
train_accuracy_std	0.0
test_recall_mean	0.3751
test_recall_std	0.0602
train_recall_mean	1.0
train_recall_std	0.0
test_precision_mean	0.427
test_precision_std	0.0404
train_precision_mean	1.0
train_precision_std	0.0
test_roc_auc_mean	0.6299
test_roc_auc_std	0.0508
train_roc_auc_mean	1.0
train_roc_auc_std	0.0
test_f1_mean	0.3972
test_f1_std	0.0455
train_f1_mean	1.0
train_f1_std	0.0<jupyter_text>Now, we can loop through all FV-types and event-types, and collect the evaluation results. Let's beatify the output of the evaluation of single FV-Event pair.<jupyter_code># Evaluate one FV-Event pair
def evaluate_fv_event_pair(fv, event, fv_id=None, kfold=5, randseed=123):
    from sklearn.model_selection import StratifiedKFold
    from sklearn.linear_model import LogisticRegression
    from sklearn.model_selection import cross_val_predict, cross_validate
    # Create CV folds
    x, y, dates = clean_nans(x=fv, y=event)
    skf = StratifiedKFold(n_splits=kfold, shuffle=True, random_state=randseed)
    clf = LogisticRegression(random_state=0, max_iter=10000)
    # Get predictions from CV
    y_pred = cross_val_predict(clf, x, y, cv=skf)
    eval_all = evaluate_binary(y, y_pred, id=fv_id)
    #print(eval_all)
    # Get evaluations for each CV fold
    eval_cv = cross_validate(clf, x, y, cv=skf, return_train_score=True,
                             scoring=['accuracy', 'recall', 'precision', 'roc_auc', 'f1'])
    for k in eval_cv.keys():
        eval_all[k+'_mean'] = np.round(np.mean(eval_cv[k]),4)
        eval_all[k+'_std'] = np.round(np.std(eval_cv[k]),4)
    #print(eval_cv)
    return(eval_all, eval_cv)


# test
eval_all, eval_cv = evaluate_fv_event_pair(fv, events['HRD'], fv_id='PCA-HRD', kfold=10)
print(eval_all)
# Parameters
FV_NAMES = ['PCA', 'CAE', 'CVAE', 'Pretrained-BigEarth', 'Pretrained-ImageNet']
FV_FILES = ['fv_pca.zip', 'fv_cae.zip', 'fv_cvae.zip', 'fv_ptbe.zip', 'fv_ptin.zip']
EVENT_NAMES = ['CS', 'TYW', 'NWPTY', 'FT', 'NE', 'SWF', 'HRD', 'HRH']
EVENT_FILE = 'tad_filtered.csv'
CV_FOLD = 10
DATAPATH = '../data/'

events = read_events(DATAPATH+EVENT_FILE, verbose=1)
results = []
cv_details = {}

# Loop
for i in range(len(FV_NAMES)):
    for j in range(len(EVENT_NAMES)):
        fv = pd.read_csv(DATAPATH+FV_FILES[i], compression='zip', index_col=0)
        fv_name = FV_NAMES[i]
        event_name = EVENT_NAMES[j]
        exp_id = fv_name+'-'+event_name
        print(exp_id)
        eval_all, eval_cv = evaluate_fv_event_pair(fv, events[event_name], fv_id=exp_id, kfold=10)
        results.append(eval_all)
        cv_details[exp_id] = eval_cv


pd.DataFrame(results).to_csv('../data/exp_results.csv', index=False)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.14646641052509024, 'fraction_numerical': 0.027477005472115498, 'mean_word_length': 4.342039800995025, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 14, 'https://': 5, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '644324', 'license': 'CC0-1.0', 'n_tokens_mistral': 3480, 'n_tokens_neox': 3265, 'n_words': 653, 'path': 'notebook/10_Cross_validation_and_more_evaluation_metrics.ipynb', 'repo_name': 'tingsyo/dlgridsat', 'size': '8589'}