<jupyter_start><jupyter_text>Deepscent development[![(Open In Colab)](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/Withington/deepscent/blob/master/notebooks/deepscent_dev.ipynb)Train and test the models used in developing Deepscent. The 'GunPoint' dataset from the [UEA & UCR Time Series Classification Repository](http://www.timeseriesclassification.com "timeseriesclassification.com") is used in place of the detection dogs' data.MIT license to use [software by <NAME>](https://github.com/cauchyturing/UCR_Time_Series_Classification_Deep_Learning_Baseline/blob/master/README.md).<jupyter_code>import os
import sys
from pathlib import Path
import time
from datetime import datetime
from dateutil.tz import gettz
import itertools

import numpy as np
import pandas as pd
import tensorflow as tf
import tensorflow.keras as keras

from tensorflow.keras.models import Model, Sequential
from tensorflow.keras.layers import Input, Dense, Activation, Dropout
from tensorflow.keras import regularizers
from tensorflow.keras.initializers import RandomUniform
from tensorflow.keras import utils
from tensorflow.keras.callbacks import ReduceLROnPlateau, EarlyStopping

import matplotlib.pyplot as plt
from matplotlib.ticker import FuncFormatter
import seaborn as sns
from sklearn.model_selection import KFold, RepeatedStratifiedKFold
from sklearn import preprocessing
from sklearn.metrics import confusion_matrix, roc_curve, roc_auc_score, classification_report

np.random.seed(999123)<jupyter_output><empty_output><jupyter_text>User inputs<jupyter_code># Select a model with hyperparameters as per Wang et al. (2017): 
# MLP, FCN, ResNet
# or
# select a model with hyperparameters tuned to optimise performance on the ACI detection dogs dataset:
# MLP_tuned, FCN_tuned, ResNet_tuned, CNN
model_type = 'MLP_tuned' # MLP, MLP_tuned, FCN, FCN_tuned, CNN, ResNet, ResNet_tuned

fname = 'GunPoint' # The dataset name, e.g. GunPoint, private_correct_plus

batch_size = 32 
k = 3 # k-fold cross validation: number of folds. If k=1, the original test-train split is used.
m = 1 # k-fold cross validation: number of repetitions (if k>1).
epochs_dict = {'MLP':5000, 'FCN':2000, 'ResNet':1500, 'MLP_tuned':500, 'FCN_tuned':1000, 'CNN':1000, 'ResNet_tuned':1000}
nb_epochs = epochs_dict[model_type]

truncate_data = False # Truncate pressure samples to first n data points
filter_data = False # Filter out noise below a threshold

data_augmentation = False
tensorboard = True # Set to True to write logs for use by TensorBoard
k_fold_seed = 765432

# Output directories
logs_dir = '../logs'
tensorboard_dir = '../logs/tensorboard'
timestamp = '{:%Y-%m-%dT%H:%M}'.format(datetime.now(gettz("Europe/London")))
logs_dir = logs_dir +'/' + timestamp
tensorboard_dir = tensorboard_dir +'/' + timestamp<jupyter_output><empty_output><jupyter_text>Tools<jupyter_code>def plot_confusion_matrix(cm, title='Normalised confusion matrix', name=''):
    ''' Plot the normalised confusion matrix
    Parameters
    cm : array - normalised confusion matrix
    Scikit-learn: Machine Learning in Python, Pedregosa et al., JMLR 12, pp. 2825-2830, 2011.
    'Confusion Matrix' https://scikit-learn.org/stable/auto_examples/model_selection/plot_confusion_matrix.html#sphx-glr-auto-examples-model-selection-plot-confusion-matrix-py
    '''
    classes = ['Positive', 'Negative']
    cmap=plt.cm.Blues
    sns.set_style('dark')
    plt.figure()
    plt.imshow(cm, interpolation='nearest', cmap=cmap)
    plt.title(title)
    plt.colorbar(format=FuncFormatter('{0:.0%}'.format))
    tick_marks = np.arange(len(classes))
    plt.xticks(tick_marks, classes, rotation=45)
    plt.yticks(tick_marks, classes)
    plt.clim(0, 1)
    fmt = '.0%'
    thresh = cm.max() / 2.
    for i, j in itertools.product(range(cm.shape[0]), range(cm.shape[1])):
        plt.text(j, i, format(cm[i, j], fmt),
                 horizontalalignment="center",
                 color="white" if cm[i, j] > thresh else "black")
    plt.ylabel('True class')
    plt.xlabel('Predicted class')
    plt.tight_layout()
    file_name = 'cm_deepscent_dev_'+name+'.png'
    plt.savefig(file_name, bbox_inches='tight')
        
        
def plot_roc(y_true, y_probs, name): 
    ''' Plot ROC and return AUC
    Parameters
    y_true : vector of true class labels
    y_probs : vector of predicted probabilities
    Returns
    auc : float
    '''
    fpr, tpr, thresholds = roc_curve(y_true, y_probs)
    auc = roc_auc_score(y_true, y_probs)
    sns.set_style('whitegrid')
    plt.figure()
    plt.plot(fpr, tpr, color='darkorange',
             lw=2, label='ROC curve (area = %0.2f)' % auc)
    plt.plot([0, 1], [0, 1], color='navy', lw=2, linestyle='--')
    plt.xlim([0.0, 1.0])
    plt.ylim([0.0, 1.05])
    plt.xlabel('False Positive Rate')
    plt.ylabel('True Positive Rate')
    plt.title('Receiver operating characteristic curve')
    plt.legend(loc="lower right")
    file_name = 'roc_deepscent_dev_'+name+'.png'
    plt.savefig(file_name, bbox_inches='tight')
    return auc


def filter_out(x, threshold):
    ''' Filter out any data points in x that are below the threshold by setting them to zero.
    Return the modified data, x '''
    if x < threshold:
        return 0
    return x
    
    
def preprocess(X):
    ''' Apply preprocessing to the input data X'''
    if filter_data:
        threshold = 0.1
        X = np.piecewise(X, [X < threshold, X >= threshold], [lambda X: 0, lambda X: X])
    return X
    
    
def readucr(filename):
    ''' Load a dataset from a file in UCR format
    space delimited, class labels in the first column.
    Returns
    X : input data, one sample per row
    Y : class labels corresponding to each row of X
    '''
    data = np.loadtxt(Path(filename))
    Y = data[:,0]
    X = data[:,1:]
    if truncate_data:
        X = X[:,:300]
    X = preprocess(X)
    return X, Y


def load_data_from_web(url, sep=' '):
    ''' Load the data file from a url.
    File format - UCR TSC Archive
    i.e. space delimited, class labels in the first column.
    Returns
    X : input data, one sample per row
    Y : class labels corresponding to each row of X
    '''
    df = pd.read_csv(url, sep=sep, header=None)
    print('Loaded data from', url)
    Y = df.values[:,0].astype(int)
    X = df.values[:,1:]
    if truncate_data:
        X = X[:,:300]
    X = preprocess(X)
    return X, Y


def reshape(x, model_type):
    ''' Reshape data into input format for the selected DNN '''
    if model_type == 'ResNet':
        return reshape_2d(x)
    elif model_type == 'FCN' or model_type == 'FCN_tuned' or model_type == 'CNN' or model_type == 'ResNet_tuned':
        return reshape_1d(x)
    elif model_type == 'MLP' or model_type == 'MLP_tuned':
        return x
    else:
        raise ValueError('Unrecognised model type')
    return x


def augment_data(x, y):
    ''' Return n times as many data samples, x, and labels, y. The augmented data is generated 
    by applying a shift to each row of x and appending these new rows to x '''
    m = x.shape[1]
    x_new = x
    y_new = y
    for shift in range(-50, 60, 10):
        x_aug = np.zeros_like(x)
        if shift < 0:
            x_aug[:,:m+shift] = x[:,-shift:]
        elif shift > 0:
            x_aug[:,shift:] = x[:,:m-shift]
        elif shift == 0:
            continue
        x_new = np.concatenate((x_new, x_aug), axis=0)
        y_new = np.concatenate((y_new, y), axis=0)
    return x_new, y_new<jupyter_output><empty_output><jupyter_text>Load data<jupyter_code>do_end_test = False    # For each fold, evaluate model on the end_test set too.

if 'google.colab' in sys.modules:
    url_data_dir = 'https://raw.githubusercontent.com/Withington/deepscent/master/data'
    root = url_data_dir+'/'+fname+'/'+fname
    sep = ' ' if 'DetectionDogMockData' in fname else '  '
    x_train, y_train = load_data_from_web(root+'_TRAIN.txt', sep) 
    x_test, y_test = load_data_from_web(root+'_TEST.txt', sep) 
    if 'DetectionDogMockData' in fname:
        x_other, y_other= load_data_from_web(root+'_END_TEST.txt')
        do_end_test = True
else:   
    if 'private' in fname:
        fdir = '../data/private_data/private_events_dev2' 
    else:
        fdir = '../data' 
    root = fdir+'/'+fname+'/'+fname
    x_train, y_train = readucr(root+'_TRAIN.txt')
    x_test, y_test = readucr(root+'_TEST.txt')
    if 'correct_plus' in fname or 'DetectionDogMockData' in fname:
        x_other, y_other = readucr(root+'_END_TEST.txt')
        do_end_test = True
        
print('Data loaded from', root+'_...txt')
print('Training dataset size:', x_train.shape[0])
print('Test set size:', x_test.shape[0])
if do_end_test:
    print('Alternative test set size:', x_other.shape[0])<jupyter_output><empty_output><jupyter_text>Build DNNBuild a binary classifier. Model types: MLP, FCN, ResNet, CNN. ResNetResNet with hyperparameters as per Wang et al. (2017).<jupyter_code>def reshape_2d(x):
    ''' Reshape data into input format for ResNet '''
    x = x.reshape(x.shape + (1,1,))
    return x


def build_resnet(input_shape, nb_classes):
    ''' Build ResNet DNN and return input and output tensors '''
    # Parameters
    k = 1 # kernel multiplier
    n_feature_maps = 128
    
    print ('build conv_x')
    x = Input(shape=(input_shape))
    conv_x = keras.layers.BatchNormalization()(x)
    conv_x = keras.layers.Conv2D(n_feature_maps, 8*k, 1, padding='same')(conv_x)
    conv_x = keras.layers.BatchNormalization()(conv_x)
    conv_x = Activation('relu')(conv_x)
     
    print ('build conv_y')
    conv_y = keras.layers.Conv2D(n_feature_maps, 5*k, 1, padding='same')(conv_x)
    conv_y = keras.layers.BatchNormalization()(conv_y)
    conv_y = Activation('relu')(conv_y)
     
    print ('build conv_z')
    conv_z = keras.layers.Conv2D(n_feature_maps, 3*k, 1, padding='same')(conv_y)
    conv_z = keras.layers.BatchNormalization()(conv_z)
     
    is_expand_channels = not (input_shape[-1] == n_feature_maps)
    if is_expand_channels:
        print ('Expanding channels')
        shortcut_y = keras.layers.Conv2D(n_feature_maps, 1, 1,padding='same')(x)
        shortcut_y = keras.layers.BatchNormalization()(shortcut_y)
    else:
        shortcut_y = keras.layers.BatchNormalization()(x)
    print ('Merging skip connection')
    y = keras.layers.add([shortcut_y, conv_z])
    y = Activation('relu')(y)
     
    print ('build conv_x')
    x1 = y
    conv_x = keras.layers.Conv2D(n_feature_maps*2, 8*k, 1, padding='same')(x1)
    conv_x = keras.layers.BatchNormalization()(conv_x)
    conv_x = Activation('relu')(conv_x)
         
    print ('build conv_y')
    conv_y = keras.layers.Conv2D(n_feature_maps*2, 5*k, 1, padding='same')(conv_x)
    conv_y = keras.layers.BatchNormalization()(conv_y)
    conv_y = Activation('relu')(conv_y)
     
    print ('build conv_z')
    conv_z = keras.layers.Conv2D(n_feature_maps*2, 3*k, 1, padding='same')(conv_y)
    conv_z = keras.layers.BatchNormalization()(conv_z)
     
    is_expand_channels = not (input_shape[-1] == n_feature_maps*2)
    if is_expand_channels:
        print ('Expanding channels')
        shortcut_y = keras.layers.Conv2D(n_feature_maps*2, 1, 1,padding='same')(x1)
        shortcut_y = keras.layers.BatchNormalization()(shortcut_y)
    else:
        shortcut_y = keras.layers.BatchNormalization()(x1)
    print ('Merging skip connection')
    y = keras.layers.add([shortcut_y, conv_z])
    y = Activation('relu')(y)
     
    print ('build conv_x')
    x1 = y
    conv_x = keras.layers.Conv2D(n_feature_maps*2, 8*k, 1, padding='same')(x1)
    conv_x = keras.layers.BatchNormalization()(conv_x)
    conv_x = Activation('relu')(conv_x)
     
    print ('build conv_y')
    conv_y = keras.layers.Conv2D(n_feature_maps*2, 5*k, 1, padding='same')(conv_x)
    conv_y = keras.layers.BatchNormalization()(conv_y)
    conv_y = Activation('relu')(conv_y)
     
    print ('build conv_z')
    conv_z = keras.layers.Conv2D(n_feature_maps*2, 3*k, 1, padding='same')(conv_y)
    conv_z = keras.layers.BatchNormalization()(conv_z)

    is_expand_channels = not (input_shape[-1] == n_feature_maps*2)
    if is_expand_channels:
        print ('Expanding channels')
        shortcut_y = keras.layers.Conv2D(n_feature_maps*2, 1, 1,padding='same')(x1)
        shortcut_y = keras.layers.BatchNormalization()(shortcut_y)
    else:
        shortcut_y = keras.layers.BatchNormalization()(x1)
    print ('Merging skip connection')
    y = keras.layers.add([shortcut_y, conv_z])
    y = Activation('relu')(y)
     
    full = keras.layers.GlobalAveragePooling2D()(y)   
    out = Dense(1, activation='sigmoid')(full)
    print ('        -- model was built.')
    return x, out<jupyter_output><empty_output><jupyter_text>ResNet tunedResNet with hyperparameters tuned to optimise performance on the ACI dataset.<jupyter_code>def build_resnet_tuned(input_shape):
    ''' Return ResNet model '''  
    # Hyperparameters
    num_features0 = 64
    num_features1 = 128
    filter_size = 4
    pooling_size = 8
    dropout = 0.5  
    
    # Preparation block
    x = Input(shape=(input_shape))
    conv = keras.layers.Conv1D(num_features0, filter_size, padding='same')(x)
    conv = keras.layers.BatchNormalization()(conv)
    conv = Activation('relu')(conv)
    conv = keras.layers.MaxPooling1D(pooling_size)(conv)
    
    # First block
    skip = conv
    conv = keras.layers.Conv1D(num_features0, filter_size, padding='same')(conv)
    conv = keras.layers.BatchNormalization()(conv)
    conv = Activation('relu')(conv)
    
    conv = keras.layers.Conv1D(num_features0, filter_size, padding='same')(conv)
    conv = keras.layers.BatchNormalization()(conv)
    conv = Activation('relu')(conv)
    
    conv = keras.layers.Conv1D(num_features1, filter_size, padding='same')(conv)
    conv = keras.layers.BatchNormalization()(conv)
    conv = Activation('relu')(conv)
    
    conv = keras.layers.Conv1D(num_features1, filter_size, padding='same')(conv)
    conv = keras.layers.BatchNormalization()(conv)
    shortcut = keras.layers.Conv1D(num_features1, 1, padding='same')(skip)
    shortcut = keras.layers.BatchNormalization()(shortcut)
    conv = keras.layers.add([conv, shortcut])
    conv = Activation('relu')(conv)
    
    # Second block
    skip = conv
    conv = keras.layers.Conv1D(num_features0, filter_size, padding='same')(conv)
    conv = keras.layers.BatchNormalization()(conv)
    conv = Activation('relu')(conv)
    
    conv = keras.layers.Conv1D(num_features0, filter_size, padding='same')(conv)
    conv = keras.layers.BatchNormalization()(conv)
    conv = Activation('relu')(conv)
    
    conv = keras.layers.Conv1D(num_features1, filter_size, padding='same')(conv)
    conv = keras.layers.BatchNormalization()(conv)
    conv = Activation('relu')(conv)
    
    conv = keras.layers.Conv1D(num_features1, filter_size, padding='same')(conv)
    conv = keras.layers.BatchNormalization()(conv)
    shortcut = keras.layers.Conv1D(num_features1, 1, padding='same')(skip)
    shortcut = keras.layers.BatchNormalization()(shortcut)
    conv = keras.layers.add([conv, shortcut])
    conv = Activation('relu')(conv)
    
    # Third block
    skip = conv
    conv = keras.layers.Conv1D(num_features0*2, filter_size, padding='same')(conv)
    conv = keras.layers.BatchNormalization()(conv)
    conv = Activation('relu')(conv)
    
    conv = keras.layers.Conv1D(num_features0*2, filter_size, padding='same')(conv)
    conv = keras.layers.BatchNormalization()(conv)
    conv = Activation('relu')(conv)
    
    conv = keras.layers.Conv1D(num_features1*2, filter_size, padding='same')(conv)
    conv = keras.layers.BatchNormalization()(conv)
    conv = Activation('relu')(conv)
    
    conv = keras.layers.Conv1D(num_features1*2, filter_size, padding='same')(conv)
    conv = keras.layers.BatchNormalization()(conv)
    shortcut = keras.layers.Conv1D(num_features1*2, 1, padding='same')(skip)
    shortcut = keras.layers.BatchNormalization()(shortcut)
    conv = keras.layers.add([conv, shortcut])
    conv = Activation('relu')(conv)
    
    # Output block
    full = keras.layers.GlobalAveragePooling1D()(conv)
    y = Dropout(dropout, name='Dropout')(full)
    out = Dense(1, activation='sigmoid')(full)
    return x, out<jupyter_output><empty_output><jupyter_text>FCNWith hyperparameters as per Wang et al. (2017).<jupyter_code>def reshape_1d(x):
    ''' Reshape data into input format for FCN or CNN'''
    x = x.reshape(x.shape + (1,))
    return x
    
    
def build_fcn(input_shape, nb_classes):
    ''' Build Fully Convolutional Network (FCN) and return input and output tensors '''
    # Parameters
    k = 1 # kernel multiplier
    n_feature_maps = 128
    
    print ('build conv_x')
    x = Input(shape=(input_shape))
    conv_x = x
    #conv_x = keras.layers.BatchNormalization()(conv_x)
    conv_x = keras.layers.Conv1D(n_feature_maps, 8*k, 1, padding='same')(conv_x)
    conv_x = keras.layers.BatchNormalization()(conv_x)
    conv_x = Activation('relu')(conv_x)
     
    print ('build conv_y')
    conv_y = keras.layers.Conv1D(n_feature_maps*2, 5*k, 1, padding='same')(conv_x)
    conv_y = keras.layers.BatchNormalization()(conv_y)
    conv_y = Activation('relu')(conv_y)
     
    print ('build conv_z')
    conv_z = keras.layers.Conv1D(n_feature_maps, 3*k, 1, padding='same')(conv_y)
    conv_z = keras.layers.BatchNormalization()(conv_z)
    conv_z = Activation('relu')(conv_z)
     
    full = keras.layers.GlobalAveragePooling1D()(conv_z)
    out = Dense(1, activation='sigmoid')(full)
    return x, out<jupyter_output><empty_output><jupyter_text>FCN tunedWith hyperparameters tuned to optimise performance on the ACI dataset.<jupyter_code>def build_fcn_tuned(input_shape, nb_classes):
    ''' Build Fully Convolutional Network (FCN) and return input and output tensors '''
    # Parameters
    feature_maps_a = 32
    feature_maps_b = 64
    feature_maps_c = 32
    filter_a = 4
    filter_b = 4
    filter_c = 4
    
    print ('build conv_x')
    x = Input(shape=(input_shape))
    conv_x = x
    conv_x = keras.layers.Conv1D(feature_maps_a, filter_a, 1, padding='same')(conv_x)
    conv_x = keras.layers.BatchNormalization()(conv_x)
    conv_x = Activation('relu')(conv_x)
     
    print ('build conv_y')
    conv_y = keras.layers.Conv1D(feature_maps_b, filter_b, 1, padding='same')(conv_x)
    conv_y = keras.layers.BatchNormalization()(conv_y)
    conv_y = Activation('relu')(conv_y)
     
    print ('build conv_z')
    conv_z = keras.layers.Conv1D(feature_maps_c, filter_c, 1, padding='same')(conv_y)
    conv_z = keras.layers.BatchNormalization()(conv_z)
    conv_z = Activation('relu')(conv_z)
     
    full = keras.layers.GlobalAveragePooling1D()(conv_z)
    out = Dense(1, activation='sigmoid')(full)
    return x, out<jupyter_output><empty_output><jupyter_text>CNNWith hyperparameters tuned to optimise performance on the ACI dataset.Using the CNN architecture of[<NAME>, 2018, Introduction to 1D Convolutional Neural Networks in Keras for Time Sequences](https://blog.goodaudience.com/introduction-to-1d-convolutional-neural-networks-in-keras-for-time-sequences-3a7ff801a2cf).<jupyter_code>def build_cnn_harus(input_shape, nb_classes):
    ''' Build a CNN and return input and output tensors '''
    # Parameters
    n_features_a = 128 # Ackermann 100 
    n_features_b = 128 # Ackermann 160
    filter_size = 16   # Ackermann 10
    pooling_size = 16   # Ackermann 3
    dropout = 0.5      # Ackermann 0.5
    has_dense_layer = True
    
    print ('build CNN HARUS')
    x = Input(shape=(input_shape))
    conv_x = x
    conv_x = keras.layers.Conv1D(n_features_a, filter_size, activation='relu')(conv_x)
    conv_x = keras.layers.Conv1D(n_features_a, filter_size, activation='relu')(conv_x)
    conv_x = keras.layers.MaxPooling1D(pooling_size)(conv_x)
    conv_x = keras.layers.Conv1D(n_features_b, filter_size, activation='relu')(conv_x)
    
    if True:
        conv_x = keras.layers.Conv1D(n_features_b, filter_size, activation='relu')(conv_x)
    else:
        conv_x = keras.layers.Conv1D(n_features_b, filter_size)(conv_x)
        conv_x = keras.layers.BatchNormalization()(conv_x)
        conv_x = Activation('relu')(conv_x)
        
    if has_dense_layer: # End with a fully connected layer
        full = keras.layers.GlobalAveragePooling1D()(conv_x)
        full = Dropout(dropout,name='Dropout')(full)
        out = Dense(1, activation='sigmoid')(full)
    else:
        conv_x = keras.layers.Conv1D(16, filter_size, activation='relu')(conv_x)
        conv_x = Dropout(dropout,name='Dropout')(conv_x)
        conv_x = keras.layers.Conv1D(1, filter_size, activation='relu')(conv_x)
        conv_x = keras.layers.GlobalAveragePooling1D()(conv_x)
        out = Activation(activation='sigmoid')(conv_x)
    return x, out<jupyter_output><empty_output><jupyter_text>MLPWith hyperparameters as per Wang et al. (2017).<jupyter_code>def build_mlp(input_shape, nb_classes):
    num = 500
    x = Input(shape=(input_shape))
    y = Dropout(0.1,name='WDrop010')(x)
    y = Dense(num, activation='relu', name='WDense010')(y)
    y = Dropout(0.2,name='WDrop020')(y)
    y = Dense(num, activation='relu', name='WDense020')(y)
    y = Dropout(0.2,name='WDrop021')(y)
    y = Dense(num, activation='relu', name='WDense021')(y)
    y = Dropout(0.3,name='WDrop031')(y)
    out = Dense(1, activation='sigmoid', name='WDense080')(y)
    return x, out<jupyter_output><empty_output><jupyter_text>MLP tunedWith hyperparameters tuned to optimise performance on the ACI dataset.<jupyter_code>def build_mlp_tuned(input_shape, nb_classes):
    ''' Build a Multilayer Perceptron (MLP) and return input and output tensors '''
    drop = 0.2
    num = 16
    l2 = 0.1
    x = Input(shape=(input_shape))
    y = Dropout(drop,name='DropInput')(x)
    y = Dense(num, kernel_regularizer=regularizers.l2(l2), activation='relu', name='Dense010')(y)
    y = Dropout(drop,name='Drop010')(y)
    y = Dense(num, kernel_regularizer=regularizers.l2(l2), activation='relu', name='Dense020')(y)
    y = Dropout(drop,name='Drop020')(y)
    y = Dense(num, kernel_regularizer=regularizers.l2(l2), activation='relu', name='Dense030')(y)
    y = Dropout(drop,name='Drop030')(y)
    out = Dense(1, activation='sigmoid', name='DenseOutput')(y)
    return x, out<jupyter_output><empty_output><jupyter_text>Function: train model<jupyter_code>def train_model(fname, x_train, y_train, x_test, y_test, label="0"):
    ''' Build and train a DNN. Return summary info and a trained model '''
    print('Running dataset', fname)
    nb_classes = len(np.unique(y_test))
    if nb_classes != 2:
        raise 'Number of classes must be 2 to use this binary classifier'
    
    if data_augmentation:
        x_train, y_train = augment_data(x_train, y_train)
     
    Y_train = (y_train - y_train.min())/(y_train.max()-y_train.min())*(nb_classes-1)
    Y_test = (y_test - y_test.min())/(y_test.max()-y_test.min())*(nb_classes-1)
     
    x_train_mean = x_train.mean()
    x_train_std = x_train.std()
    x_train = (x_train - x_train_mean)/(x_train_std) 
    x_test = (x_test - x_train_mean)/(x_train_std)
     
    x_train = reshape(x_train, model_type)
    x_test = reshape(x_test, model_type)
    if model_type == 'MLP':
        x, y = build_mlp(x_train.shape[1:], nb_classes)
    elif model_type == 'MLP_tuned':
        x, y = build_mlp_tuned(x_train.shape[1:], nb_classes)
    elif model_type == 'ResNet':
            x, y = build_resnet(x_train.shape[1:], nb_classes)
    elif model_type == 'ResNet_tuned':
        x, y = build_resnet_tuned(x_train.shape[1:])
    elif model_type == 'FCN':
        x, y = build_fcn(x_train.shape[1:], nb_classes)
    elif model_type == 'FCN_tuned':
        x, y = build_fcn_tuned(x_train.shape[1:], nb_classes)
    elif model_type == 'CNN':
        x, y = build_cnn_harus(x_train.shape[1:], nb_classes)
    model = Model(x, y)
    print(model.summary())
    
    optimizer = keras.optimizers.Adam()
    model.compile(loss='binary_crossentropy',
                  optimizer=optimizer,
                  metrics=['acc'])
    
    Path(logs_dir+'/'+fname).mkdir(parents=True, exist_ok=True) 
    reduce_lr = ReduceLROnPlateau(monitor='loss', factor=0.5,
                      patience=50, min_lr=0.0001) 
    callbacks = [reduce_lr]
    if tensorboard:
        tb_dir = tensorboard_dir+'/'+fname+'_'+label
        Path(tb_dir).mkdir(parents=True, exist_ok=True) 
        print('Tensorboard logs in', tb_dir)
        callbacks.append(keras.callbacks.TensorBoard(log_dir=tb_dir, histogram_freq=0))
  
    start = time.time()
    hist = model.fit(x_train, Y_train, batch_size=batch_size, epochs=nb_epochs,
              verbose=1, validation_data=(x_test, Y_test), callbacks=callbacks)
    end = time.time()
    log = pd.DataFrame(hist.history) 
    
    # Print results
    duration_seconds = round(end-start)
    duration_minutes = str(round((end-start)/60))
    print('Training complete on', fname, 'Duration:', duration_seconds, 'secs; about', duration_minutes, 'minutes.')
    
    # Print and save results. Print the testing results that have the lowest training loss.
    print('Selected the test result with the lowest training loss. Loss and validation accuracy are -')
    idx = log['loss'].idxmin()
    loss = log.loc[idx]['loss']
    val_acc = log.loc[idx]['val_acc']
    epoch = idx + 1
    print(loss, val_acc, 'at index', str(idx), ' (epoch ', str(epoch), ')')
    summary = '|' + label + '  |'+str(loss)+'  |'+str(val_acc)+' |'+str(epoch)+' |'+ duration_minutes + 'mins  |'
    summary_csv = label+','+str(loss)+','+str(val_acc)+','+str(epoch)+','+ duration_minutes 
    
    # Save summary file and log file.
    print('Tensorboard logs in', tb_dir)
    history_file = logs_dir+'/'+fname+'/history_'+label+'.csv'
    print('Saving logs to', history_file)
    log.to_csv(history_file)
    
    model_params = {'x_train_mean':x_train_mean, 'x_train_std':x_train_std, 'threshold':0.5}
    return summary, summary_csv, model, model_params<jupyter_output><empty_output><jupyter_text>Train DNN<jupyter_code>''' Train a model, using repeated k-fold cross validation, if selected '''

results_file = logs_dir+'/'+fname+'/deepscent_dev_summary.csv'
results = []

if k > 1: # k-fold cross validation
    x_all = np.concatenate((x_train, x_test), axis=0)
    y_all = np.concatenate((y_train, y_test), axis=0)
    kfold = RepeatedStratifiedKFold(n_splits=k, n_repeats=m, random_state=k_fold_seed)
    count = 0
    for train, test in kfold.split(x_all, y_all):
        x_train, y_train, x_test, y_test = x_all[train], y_all[train], x_all[test], y_all[test]
        summary, summary_csv, model, model_params = train_model(fname, x_train, y_train, x_test, y_test, str(count))
        if do_end_test:
            x_in = (x_other - model_params['x_train_mean'])/(model_params['x_train_std'])
            x_in = reshape(x_in, model_type)
            _, end_test_acc = model.evaluate(x_in, y_other, batch_size=batch_size)
            summary = summary + str(end_test_acc) +' |'
            summary_csv = summary_csv + ',' + str(end_test_acc)
        with open(results_file, 'a+') as f:
            f.write(summary_csv)
            f.write('\n')
            print('Added summary row to ', results_file)
        results.append(summary)
        count = count + 1
else:
    summary, summary_csv, model, model_params = train_model(fname, x_train, y_train, x_test, y_test)
    if do_end_test:
        x_in = (x_other - model_params['x_train_mean'])/(model_params['x_train_std'])
        x_in = reshape(x_in, model_type)
        _, end_test_acc = model.evaluate(x_in, y_other, batch_size=batch_size)
        summary = summary + str(end_test_acc) +' |'
        summary_csv = summary_csv + ',' + str(end_test_acc)
    with open(results_file, 'a+') as f:
        f.write(summary_csv)
        f.write('\n')
        print('Added summary row to ', results_file)
    results.append(summary)
        
print('DONE')
print(fname, timestamp)
print('train:test', y_train.shape[0], y_test.shape[0])
for each in results:
    print(each)
# Print when done
print('Done at:' , '{:%Y-%m-%dT%H:%M}'.format(datetime.now(gettz("Europe/London"))))<jupyter_output><empty_output><jupyter_text>Results<jupyter_code>file = results_file
if do_end_test:
    data = pd.read_csv(file, header=None, names=['run','loss','val_acc','epoch','time', 'end_test_acc'])
    all_data = [data['val_acc'], data['end_test_acc']]
    all_names = ['Validation test set', 'Alternative test set']
else:
    data = pd.read_csv(file, header=None, names=['run','loss','val_acc','epoch','time'])
    all_data = [data['val_acc']]
    all_names = ['Validation test set']
    
# Box plot
sns.set(style="whitegrid")
ax = sns.boxplot(data=all_data)
ax = sns.swarmplot(data=all_data, color='black')
ax.set_xlabel('')
ax.set_ylabel('Accuracy')
plt.suptitle('K-fold cross validation results')
ax.yaxis.set_major_formatter(FuncFormatter('{0:.0%}'.format))
plt.xticks(np.arange(len(all_names)), all_names)

# Print results
print('Training dataset size:', x_train.shape[0])
print('Test set size:', x_test.shape[0])
if do_end_test:
    print('Alternative test set size:', x_other.shape[0])
accuracy = data['val_acc']
print(file, '\n')
print(data)
print('\nValidation accuracy mean average:', data['val_acc'].mean())
print('Validation accuracy sample standard deviation:', data['val_acc'].std())
if do_end_test:
    print('Alternative test results:')
    print('Accuracy mean average:', data['end_test_acc'].mean())
    print('Accuracy sample standard deviation:', data['end_test_acc'].std())<jupyter_output><empty_output><jupyter_text>Predictions<jupyter_code>def predictions(model, model_params, model_type, 
                x_input, y_input, name):
    ''' Use the model to make predictions on x_input data. Plot the confusion matrix and ROC. 
    Return accuracy, AUC and test set class balance. '''    
    y_input = y_input - y_input.min()
    x_input = (x_input - model_params['x_train_mean'])/(model_params['x_train_std'])
    x_input = reshape(x_input, model_type)
    # Class balance
    n0 = (y_input == 0).sum()
    n1 = (y_input == 1).sum()
    
    # Calculate model prediction
    threshold = model_params['threshold']
    y_probs = model.predict_on_batch(x_input)
    if threshold == 0.5:
        y_pred = np.round(y_probs).flatten()
    else:
        y_pred = y_probs.flatten()
        y_pred[y_pred > threshold] = 1
        y_pred[y_pred <= threshold] = 0
        
    cm = confusion_matrix(y_input, y_pred, labels=[1,0])
    acc_calc = (cm[0][0]+cm[1][1])/(cm.sum())
    cm_norm = cm.astype('float') / cm.sum(axis=1)[:, np.newaxis]
    print('Operating point:',threshold)   
    print('Accuracy:',acc_calc)
    class_balance = n0/(n0+n1)
    print('Class balance in test set:', class_balance)
    title = 'Normalised confusion matrix'
    plot_confusion_matrix(cm_norm, title=title, name=name)

    # ROC and AUC
    auc = plot_roc(y_input, y_probs, name=name)
    print('AUC:', auc)
    
    return acc_calc, auc, class_balance
''' Make predictions using the last model that was trained '''
operating_point = 0.5
model_params['threshold'] = operating_point
if do_end_test:
    print('Predictions on the alternative test set')
    x_in = x_other
    y_in = y_other
else:
    x_in = x_test
    y_in = y_test
acc_calc, auc, class_balance = predictions(model, model_params, model_type, x_in, y_in, fname)<jupyter_output><empty_output><jupyter_text>Save model<jupyter_code>modelfile = logs_dir+'/'+fname+'/model'
model_json = model.to_json()
with open(modelfile+'.json', 'w') as json_file:
    json_file.write(model_json)
# Save the model's weights
model.save_weights(modelfile+'.h5')
print('Model saved to', modelfile)
# Save the other model parameters (mean and std dev of training data)
model_params          
with open(logs_dir+'/'+fname+'/model_params.csv', 'a+') as f:
    for key in model_params.keys():
        f.write("%s,%s\n"%(key,model_params[key]))<jupyter_output><empty_output><jupyter_text>TestTest that the notebook runs as expected.<jupyter_code># Expected values for testing
assert(k == 3) 
assert(m == 1) # For speed of testing, set m interations to 1.
assert(model_type == 'MLP_tuned' )
assert(fname == 'GunPoint')

# Test
tol = 1e-6
expected = 0.5
assert(class_balance > expected-tol and class_balance < expected+tol), 'expected: '+str(expected)+' got: '+str(class_balance)

model_tol = 0.1
expected = 0.97
assert(auc > expected-model_tol and auc < expected+model_tol), 'expected: '+str(expected)+' got: '+str(auc)

expected = 0.89
assert(acc_calc > expected-model_tol and acc_calc < expected+model_tol), 'expected: '+str(expected)+' got: '+str(acc_calc)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1344077800544359, 'fraction_numerical': 0.01474051194226123, 'mean_word_length': 3.898876404494382, 'pattern_counts': {'":': 0, '<': 78, '<?xml version=': 0, '>': 80, 'https://': 6, 'lorem ipsum': 0, 'www.': 1, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '17', 'dir': 'jupyter-structured-clean-dedup', 'id': '317158', 'license': 'MIT', 'n_tokens_mistral': 12025, 'n_tokens_neox': 11572, 'n_words': 2414, 'path': 'notebooks/deepscent_dev.ipynb', 'repo_name': 'Withington/deepscent', 'size': '32715'}