<jupyter_start><jupyter_text>DevNetTrain and test the models used in developing Deepscent.MIT license to use [software by <NAME>](https://github.com/cauchyturing/UCR_Time_Series_Classification_Deep_Learning_Baseline/blob/master/README.md).<jupyter_code>import os
from pathlib import Path
import time
from datetime import datetime
from dateutil.tz import gettz
import itertools

import numpy as np
import pandas as pd
import tensorflow as tf
import tensorflow.keras as keras

from tensorflow.keras.models import Model, Sequential
from tensorflow.keras.layers import Input, Dense, Activation, Dropout
from tensorflow.keras import regularizers
from tensorflow.keras.initializers import RandomUniform
from tensorflow.keras import utils
from tensorflow.keras.callbacks import ReduceLROnPlateau, EarlyStopping

import matplotlib.pyplot as plt
from matplotlib.ticker import FuncFormatter
import seaborn as sns
from sklearn.model_selection import KFold, RepeatedStratifiedKFold
from sklearn import preprocessing
from sklearn.metrics import confusion_matrix, roc_curve, roc_auc_score, classification_report

np.random.seed(999123)<jupyter_output><empty_output><jupyter_text>User inputs<jupyter_code># Select a model with hyperparameters as per Wang et al. (2017): 
# MLP, FCN, ResNet
# or
# select a model with hyperparameters tuned to optimise performance on the ACI detection dogs dataset:
# MLP_tuned, FCN_tuned, ResNet_tuned, CNN
model_type = 'ResNet_tuned' # MLP, MLP_tuned, FCN, FCN_tuned, CNN, ResNet, ResNet_tuned

# Provide the dataset directory name. Select 'GunPoint' to use a dataset from the UCR TSC Archive.
flist = ['private_correct_plus'] # private_balanced, GunPoint, private_dog0_correct_plus, private_correct_plus

batch_size = 32 
k = 3 # k-fold cross validation: number of folds. If k=1, the original test-train split is used.
m = 4 # k-fold cross validation: number of repetitions (if k>1).
epochs_dict = {'MLP':5000, 'FCN':2000, 'ResNet':1500, 'MLP_tuned':500, 'FCN_tuned':1000, 'CNN':1000, 'ResNet_tuned':1000}
nb_epochs = epochs_dict[model_type]

do_end_test = True    # For each fold, evaluate model on the end_test set too.
truncate_data = False # Truncate pressure samples to first n data points
filter_data = False # Filter out noise below a threshold

data_augmentation = False
tensorboard = True # Set to True to write logs for use by TensorBoard
k_fold_seed = 765432

# Output directories
logs_dir = '../../logs'
tensorboard_dir = '../../logs/tensorboard'
timestamp = '{:%Y-%m-%dT%H:%M}'.format(datetime.now(gettz("Europe/London")))
logs_dir = logs_dir +'/' + timestamp
tensorboard_dir = tensorboard_dir +'/' + timestamp

# Input directory
if 'private' in flist[0]:
    fdir = '../../data/private_data/private_events_dev2' 
else:
    fdir = '../../data' 
    
if not 'correct_plus' in flist[0]:
    do_end_test = False<jupyter_output><empty_output><jupyter_text>Tools<jupyter_code>def plot_confusion_matrix(cm, title='Normalised confusion matrix', name=''):
    ''' Plot the normalised confusion matrix
    Parameters
    cm : array - normalised confusion matrix
    Scikit-learn: Machine Learning in Python, Pedregosa et al., JMLR 12, pp. 2825-2830, 2011.
    'Confusion Matrix' https://scikit-learn.org/stable/auto_examples/model_selection/plot_confusion_matrix.html#sphx-glr-auto-examples-model-selection-plot-confusion-matrix-py
    '''
    classes = ['Positive', 'Negative']
    cmap=plt.cm.Blues
    sns.set_style('dark')
    plt.figure()
    plt.imshow(cm, interpolation='nearest', cmap=cmap)
    plt.title(title)
    plt.colorbar(format=FuncFormatter('{0:.0%}'.format))
    tick_marks = np.arange(len(classes))
    plt.xticks(tick_marks, classes, rotation=45)
    plt.yticks(tick_marks, classes)
    plt.clim(0, 1)
    fmt = '.0%'
    thresh = cm.max() / 2.
    for i, j in itertools.product(range(cm.shape[0]), range(cm.shape[1])):
        plt.text(j, i, format(cm[i, j], fmt),
                 horizontalalignment="center",
                 color="white" if cm[i, j] > thresh else "black")
    plt.ylabel('True class')
    plt.xlabel('Predicted class')
    plt.tight_layout()
    file_name = 'cm_devnet_'+name+'.png'
    plt.savefig(file_name, bbox_inches='tight')
        
        
def plot_roc(y_true, y_probs, name): 
    ''' Plot ROC and return AUC
    Parameters
    y_true : vector of true class labels
    y_probs : vector of predicted probabilities
    Returns
    auc : float
    '''
    fpr, tpr, thresholds = roc_curve(y_true, y_probs)
    auc = roc_auc_score(y_true, y_probs)
    sns.set_style('whitegrid')
    plt.figure()
    plt.plot(fpr, tpr, color='darkorange',
             lw=2, label='ROC curve (area = %0.2f)' % auc)
    plt.plot([0, 1], [0, 1], color='navy', lw=2, linestyle='--')
    plt.xlim([0.0, 1.0])
    plt.ylim([0.0, 1.05])
    plt.xlabel('False Positive Rate')
    plt.ylabel('True Positive Rate')
    plt.title('Receiver operating characteristic curve')
    plt.legend(loc="lower right")
    file_name = 'roc_devnet_'+name+'.png'
    plt.savefig(file_name, bbox_inches='tight')
    return auc


def filter_out(x, threshold):
    ''' Filter out any data points in x that are below the threshold by setting them to zero.
    Return the modified data, x '''
    if x < threshold:
        return 0
    return x
    
    
def preprocess(X):
    ''' Apply preprocessing to the input data X'''
    if filter_data:
        threshold = 0.1
        X = np.piecewise(X, [X < threshold, X >= threshold], [lambda X: 0, lambda X: X])
    return X
    
    
def readucr(filename):
    ''' Load a dataset from a file in UCR format
    space delimited, class labels in the first column.
    Returns
    X : DNN input data
    Y : class labels
    '''
    data = np.loadtxt(Path(filename))
    Y = data[:,0]
    X = data[:,1:]
    if truncate_data:
        X = X[:,:300]
    X = preprocess(X)
    return X, Y
   

def reshape(x, model_type):
    ''' Reshape data into input format for the selected DNN '''
    if model_type == 'ResNet':
        return reshape_2d(x)
    elif model_type == 'FCN' or model_type == 'FCN_tuned' or model_type == 'CNN' or model_type == 'ResNet_tuned':
        return reshape_1d(x)
    elif model_type == 'MLP' or model_type == 'MLP_tuned':
        return x
    else:
        raise ValueError('Unrecognised model type')
    return x


def augment_data(x, y):
    ''' Return n times as many data samples, x, and labels, y. The augmented data is generated 
    by applying a shift to each row of x and appending these new rows to x '''
    m = x.shape[1]
    x_new = x
    y_new = y
    for shift in range(-50, 60, 10):
        x_aug = np.zeros_like(x)
        if shift < 0:
            x_aug[:,:m+shift] = x[:,-shift:]
        elif shift > 0:
            x_aug[:,shift:] = x[:,:m-shift]
        elif shift == 0:
            continue
        x_new = np.concatenate((x_new, x_aug), axis=0)
        y_new = np.concatenate((y_new, y), axis=0)
    return x_new, y_new<jupyter_output><empty_output><jupyter_text>Build DNNBuild a binary classifier. Model types: MLP, FCN, ResNet, CNN. ResNetResNet with hyperparameters as per Wang et al. (2017).<jupyter_code>def reshape_2d(x):
    ''' Reshape data into input format for ResNet '''
    x = x.reshape(x.shape + (1,1,))
    return x


def build_resnet(input_shape, nb_classes):
    ''' Build ResNet DNN and return input and output tensors '''
    # Parameters
    k = 1 # kernel multiplier
    n_feature_maps = 128
    
    print ('build conv_x')
    x = Input(shape=(input_shape))
    conv_x = keras.layers.BatchNormalization()(x)
    conv_x = keras.layers.Conv2D(n_feature_maps, 8*k, 1, padding='same')(conv_x)
    conv_x = keras.layers.BatchNormalization()(conv_x)
    conv_x = Activation('relu')(conv_x)
     
    print ('build conv_y')
    conv_y = keras.layers.Conv2D(n_feature_maps, 5*k, 1, padding='same')(conv_x)
    conv_y = keras.layers.BatchNormalization()(conv_y)
    conv_y = Activation('relu')(conv_y)
     
    print ('build conv_z')
    conv_z = keras.layers.Conv2D(n_feature_maps, 3*k, 1, padding='same')(conv_y)
    conv_z = keras.layers.BatchNormalization()(conv_z)
     
    is_expand_channels = not (input_shape[-1] == n_feature_maps)
    if is_expand_channels:
        shortcut_y = keras.layers.Conv2D(n_feature_maps, 1, 1,padding='same')(x)
        shortcut_y = keras.layers.BatchNormalization()(shortcut_y)
    else:
        shortcut_y = keras.layers.BatchNormalization()(x)
    print ('Merging skip connection')
    y = keras.layers.add([shortcut_y, conv_z])
    y = Activation('relu')(y)
     
    print ('build conv_x')
    x1 = y
    conv_x = keras.layers.Conv2D(n_feature_maps*2, 8*k, 1, padding='same')(x1)
    conv_x = keras.layers.BatchNormalization()(conv_x)
    conv_x = Activation('relu')(conv_x)
         
    print ('build conv_y')
    conv_y = keras.layers.Conv2D(n_feature_maps*2, 5*k, 1, padding='same')(conv_x)
    conv_y = keras.layers.BatchNormalization()(conv_y)
    conv_y = Activation('relu')(conv_y)
     
    print ('build conv_z')
    conv_z = keras.layers.Conv2D(n_feature_maps*2, 3*k, 1, padding='same')(conv_y)
    conv_z = keras.layers.BatchNormalization()(conv_z)
     
    is_expand_channels = not (input_shape[-1] == n_feature_maps*2)
    if is_expand_channels:
        shortcut_y = keras.layers.Conv2D(n_feature_maps*2, 1, 1,padding='same')(x1)
        shortcut_y = keras.layers.BatchNormalization()(shortcut_y)
    else:
        shortcut_y = keras.layers.BatchNormalization()(x1)
    print ('Merging skip connection')
    y = keras.layers.add([shortcut_y, conv_z])
    y = Activation('relu')(y)
     
    print ('build conv_x')
    x1 = y
    conv_x = keras.layers.Conv2D(n_feature_maps*2, 8*k, 1, padding='same')(x1)
    conv_x = keras.layers.BatchNormalization()(conv_x)
    conv_x = Activation('relu')(conv_x)
     
    print ('build conv_y')
    conv_y = keras.layers.Conv2D(n_feature_maps*2, 5*k, 1, padding='same')(conv_x)
    conv_y = keras.layers.BatchNormalization()(conv_y)
    conv_y = Activation('relu')(conv_y)
     
    print ('build conv_z')
    conv_z = keras.layers.Conv2D(n_feature_maps*2, 3*k, 1, padding='same')(conv_y)
    conv_z = keras.layers.BatchNormalization()(conv_z)

    is_expand_channels = not (input_shape[-1] == n_feature_maps*2)
    if is_expand_channels:
        shortcut_y = keras.layers.Conv2D(n_feature_maps*2, 1, 1,padding='same')(x1)
        shortcut_y = keras.layers.BatchNormalization()(shortcut_y)
    else:
        shortcut_y = keras.layers.BatchNormalization()(x1)
    print ('Merging skip connection')
    y = keras.layers.add([shortcut_y, conv_z])
    y = Activation('relu')(y)
     
    full = keras.layers.GlobalAveragePooling2D()(y)   
    out = Dense(1, activation='sigmoid')(full)
    print ('        -- model was built.')
    return x, out<jupyter_output><empty_output><jupyter_text>ResNet tunedResNet with hyperparameters tuned to optimise performance on the ACI dataset.<jupyter_code>def build_resnet_tuned(input_shape, num_features0, num_features1, filter_size, pooling_size, dropout):
    ''' Return ResNet model '''
    nb_classes = 2
    print(input_shape, num_features0, num_features1, filter_size, pooling_size, dropout)
    
    # Preparation block
    x = Input(shape=(input_shape))
    conv = keras.layers.Conv1D(num_features0, filter_size, padding='same')(x)
    conv = keras.layers.BatchNormalization()(conv)
    conv = Activation('relu')(conv)
    conv = keras.layers.MaxPooling1D(pooling_size)(conv)
    
    # First block
    skip = conv
    conv = keras.layers.Conv1D(num_features0, filter_size, padding='same')(conv)
    conv = keras.layers.BatchNormalization()(conv)
    conv = Activation('relu')(conv)
    
    conv = keras.layers.Conv1D(num_features0, filter_size, padding='same')(conv)
    conv = keras.layers.BatchNormalization()(conv)
    conv = Activation('relu')(conv)
    
    conv = keras.layers.Conv1D(num_features1, filter_size, padding='same')(conv)
    conv = keras.layers.BatchNormalization()(conv)
    conv = Activation('relu')(conv)
    
    conv = keras.layers.Conv1D(num_features1, filter_size, padding='same')(conv)
    conv = keras.layers.BatchNormalization()(conv)
    shortcut = keras.layers.Conv1D(num_features1, 1, padding='same')(skip)
    shortcut = keras.layers.BatchNormalization()(shortcut)
    conv = keras.layers.add([conv, shortcut])
    conv = Activation('relu')(conv)
    
    # Second block
    skip = conv
    conv = keras.layers.Conv1D(num_features0, filter_size, padding='same')(conv)
    conv = keras.layers.BatchNormalization()(conv)
    conv = Activation('relu')(conv)
    
    conv = keras.layers.Conv1D(num_features0, filter_size, padding='same')(conv)
    conv = keras.layers.BatchNormalization()(conv)
    conv = Activation('relu')(conv)
    
    conv = keras.layers.Conv1D(num_features1, filter_size, padding='same')(conv)
    conv = keras.layers.BatchNormalization()(conv)
    conv = Activation('relu')(conv)
    
    conv = keras.layers.Conv1D(num_features1, filter_size, padding='same')(conv)
    conv = keras.layers.BatchNormalization()(conv)
    shortcut = keras.layers.Conv1D(num_features1, 1, padding='same')(skip)
    shortcut = keras.layers.BatchNormalization()(shortcut)
    conv = keras.layers.add([conv, shortcut])
    conv = Activation('relu')(conv)
    
    # Third block
    skip = conv
    conv = keras.layers.Conv1D(num_features0*2, filter_size, padding='same')(conv)
    conv = keras.layers.BatchNormalization()(conv)
    conv = Activation('relu')(conv)
    
    conv = keras.layers.Conv1D(num_features0*2, filter_size, padding='same')(conv)
    conv = keras.layers.BatchNormalization()(conv)
    conv = Activation('relu')(conv)
    
    conv = keras.layers.Conv1D(num_features1*2, filter_size, padding='same')(conv)
    conv = keras.layers.BatchNormalization()(conv)
    conv = Activation('relu')(conv)
    
    conv = keras.layers.Conv1D(num_features1*2, filter_size, padding='same')(conv)
    conv = keras.layers.BatchNormalization()(conv)
    shortcut = keras.layers.Conv1D(num_features1*2, 1, padding='same')(skip)
    shortcut = keras.layers.BatchNormalization()(shortcut)
    conv = keras.layers.add([conv, shortcut])
    conv = Activation('relu')(conv)
    
    # Output block
    full = keras.layers.GlobalAveragePooling1D()(conv)
    y = Dropout(dropout, name='Dropout')(full)
    out = Dense(1, activation='sigmoid')(full)
    return x, out<jupyter_output><empty_output><jupyter_text>FCNWith hyperparameters as per Wang et al. (2017).<jupyter_code>def reshape_1d(x):
    ''' Reshape data into input format for FCN or CNN'''
    x = x.reshape(x.shape + (1,))
    return x
    
    
def build_fcn(input_shape, nb_classes):
    ''' Build Fully Convolutional Network (FCN) and return input and output tensors '''
    # Parameters
    k = 1 # kernel multiplier
    n_feature_maps = 128
    
    print ('build conv_x')
    x = Input(shape=(input_shape))
    conv_x = x
    #conv_x = keras.layers.BatchNormalization()(conv_x)
    conv_x = keras.layers.Conv1D(n_feature_maps, 8*k, 1, padding='same')(conv_x)
    conv_x = keras.layers.BatchNormalization()(conv_x)
    conv_x = Activation('relu')(conv_x)
     
    print ('build conv_y')
    conv_y = keras.layers.Conv1D(n_feature_maps*2, 5*k, 1, padding='same')(conv_x)
    conv_y = keras.layers.BatchNormalization()(conv_y)
    conv_y = Activation('relu')(conv_y)
     
    print ('build conv_z')
    conv_z = keras.layers.Conv1D(n_feature_maps, 3*k, 1, padding='same')(conv_y)
    conv_z = keras.layers.BatchNormalization()(conv_z)
    conv_z = Activation('relu')(conv_z)
    
    #print ('build conv_za')
    #conv_z = keras.layers.Conv1D(n_feature_maps, 3*k, 1, padding='same')(conv_z)
    #conv_z = keras.layers.BatchNormalization()(conv_z)
    #conv_z = Activation('relu')(conv_z)
    
    #print ('build conv_zb')
    #conv_z = keras.layers.Conv1D(n_feature_maps, 3*k, 1, padding='same')(conv_z)
    #conv_z = keras.layers.BatchNormalization()(conv_z)
    #conv_z = Activation('relu')(conv_z)
     
    full = keras.layers.GlobalAveragePooling1D()(conv_z)
    #full = Dense(128, activation='relu')(full)
    out = Dense(1, activation='sigmoid')(full)
    return x, out<jupyter_output><empty_output><jupyter_text>FCN tunedWith hyperparameters tuned to optimise performance on the ACI dataset.<jupyter_code>def build_fcn_tuned(input_shape, nb_classes):
    ''' Build Fully Convolutional Network (FCN) and return input and output tensors '''
    # Parameters
    feature_maps_a = 32
    feature_maps_b = 64
    feature_maps_c = 32
    filter_a = 4
    filter_b = 4
    filter_c = 4
    
    print ('build conv_x')
    x = Input(shape=(input_shape))
    conv_x = x
    conv_x = keras.layers.Conv1D(feature_maps_a, filter_a, 1, padding='same')(conv_x)
    conv_x = keras.layers.BatchNormalization()(conv_x)
    conv_x = Activation('relu')(conv_x)
     
    print ('build conv_y')
    conv_y = keras.layers.Conv1D(feature_maps_b, filter_b, 1, padding='same')(conv_x)
    conv_y = keras.layers.BatchNormalization()(conv_y)
    conv_y = Activation('relu')(conv_y)
     
    print ('build conv_z')
    conv_z = keras.layers.Conv1D(feature_maps_c, filter_c, 1, padding='same')(conv_y)
    conv_z = keras.layers.BatchNormalization()(conv_z)
    conv_z = Activation('relu')(conv_z)
     
    full = keras.layers.GlobalAveragePooling1D()(conv_z)
    out = Dense(1, activation='sigmoid')(full)
    return x, out<jupyter_output><empty_output><jupyter_text>CNNWith hyperparameters tuned to optimise performance on the ACI dataset.Using the CNN architecture of[<NAME>, 2018, Introduction to 1D Convolutional Neural Networks in Keras for Time Sequences](https://blog.goodaudience.com/introduction-to-1d-convolutional-neural-networks-in-keras-for-time-sequences-3a7ff801a2cf).<jupyter_code>def build_cnn_harus(input_shape, nb_classes):
    ''' Build a CNN and return input and output tensors '''
    # Parameters
    n_features_a = 128 # Ackermann 100 
    n_features_b = 128 # Ackermann 160
    filter_size = 16   # Ackermann 10
    pooling_size = 16   # Ackermann 3
    dropout = 0.7      # Ackermann 0.5
    has_dense_layer = True
    
    print ('build CNN HARUS')
    x = Input(shape=(input_shape))
    conv_x = x
    conv_x = keras.layers.Conv1D(n_features_a, filter_size, activation='relu')(conv_x)
    conv_x = keras.layers.Conv1D(n_features_a, filter_size, activation='relu')(conv_x)
    conv_x = keras.layers.MaxPooling1D(pooling_size)(conv_x)
    conv_x = keras.layers.Conv1D(n_features_b, filter_size, activation='relu')(conv_x)
    
    if True:
        conv_x = keras.layers.Conv1D(n_features_b, filter_size, activation='relu')(conv_x)
    else:
        conv_x = keras.layers.Conv1D(n_features_b, filter_size)(conv_x)
        conv_x = keras.layers.BatchNormalization()(conv_x)
        conv_x = Activation('relu')(conv_x)
        
    if has_dense_layer: # End with a fully connected layer
        full = keras.layers.GlobalAveragePooling1D()(conv_x)
        full = Dropout(dropout,name='Dropout')(full)
        out = Dense(1, activation='sigmoid')(full)
    else:
        conv_x = keras.layers.Conv1D(16, filter_size, activation='relu')(conv_x)
        conv_x = Dropout(dropout,name='Dropout')(conv_x)
        conv_x = keras.layers.Conv1D(1, filter_size, activation='relu')(conv_x)
        conv_x = keras.layers.GlobalAveragePooling1D()(conv_x)
        out = Activation(activation='sigmoid')(conv_x)
    return x, out<jupyter_output><empty_output><jupyter_text>MLPWith hyperparameters as per Wang et al. (2017).<jupyter_code>def build_mlp(input_shape, nb_classes):
    num = 500
    x = Input(shape=(input_shape))
    y = Dropout(0.1,name='WDrop010')(x)
    y = Dense(num, activation='relu', name='WDense010')(y)
    y = Dropout(0.2,name='WDrop020')(y)
    y = Dense(num, activation='relu', name='WDense020')(y)
    y = Dropout(0.2,name='WDrop021')(y)
    y = Dense(num, activation='relu', name='WDense021')(y)
    y = Dropout(0.3,name='WDrop031')(y)
    out = Dense(1, activation='sigmoid', name='WDense080')(y)
    return x, out<jupyter_output><empty_output><jupyter_text>MLP tunedWith hyperparameters tuned to optimise performance on the ACI dataset.<jupyter_code>def build_mlp_tuned(input_shape, nb_classes):
    ''' Build a Multilayer Perceptron (MLP) and return input and output tensors '''
    drop = 0.2
    num = 16
    l2 = 0.1
    x = Input(shape=(input_shape))
    y = Dropout(drop,name='DropInput')(x)
    y = Dense(num, kernel_regularizer=regularizers.l2(l2), activation='relu', name='Dense010')(y)
    y = Dropout(drop,name='Drop010')(y)
    y = Dense(num, kernel_regularizer=regularizers.l2(l2), activation='relu', name='Dense020')(y)
    y = Dropout(drop,name='Drop020')(y)
    y = Dense(num, kernel_regularizer=regularizers.l2(l2), activation='relu', name='Dense030')(y)
    y = Dropout(drop,name='Drop030')(y)
    out = Dense(1, activation='sigmoid', name='DenseOutput')(y)
    return x, out<jupyter_output><empty_output><jupyter_text>Function: train model<jupyter_code>def train_model(fname, x_train, y_train, x_test, y_test, label="0"):
    ''' Build and train a DNN. Return summary info and a trained model '''
    print('Running dataset', fname)
    nb_classes = len(np.unique(y_test))
    if nb_classes != 2:
        raise 'Number of classes must be 2 to use this binary classifier'
    
    if data_augmentation:
        x_train, y_train = augment_data(x_train, y_train)
     
    Y_train = (y_train - y_train.min())/(y_train.max()-y_train.min())*(nb_classes-1)
    Y_test = (y_test - y_test.min())/(y_test.max()-y_test.min())*(nb_classes-1)
     
    x_train_mean = x_train.mean()
    x_train_std = x_train.std()
    x_train = (x_train - x_train_mean)/(x_train_std) 
    x_test = (x_test - x_train_mean)/(x_train_std)
     
    x_train = reshape(x_train, model_type)
    x_test = reshape(x_test, model_type)
    if model_type == 'MLP':
        x, y = build_mlp(x_train.shape[1:], nb_classes)
    elif model_type == 'MLP_tuned':
        x, y = build_mlp_tuned(x_train.shape[1:], nb_classes)
    elif model_type == 'ResNet':
            x, y = build_resnet(x_train.shape[1:], nb_classes)
    elif model_type == 'ResNet_tuned':
        num_features0 = 64
        num_features1 = 128
        filter_size = 4
        pooling_size = 8
        dropout = 0.5
        x, y = build_resnet_tuned(x_train.shape[1:], num_features0, num_features1, filter_size, pooling_size, dropout)
    elif model_type == 'FCN':
        x, y = build_fcn(x_train.shape[1:], nb_classes)
    elif model_type == 'FCN_tuned':
        x, y = build_fcn_tuned(x_train.shape[1:], nb_classes)
    elif model_type == 'CNN':
        x, y = build_cnn_harus(x_train.shape[1:], nb_classes)
    model = Model(x, y)
    print(model.summary())
    
    optimizer = keras.optimizers.Adam()
    model.compile(loss='binary_crossentropy',
                  optimizer=optimizer,
                  metrics=['acc'])
    
    Path(logs_dir+'/'+fname).mkdir(parents=True, exist_ok=True) 
    reduce_lr = ReduceLROnPlateau(monitor='loss', factor=0.5,
                      patience=50, min_lr=0.0001) 
    callbacks = [reduce_lr]
    if tensorboard:
        tb_dir = tensorboard_dir+'/'+fname+'_'+label
        Path(tb_dir).mkdir(parents=True, exist_ok=True) 
        print('Tensorboard logs in', tb_dir)
        callbacks.append(keras.callbacks.TensorBoard(log_dir=tb_dir, histogram_freq=0))
  
    start = time.time()
    hist = model.fit(x_train, Y_train, batch_size=batch_size, epochs=nb_epochs,
              verbose=1, validation_data=(x_test, Y_test), callbacks=callbacks)
    end = time.time()
    log = pd.DataFrame(hist.history) 
    
    # Print results
    duration_seconds = round(end-start)
    duration_minutes = str(round((end-start)/60))
    print('Training complete on', fname, 'Duration:', duration_seconds, 'secs; about', duration_minutes, 'minutes.')
    
    # Print and save results. Print the testing results which has the lowest training loss.
    print('Selected the test result with the lowest training loss. Loss and validation accuracy are -')
    idx = log['loss'].idxmin()
    loss = log.loc[idx]['loss']
    val_acc = log.loc[idx]['val_acc']
    epoch = idx + 1
    print(loss, val_acc, 'at index', str(idx), ' (epoch ', str(epoch), ')')
    summary = '|' + label + '  |'+str(loss)+'  |'+str(val_acc)+' |'+str(epoch)+' |'+ duration_minutes + 'mins  |'
    summary_csv = label+','+str(loss)+','+str(val_acc)+','+str(epoch)+','+ duration_minutes 
    
    # Save summary file and log file.
    print('Tensorboard logs in', tb_dir) 
    print('Saving logs to',logs_dir+'/'+fname+'/history_'+label+'.csv')
    log.to_csv(logs_dir+'/'+fname+'/history_'+label+'.csv')
    
    model_params = {'x_train_mean':x_train_mean, 'x_train_std':x_train_std}
    return summary, summary_csv, model, model_params<jupyter_output><empty_output><jupyter_text>Train DNN<jupyter_code>''' Train a model, using repeated k-fold cross validation, if selected '''

results = []
for each in flist:
    fname = each
    x_train, y_train = readucr(fdir+'/'+fname+'/'+fname+'_TRAIN.txt')
    x_test, y_test = readucr(fdir+'/'+fname+'/'+fname+'_TEST.txt')
    if do_end_test:
        x_other, y_other = readucr(fdir+'/'+fname+'/'+fname+'_END_TEST.txt')
    # k-fold cross validation setup
    if k > 1:
        x_all = np.concatenate((x_train, x_test), axis=0)
        y_all = np.concatenate((y_train, y_test), axis=0)
        kfold = RepeatedStratifiedKFold(n_splits=k, n_repeats=m, random_state=k_fold_seed)
        count = 0
        for train, test in kfold.split(x_all, y_all):
            x_train, y_train, x_test, y_test = x_all[train], y_all[train], x_all[test], y_all[test]
            summary, summary_csv, model, model_params = train_model(fname, x_train, y_train, x_test, y_test, str(count))
            if do_end_test:
                x_in = (x_other - model_params['x_train_mean'])/(model_params['x_train_std'])
                x_in = reshape(x_in, model_type)
                _, end_test_acc = model.evaluate(x_in, y_other, batch_size=batch_size)
                summary = summary + str(end_test_acc) +' |'
                summary_csv = summary_csv + ',' + str(end_test_acc)
            with open(logs_dir+'/'+fname+'/devnet_summary.csv', 'a+') as f:
                f.write(summary_csv)
                f.write('\n')
                print('Added summary row to ', logs_dir+'/'+fname+'/devnet_summary.csv')
            results.append(summary)
            count = count + 1
    else:
        summary, summary_csv, model, model_params = train_model(fname, x_train, y_train, x_test, y_test)
        if do_end_test:
            x_in = (x_other - model_params['x_train_mean'])/(model_params['x_train_std'])
            x_in = reshape(x_in, model_type)
            _, end_test_acc = model.evaluate(x_in, y_other, batch_size=batch_size)
            summary = summary + str(end_test_acc) +' |'
            summary_csv = summary_csv + ',' + str(end_test_acc)
        with open(logs_dir+'/'+fname+'/devnet_summary.csv', 'a+') as f:
            f.write(summary_csv)
            f.write('\n')
            print('Added summary row to ', logs_dir+'/'+fname+'/devnet_summary.csv')
        results.append(summary)
        
print('DONE')
print(fname, timestamp)
print('train:test', y_train.shape[0], y_test.shape[0])
for each in results:
    print(each)
# Print when done
print('Done at:' , '{:%Y-%m-%dT%H:%M}'.format(datetime.now(gettz("Europe/London"))))<jupyter_output><empty_output><jupyter_text>Quantiles<jupyter_code>file =  logs_dir+'/'+fname+'/devnet_summary.csv'

print(do_end_test)
if do_end_test:
    data = pd.read_csv(file, header=None, names=['run','loss','val_acc','epoch','time', 'end_test_acc'])
else:
    data = pd.read_csv(file, header=None, names=['run','loss','val_acc','epoch','time'])
    
accuracy = data['val_acc']
print(file)
print('Accuracy mean, sample std dev and 95% confidence level is', accuracy.mean(), accuracy.std(), accuracy.std()*2.262)
print('95% quantile interval is', accuracy.quantile(0.0025), 'to', accuracy.quantile(0.975))
data.boxplot(column=['val_acc'], whis=[2.5,97.5])

if do_end_test:
    result_set = {'acc_mean':accuracy.mean(), 'std':accuracy.std(), 
                  'lower':accuracy.quantile(0.0025), 'upper':accuracy.quantile(0.975),
                 'end_test_acc_av':data['end_test_acc'].mean(),
                 'end_test_acc_std':data['end_test_acc'].std()}
else:
    result_set = {'acc_mean':accuracy.mean(), 'std':accuracy.std(), 
              'lower':accuracy.quantile(0.0025), 'upper':accuracy.quantile(0.975)}
    
print(data)<jupyter_output><empty_output><jupyter_text>Metrics<jupyter_code>''' Use trained model (after all epochs) to make predictions '''

def predictions(model, model_params, model_type, 
                x_input, y_input, name, threshold=0.5):
    ''' Use the model to make predictions on x_input data. Return the predictions and the calculated accuracy. '''    
    do_print = True
    y_input = y_input - y_input.min()
    x_input = (x_input - model_params['x_train_mean'])/(model_params['x_train_std'])
    x_input = reshape(x_input, model_type)
    nb_classes = len(np.unique(y_input))
    y_input = (y_input - y_input.min())/(y_input.max()-y_input.min())*(nb_classes-1)
    # Class balance
    n0 = (y_input == 0).sum()
    n1 = (y_input == 1).sum()
    
    # Calculate model prediction
    y_probs = model.predict_on_batch(x_input)
    if threshold == 0.5:
        y_pred = np.round(y_probs).flatten()
    else:
        y_pred = y_probs.flatten()
        y_pred[y_pred > threshold] = 1
        y_pred[y_pred <= threshold] = 0
        
    cm = confusion_matrix(y_input, y_pred, labels=[1,0])
    acc_calc = (cm[0][0]+cm[1][1])/(cm.sum())
    cm_norm = cm.astype('float') / cm.sum(axis=1)[:, np.newaxis]
    if do_print:
        print('Predicted class probabilities:\n', y_probs[:5,:])
        print('Pred', y_pred[:20])
        print('True', y_input[:20].astype(int))
        print(cm)
        print('Calculated accuracy:',acc_calc)
        print('Class balance in test set:', n0, 'to', n1, 'i.e.', n0/(n0+n1))
        print('Normalised confusion matrix:\n', cm_norm)
    title = 'Normalised confusion matrix'
    plot_confusion_matrix(cm_norm, title=title, name=name)

    # ROC and AUC
    auc = plot_roc(y_input, y_probs, name=name)
    print('AUC:', auc)

    report = classification_report(y_input, y_pred)
    print('\n', report)
    print('\nmicro av - averaging the total true positives, false negatives and false positives')
    print('macro av - averaging the unweighted mean per label')
    print('weighted av - averaging the support-weighted mean per label')
    return y_pred, acc_calc

y_pred, acc = predictions(model, model_params, model_type, x_test, y_test, fname)
result_set['this_model_acc'] = acc

# Check results using model.evaluate
x_in = (x_test - model_params['x_train_mean'])/(model_params['x_train_std'])
x_in = reshape(x_in, model_type)
print('model.evaluate : val_loss, val_acc', model.evaluate(x_in, y_test, batch_size=batch_size))<jupyter_output><empty_output><jupyter_text>Change operating point<jupyter_code>name = fname+'oppoint'
threshold = 0.3
y_pred, acc = predictions(model, model_params, model_type, x_test, y_test, name, threshold)

# Check results using model.evaluate
x_in = (x_test - model_params['x_train_mean'])/(model_params['x_train_std'])
x_in = reshape(x_in, model_type)
print('model.evaluate : val_loss, val_acc', model.evaluate(x_in, y_test, batch_size=batch_size))<jupyter_output><empty_output><jupyter_text>Plot data samples<jupyter_code>def plot_samples(x, y_true, y_pred, title, meta=None):
    ''' Plot the data samples, grouped as TP, FN, TN, FP as determined by the 
    samples true class (y_true) and the predicted class (y_pred) '''
    nb_classes = len(np.unique(y_true))
    y_true = (y_true - y_true.min())/(y_true.max()-y_true.min())*(nb_classes-1)
    if meta is not None:
        print(title)
    n_plots = 10
    fig, ax = plt.subplots(n_plots, 4, sharex='col', sharey='row', figsize=(10, 10))
    rows = [0, 0, 0, 0]
    green_red = sns.color_palette("Paired")
    colors = [green_red[3], green_red[5], green_red[2], green_red[4]]
    for i in range(len(y_pred)):
        if y_true[i]==1:
            if y_pred[i]==1:
                col = 0
            else:
                col = 1
                if meta is not None:
                    print('FN at ', meta.iloc[i]['filename'], 'sensor', meta.iloc[i]['sensor_number'])
        if y_true[i]==0:
            if y_pred[i]==0:
                col = 2
            else:
                col = 3
                if meta is not None:
                    print('FP at ', meta.iloc[i]['filename'], 'sensor', meta.iloc[i]['sensor_number'])
        row = rows[col]
        rows[col] = rows[col] + 1
        if row < n_plots:
            ax[row, col].plot(x[i], color=colors[col])
            ax[0, col].set_title('True '+str(int(y_true[i]))+': Pred '+str(y_pred[i]))
            ax[row, col].set_ylim(bottom=0, top=2.2)
    ax[n_plots-1, 0].set_ylabel('x(t)')
    ax[n_plots-1, 0].set_xlabel('time, t')
    ax[n_plots-1, 1].set_xlabel('time, t')
    fig.suptitle(title)
    plt.savefig('data_samples_'+title+'.png', bbox_inches='tight')
     
plot_samples(x_test, y_test, y_pred, fname)<jupyter_output><empty_output><jupyter_text>Compare runs<jupyter_code>file1 = '../../logs/2019-03-17T12:59/private_dog0_correct/devnet_summary.csv'
data1 = pd.read_csv(file1, header=None, names=['run','loss','val_acc','epoch','time'])
name1 = 'dog0_correct'

file = logs_dir+'/'+fname+'/devnet_summary.csv'
print('Showing results from:\n', file1, 'and\n', file)
data2 = pd.read_csv(file, header=None, names=['run','loss','val_acc','epoch','time', 'end_test_acc'])
name2 = 'this_run'
print(data2)


all_data = [data1['val_acc'], data2['val_acc']]
sns.set(style="whitegrid")
ax = sns.boxplot(data=all_data)
ax = sns.swarmplot(data=all_data, color='black')
ax.set_xlabel('DevNet')
ax.set_ylabel('validation accuracy')
ax.yaxis.set_major_formatter(FuncFormatter('{0:.0%}'.format))
plt.xticks([0, 1], [name1, name2])
plt.tight_layout()
plt.savefig('boxplot_devnet.png', bbox_inches='tight')<jupyter_output><empty_output><jupyter_text>Make predictions on other datasets<jupyter_code>if do_end_test:
    other = fname+'_END_TEST' #_dog_incorrect' # 'private_dog0_correct_plus_END_TEST'
else:
    other = fname+'_TEST'
datadir = fdir+'/'+fname
print('Testing on:', datadir+'/'+other+'.txt')
x_other, y_other = readucr(datadir+'/'+other+'.txt')
y_other_pred, other_acc = predictions(
    model, model_params, model_type, 
    x_other, y_other, other)
# Get dog result
meta = pd.read_csv(datadir+'/'+other+'_meta.txt', sep=',', parse_dates=['date'])
cm = confusion_matrix(y_other, meta['dog_pred'], labels=[1,0])
print('Dog cm \n', cm)
dog_acc = (cm[0][0]+cm[1][1])/(cm.sum())
cm_norm = cm.astype('float') / cm.sum(axis=1)[:, np.newaxis]
plot_confusion_matrix(cm_norm, title='Dog indications', name='dog_pred')
print('True', y_other[:20])
print('Dog ', meta['dog_pred'].values[:20])
print('Dog accuracy', dog_acc)

# Plot data
plot_samples(x_other, y_other, y_other_pred, 'DNN predictions', meta)
plot_samples(x_other, y_other, meta['dog_pred'], 'Dog indications', meta)
    
result_set['this_end_test_acc'] = other_acc<jupyter_output><empty_output><jupyter_text>Change the operating point<jupyter_code>threshold = 0.3
name = fname+'_END_TEST_threshold_'+str(threshold)
y_roc_pred, other_acc = predictions(
    model, model_params, model_type, 
    x_other, y_other, name, threshold=0.3)<jupyter_output><empty_output><jupyter_text>Plot the difference in dog and DNN predictions<jupyter_code>def plot_differences(x, y_pred, meta):
    ''' Plot the data samples, x, where the DNN prediction (y_pred) differs from
    the dogs indication (in meta) '''
    # Concatenate all data
    y_diff = abs(meta['dog_pred'].values-y_pred.T)
    y_diff_df = pd.DataFrame(y_diff, columns=['y_diff'])
    y_pred_df = pd.DataFrame(y_pred, columns=['y_pred'])
    x_df = pd.DataFrame(x)
    data_meta = pd.concat([y_pred_df, y_diff_df, meta], axis=1)
    meta_header = list(data_meta)
    data_meta = pd.concat([data_meta, x_df], axis=1)
    
    # Sort the data
    data_meta = data_meta.sort_values(['class', 'y_diff', 'dog_result'])
    class0 = data_meta[data_meta['class']==0]
    class0 = class0.sort_values(['y_diff', 'dog_result'], ascending=[False, False])
    class1 = data_meta[data_meta['class']==1]
    class1 = class1.sort_values(['y_diff', 'dog_result'], ascending=[False, True])

    # Plot the data where dog and DNN did not agree
    for this_class in [class1, class0]:
        # Get x data
        this_x = this_class[this_class.columns.difference(meta_header)]
        assert this_x.shape[1] == 1000
        # Count plots required
        n = 0
        for i in range(len(this_class)):
            if this_class.iloc[i]['y_pred'] != this_class.iloc[i]['dog_pred']:
                n = n + 1
        # Create the plots
        fig, ax = plt.subplots(n, 3, sharex='col', sharey='row', figsize=(10, 4.8), squeeze=False)
        class_label = this_class.iloc[0]['class']
        row = 0
        for i in range(len(this_class)):
            if row < n:
                dog_correct = this_class.iloc[i]['dog_pred'] == this_class.iloc[i]['class']
                dnn_correct = this_class.iloc[i]['y_pred'] == this_class.iloc[i]['class']
                dog_color = 'green' if dog_correct else 'red'
                dnn_color = 'green' if dnn_correct else 'red'  
                if this_class.iloc[i]['y_pred'] != this_class.iloc[i]['dog_pred']:
                    ax[row, 0].plot(this_x.iloc[i], color=dog_color)
                    ax[row, 1].plot(this_x.iloc[i], color=dnn_color)
                    ax[row, 0].set_ylim(bottom=0, top=2.2)
                    ax[row, 1].set_ylim(bottom=0, top=2.2)
                    file = this_class.iloc[i]['filename']
                    sensor = str(this_class.iloc[i]['sensor_number'])
                    ax[row, 2].text(0, 0.65, str(row+1)+') '+file+' sensor '+sensor)
                    row = row + 1
        ax[0, 0].set_title('Dog')
        ax[0, 1].set_title('DNN')
        ax[n-1, 0].set_ylabel('x(t)')
        ax[n-1, 0].set_xlabel('time, t')
        ax[n-1, 1].set_xlabel('time, t')
        ax[n-1, 2].set_xticklabels([])
        fig.suptitle('True class: '+str(class_label)+'    green=correct, red=incorrect')
        plt.savefig('DogDNN_diffs_class' + str(class_label) + '_'+fname+'.png', bbox_inches='tight')


# Operating point not set
plot_differences(x_other, y_other_pred, meta)
# With operating point threshold set
plot_differences(x_other, y_roc_pred, meta)
def plot_dnn_valueadd(x, y_pred, meta):
    ''' Plot the data samples, x, where the DNN prediction (y_pred) is 
    correct and the dogs indication was incorrect '''
    # Concatenate all data
    y_diff = abs(meta['dog_pred'].values-y_pred.T)
    y_diff_df = pd.DataFrame(y_diff, columns=['y_diff'])
    y_pred_df = pd.DataFrame(y_pred, columns=['y_pred'])
    x_df = pd.DataFrame(x)
    data_meta = pd.concat([y_pred_df, y_diff_df, meta], axis=1)
    meta_header = list(data_meta)
    data_meta = pd.concat([data_meta, x_df], axis=1)
    
    # Sort the data
    data_meta = data_meta.sort_values(['class', 'y_diff', 'dog_result'])
    class0 = data_meta[data_meta['class']==0]
    class0 = class0.sort_values(['y_diff', 'dog_result'], ascending=[False, False])
    class1 = data_meta[data_meta['class']==1]
    class1 = class1.sort_values(['y_diff', 'dog_result'], ascending=[False, True])

    # Plot the data where dog and DNN did not agree
    for this_class in [class1, class0]:
        # Get x data
        this_x = this_class[this_class.columns.difference(meta_header)]
        assert this_x.shape[1] == 1000
        # Count plots required
        n = this_class[(this_class['y_pred'] == this_class['class']) & (this_class['y_pred'] != this_class['dog_pred'])].shape[0]
        # Create the plots
        fig, ax = plt.subplots(n, 1, sharex='col', sharey='row')
        class_label = this_class.iloc[0]['class']
        row = 0
        for i in range(len(this_class)):
            if row < n:
                this_row = ax[row] if n>1 else ax
                true_class = this_class.iloc[i]['class']
                dog_class = int(this_class.iloc[i]['dog_pred'])
                dnn_class = int(this_class.iloc[i]['y_pred'])
                dog_correct = dog_class == true_class
                dnn_correct = dnn_class == true_class
                dog_color = 'green' if dog_correct else 'red'
                dnn_color = 'green' if dnn_correct else 'red'  
                if dnn_correct and not dog_correct:
                    this_row.plot(this_x.iloc[i], color=dnn_color)
                    this_row.set_ylim(bottom=0, top=2.2)
                    row = row + 1
                    this_row.set_facecolor('lightcyan' if true_class else 'lightyellow')
                    #print(meta_header)
                    print('True', true_class, ': DNN', dnn_class, ': dog', dog_class, this_class.iloc[i][['filename', 'sensor_number']])
        this_row = ax[0] if n>1 else ax
        print('n is', n)
        this_row.set_title('True '+str(true_class)+' : DNN '+str(dnn_class)+' : dog '+str(dog_class))
        this_row = ax[n-1] if n>1 else ax
        this_row.set_ylabel('x(t)')
        this_row.set_xlabel('time, t')
    
        plt.savefig('DNN_valueadd_class' + str(class_label) + '_'+fname+'.png', bbox_inches='tight')
        
plot_dnn_valueadd(x_other, y_roc_pred, meta)
def plot_similarities(x, y_true, y_pred, y_dog, title, meta):
    ''' Plot the data samples, x, where the DNN prediction (y_pred) matches
    the dogs indication (y_dog) '''
    # Calculate number of plots required
    rows = [0, 0, 0, 0]
    for i in range(len(y_pred)):
        col = -1
        if y_true[i]==1:
            if y_pred[i]==1 and y_dog[i]==1:
                col = 0
            elif y_pred[i]==0 and y_dog[i]==0:
                col = 1
        if y_true[i]==0:
            if y_pred[i]==0 and y_dog[i]==0:
                col = 2
            elif y_pred[i]==1 and y_dog[i]==1:
                col = 3
        if col != -1:
            rows[col] = rows[col]+1
    n_plots = max(rows)
    
    # Set up the subplots
    fig, ax = plt.subplots(n_plots, 4, sharex='col', sharey='row', figsize=(10, 10))
    rows = [0, 0, 0, 0]
    green_red = sns.color_palette("Paired")
    colors = [green_red[3], green_red[5], green_red[2], green_red[4]]
    meta_lists = [[], [], [], []]
    # Create each plot
    for i in range(len(y_pred)):
        col = -1
        if y_true[i]==1:
            if y_pred[i]==1 and y_dog[i]==1:
                col = 0
            elif y_pred[i]==0 and y_dog[i]==0:
                col = 1
        if y_true[i]==0:
            if y_pred[i]==0 and y_dog[i]==0:
                col = 2
            elif y_pred[i]==1 and y_dog[i]==1:
                col = 3
        if col != -1:
            row = rows[col]
            rows[col] = rows[col] + 1
            if row < n_plots:
                ax[row, col].plot(x[i], color=colors[col])
                meta_lists[col].append(meta.iloc[i])
                ax[0, col].set_title('True '+str(int(y_true[i]))+': Pred '+str(y_pred[i]))
                ax[row, col].set_ylim(bottom=0, top=2.2)
                ax[row, 0].set_yticklabels([])
    
    # Add labels and title
    for c in range(4):
        ax[n_plots-1, c].set_xlabel('time, t')
    fig.suptitle(title)
    plt.savefig('DogDNN_match_'+fname+'.png', bbox_inches='tight')
    # Print meta data
    for c in range(4):
        print('Meta data for plots in column', c)
        for r in meta_lists[c]:
            msg = r['filename'] + '\tsensor ' + str(r['sensor_number'])
            if c < 2:
                msg = msg + '\tconcentration ' +str(r['Concentration'])
            print(msg)
    
    
plot_similarities(x_other, y_other, y_roc_pred, meta['dog_pred'], 
                  'Samples where the DNN\'s prediction matched the dog\'s indication',
                  meta)<jupyter_output><empty_output><jupyter_text>Save model<jupyter_code>modelfile = logs_dir+'/'+fname+'/model'
model_json = model.to_json()
with open(modelfile+'.json', 'w') as json_file:
    json_file.write(model_json)
# Save the model's weights
model.save_weights(modelfile+'.h5')
print('Model saved to', modelfile)
# Save the other model parameters (mean and std dev of training data)
model_params          
with open(logs_dir+'/'+fname+'/model_params.csv', 'a+') as f:
    for key in model_params.keys():
        f.write("%s,%s\n"%(key,model_params[key]))
print('Results in', file)
print(result_set['lower'])
print(result_set['upper'])
print(result_set['acc_mean'])
print(result_set['std'])
print(result_set['end_test_acc_av'])
print(result_set['end_test_acc_std'])
print('(this_model_acc', result_set['this_model_acc'], ')')
print(result_set['this_end_test_acc'])<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.14262816945303977, 'fraction_numerical': 0.016629193472985908, 'mean_word_length': 3.5039101664327252, 'pattern_counts': {'":': 0, '<': 96, '<?xml version=': 0, '>': 96, 'https://': 3, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '21', 'dir': 'jupyter-structured-clean-dedup', 'id': '93439', 'license': 'MIT', 'n_tokens_mistral': 16680, 'n_tokens_neox': 16006, 'n_words': 3215, 'path': 'notebooks/development/devnet.ipynb', 'repo_name': 'Withington/deepscent', 'size': '44937'}