<jupyter_start><jupyter_text>PTB Dataset In this notebook we trained various more or less complex models on the MITBIH dataset, made predictions and estimated their performance based on the accuracy, f1 score, AUROC and AUPRC metrics   Setup<jupyter_code>import numpy as np 
import pandas as pd
import seaborn as sns
import matplotlib.pyplot as plt
import matplotlib.gridspec as gridspec
import os
from pathlib import Path
from tensorflow.keras import optimizers, losses, activations, models
from keras.callbacks import ModelCheckpoint, EarlyStopping, LearningRateScheduler, ReduceLROnPlateau
from keras.layers import Dense, Input, Dropout, Convolution1D, MaxPool1D, GlobalMaxPool1D, GlobalAveragePooling1D,concatenate,\
Dense,Dropout,LSTM,Masking,Bidirectional,Flatten, Dropout,GRU,SimpleRNN,TimeDistributed, BatchNormalization, Activation, MaxPooling1D, GlobalMaxPooling1D, Conv1D
from keras.models import Sequential,Model
import h5py
from sklearn.metrics import f1_score,accuracy_score, roc_auc_score, average_precision_score, precision_recall_curve, auc
from sklearn.neural_network import MLPClassifier
from sklearn.linear_model import LogisticRegression
from sklearn.model_selection import train_test_split
from sklearn.utils import class_weight
import collections

from google.colab import drive
drive.mount('/content/drive')
def load_ptbdb(base_path):
    df_1 = pd.read_csv(os.path.join(base_path,"data/ptbdb_normal.csv"),
                           header=None)
    df_train = df_1.sample(frac=1)
    df_2 = pd.read_csv(os.path.join(base_path,"data/ptbdb_abnormal.csv"),
                          header=None)

    df = pd.concat([df_1, df_2])

    df_train, df_test = train_test_split(df, test_size=0.2, random_state=1337, stratify=df[187])

    Y = np.array(df_train[187].values).astype(np.int8)
    X = np.array(df_train[list(range(187))].values)[..., np.newaxis]

    Y_test = np.array(df_test[187].values).astype(np.int8)
    X_test = np.array(df_test[list(range(187))].values)[..., np.newaxis]
    

    return X, X_test, Y, Y_test
from pathlib import Path

_drive_root = Path('/content/drive/MyDrive')
top_level_files = {f.name:f for f in _drive_root.iterdir() if f.is_file()}
file_bases = list(top_level_files.keys())
_test = 'Ecuador 2018 budget.gsheet'

if _test in file_bases:
    # peter is logged in
    project = Path('/content/drive/MyDrive/ETHZ-2022-S/ML-healthcare-projects/project1')
    path = str(project.resolve())
else:
    # it is not peter (and therefore lou) use standard path
    path = '/content/drive/MyDrive/ETH/'

print(f"using data folder for loading etc as \n\t:{path}")
X, X_test, Y, Y_test = load_ptbdb(base_path= path)<jupyter_output><empty_output><jupyter_text>Simple modelsWe implemented the CNN baseline model, another simpler CNN model and a simple RNN  CNN BASELINE<jupyter_code>def get_CNN_baseline_model():
    nclass = 1
    inp = Input(shape=(187, 1))
    img_1 = Convolution1D(16, kernel_size=5, activation=activations.relu, padding="valid")(inp)
    img_1 = Convolution1D(16, kernel_size=5, activation=activations.relu, padding="valid")(img_1)
    img_1 = MaxPool1D(pool_size=2)(img_1)
    img_1 = Dropout(rate=0.1)(img_1)
    img_1 = Convolution1D(32, kernel_size=3, activation=activations.relu, padding="valid")(img_1)
    img_1 = Convolution1D(32, kernel_size=3, activation=activations.relu, padding="valid")(img_1)
    img_1 = MaxPool1D(pool_size=2)(img_1)
    img_1 = Dropout(rate=0.1)(img_1)
    img_1 = Convolution1D(32, kernel_size=3, activation=activations.relu, padding="valid")(img_1)
    img_1 = Convolution1D(32, kernel_size=3, activation=activations.relu, padding="valid")(img_1)
    img_1 = MaxPool1D(pool_size=2)(img_1)
    img_1 = Dropout(rate=0.1)(img_1)
    img_1 = Convolution1D(256, kernel_size=3, activation=activations.relu, padding="valid")(img_1)
    img_1 = Convolution1D(256, kernel_size=3, activation=activations.relu, padding="valid")(img_1)
    img_1 = GlobalMaxPool1D()(img_1)
    img_1 = Dropout(rate=0.2)(img_1)

    dense_1 = Dense(64, activation=activations.relu, name="dense_1")(img_1)
    dense_1 = Dense(64, activation=activations.relu, name="dense_2")(dense_1)
    dense_1 = Dense(nclass, activation=activations.sigmoid, name="dense_3_ptbdb")(dense_1)

    model = models.Model(inputs=inp, outputs=dense_1)
    opt = optimizers.Adam(0.001)

    model.compile(optimizer=opt, loss=losses.binary_crossentropy, metrics=['acc'])
    model.summary()
    return model
model = get_CNN_baseline_model()
file_path = "baseline_CNN_ptbdb.h5"
checkpoint = ModelCheckpoint(file_path, monitor='val_acc', verbose=1, save_best_only=True, mode='max')
early = EarlyStopping(monitor="val_acc", mode="max", patience=5, verbose=1)
redonplat = ReduceLROnPlateau(monitor="val_acc", mode="max", patience=3, verbose=2)
callbacks_list = [checkpoint, early, redonplat]  # early

model.fit(X, Y, epochs=10, verbose=2, callbacks=callbacks_list, validation_split=0.1)
model.load_weights(file_path)

pred_test = model_baseline.predict(X_test)
pred_test = (pred_test>0.5).astype(np.int8)

print('Baseline model results')
f1 = f1_score(Y_test, pred_test)
print("Test f1 score : %s "% f1)

acc = accuracy_score(Y_test, pred_test)
print("Test accuracy score : %s "% acc)

# calculate Receiver Operating Characteristics AUC
roc_auc = roc_auc_score(Y_test, pred_test)
print("AUROC score : %s "% roc_auc)

# calculate precision-recall curve
precision, recall, thresholds = precision_recall_curve(Y_test, pred_test)
# calculate precision-recall AUC
prc_auc = auc(recall, precision)
print("AUPRC score : %s "% prc_auc)<jupyter_output>Model: "model"
_________________________________________________________________
 Layer (type)                Output Shape              Param #   
=================================================================
 input_1 (InputLayer)        [(None, 187, 1)]          0         
                                                                 
 conv1d (Conv1D)             (None, 183, 16)           96        
                                                                 
 conv1d_1 (Conv1D)           (None, 179, 16)           1296      
                                                                 
 max_pooling1d (MaxPooling1D  (None, 89, 16)           0         
 )                                                               
                                                                 
 dropout (Dropout)           (None, 89, 16)            0         
                                                                 
 conv1d_2 (Conv1D)           (None, 87, 32)            1568  [...]<jupyter_text>CNN<jupyter_code>CNN_model = Sequential()
CNN_model.add(Conv1D(64, kernel_size=3, activation='relu', input_shape=(187,1)))
CNN_model.add(Conv1D(32, kernel_size=3, activation='relu'))
CNN_model.add(Flatten())
CNN_model.add(Dense(2, activation='softmax'))

file_path = path + "CNN_PTB.h5"
checkpoint = ModelCheckpoint(file_path, monitor='val_accuracy', verbose=1, save_best_only=True, mode='max')
early = EarlyStopping(monitor="val_accuracy", mode="max", patience=5, verbose=1)
red = ReduceLROnPlateau(monitor="val_accuracy", mode="max", patience=3, verbose=2)
callbacks_list = [checkpoint,early, red]

CNN_model.compile(optimizer='adam', loss='sparse_categorical_crossentropy', metrics=['accuracy'])
CNN_model.summary()

CNN_model.fit(X, Y, epochs=100,batch_size=256, verbose=2, callbacks=callbacks_list, validation_split=0.1)
CNN_model.load_weights(file_path)
pred_test = CNN_model.predict(X_test)
pred_test = np.argmax(pred_test, axis=-1)

acc = accuracy_score(Y_test, pred_test)
f1 = f1_score(Y_test, pred_test, average='weighted')
roc_auc = roc_auc_score(Y_test, pred_test)
precision, recall, thresholds = precision_recall_curve(Y_test, pred_test)
prc_auc = auc(recall, precision)

print("Test accuracy score : %s "% acc)
print("Test f1 score : %s "% f1)
print("AUROC score : %s "% roc_auc)
print("AUPRC score : %s "% prc_auc)<jupyter_output>Test accuracy score : 0.9209893507385778 
Test f1 score : 0.9207445739136773 
AUROC score : 0.8985282719735986 
AUPRC score : 0.9638897863485523<jupyter_text>RNN<jupyter_code>RNN_model=Sequential()
RNN_model.add(Masking(mask_value=0.,input_shape=(187,1)))
RNN_model.add(SimpleRNN(64))
RNN_model.add(Dense(2,activation='softmax'))

file_path = path+ "RNN_PTB.h5"
checkpoint = ModelCheckpoint(file_path, monitor='val_accuracy', verbose=1, save_best_only=True, mode='max')
early = EarlyStopping(monitor="val_accuracy", mode="max", patience=5, verbose=1)
redonplat = ReduceLROnPlateau(monitor="val_accuracy", mode="max", patience=3, verbose=2)
callbacks_list = [checkpoint,early, redonplat]

RNN_model.compile(optimizer=optimizers.Adam(0.001), loss='sparse_categorical_crossentropy', metrics=['accuracy'])
RNN_model.summary()

RNN_model.fit(X, Y, epochs=10,batch_size=256, verbose=2, callbacks=callbacks_list, validation_split=0.1)
RNN_model.load_weights(file_path)
pred_test = RNN_model.predict(X_test)
pred_test = np.argmax(pred_test, axis=-1)

acc = accuracy_score(Y_test, pred_test)
f1 = f1_score(Y_test, pred_test, average='weighted')
roc_auc = roc_auc_score(Y_test, pred_test)
precision, recall, thresholds = precision_recall_curve(Y_test, pred_test)
prc_auc = auc(recall, precision)

print("Test accuracy score : %s "% acc)
print("Test f1 score : %s "% f1)
print("AUROC score : %s "% roc_auc)
print("AUPRC score : %s "% prc_auc)<jupyter_output><empty_output><jupyter_text>More advanced modelsBased on [1, 2], we have implemented simple LSTM, GRU and MLP models. Then, by trying to improve these models and looking at other studies [3], we build more complex models : Bidirectional LSTM and Bidirectional GRU.  LSTM<jupyter_code>LSTM_model = Sequential()
LSTM_model.add(Masking(mask_value=0.,input_shape=(187,1)))
LSTM_model.add(BatchNormalization())
LSTM_model.add(LSTM(64, dropout=0.2))
LSTM_model.add(BatchNormalization())
LSTM_model.add(Dense(128,activation='relu'))
LSTM_model.add(BatchNormalization())
LSTM_model.add(Dense(2,activation='softmax'))

file_path = path+ "LTSM_ptb.h5"
checkpoint = ModelCheckpoint(file_path, monitor='val_accuracy', verbose=1, save_best_only=True, mode='max')
early = EarlyStopping(monitor="val_accuracy", mode="max", patience=5, verbose=1)
red = ReduceLROnPlateau(monitor="val_accuracy", mode="max", patience=3, verbose=2)
callbacks_list = [checkpoint,early, red]

LSTM_model.compile(optimizer=optimizers.Adam(0.001), loss='sparse_categorical_crossentropy', metrics=['accuracy'])
LSTM_model.summary()

LSTM_model.fit(X, Y, epochs=100,batch_size=256, verbose=2, callbacks=callbacks_list, validation_split=0.1)
LSTM_model.load_weights(file_path)
pred_test = LSTM_model.predict(X_test)
pred_test = np.argmax(pred_test, axis=-1)

acc = accuracy_score(Y_test, pred_test)
f1 = f1_score(Y_test, pred_test, average='weighted')
roc_auc = roc_auc_score(Y_test, pred_test)
precision, recall, thresholds = precision_recall_curve(Y_test, pred_test)
prc_auc = auc(recall, precision)

print("Test accuracy score : %s "% acc)
print("Test f1 score : %s "% f1)
print("AUROC score : %s "% roc_auc)
print("AUPRC score : %s "% prc_auc)<jupyter_output><empty_output><jupyter_text>GRU<jupyter_code>GRU_model=Sequential()
GRU_model.add(Masking(mask_value=0.,input_shape=(187,1)))
GRU_model.add(BatchNormalization())
GRU_model.add(GRU(187))
GRU_model.add(BatchNormalization())
GRU_model.add(Dense(64,activation='relu'))
GRU_model.add(BatchNormalization())
GRU_model.add(Dense(2,activation='softmax'))

file_path = path + "GRU_ptb.h5"
checkpoint = ModelCheckpoint(file_path, monitor='acc', verbose=1, save_best_only=True, mode='max')
early = EarlyStopping(monitor="acc", mode="max", patience=5, verbose=1)
red = ReduceLROnPlateau(monitor="acc", mode="max", patience=3, verbose=2)
callbacks_list = [checkpoint,early, red]

GRU_model.compile(optimizer=optimizers.Adam(0.001), loss=losses.sparse_categorical_crossentropy, metrics=['acc'])
GRU_model.summary()

GRU_model.fit(X, Y, epochs=100,batch_size=256, verbose=2, callbacks=callbacks_list)
GRU_model.load_weights(file_path)
pred_test = GRU_model.predict(X_test)
pred_test = np.argmax(pred_test, axis=-1)

acc = accuracy_score(Y_test, pred_test)
f1 = f1_score(Y_test, pred_test, average='weighted')
roc_auc = roc_auc_score(Y_test, pred_test)
precision, recall, thresholds = precision_recall_curve(Y_test, pred_test)
prc_auc = auc(recall, precision)

print("Test accuracy score : %s "% acc)
print("Test f1 score : %s "% f1)
print("AUROC score : %s "% roc_auc)
print("AUPRC score : %s "% prc_auc)<jupyter_output><empty_output><jupyter_text>MLP<jupyter_code>train, test, y_train, y_test = load_ptbdb()
train, test = train[:,:,0], test[:,:,0]
clf = MLPClassifier(hidden_layer_sizes=(200,), activation='tanh',
                    random_state=1).fit(train, y_train)

pred_test = clf.predict(test)
acc = accuracy_score(y_test, pred_test)
f1 = f1_score(y_test, pred_test, average='weighted')
roc_auc = roc_auc_score(y_test, pred_test)
precision, recall, thresholds = precision_recall_curve(y_test, pred_test)
prc_auc = auc(recall, precision)

print("Test accuracy score : %s "% acc)
print("Test f1 score : %s "% f1)
print("AUROC score : %s "% roc_auc)
print("AUPRC score : %s "% prc_auc)<jupyter_output><empty_output><jupyter_text>Bidirectional GRU<jupyter_code>BidirGRU_model=Sequential()
BidirGRU_model.add(Masking(mask_value=0.,input_shape=(187,1)))
BidirGRU_model.add(BatchNormalization())
BidirGRU_model.add(Bidirectional(GRU(100,return_sequences=True)))
BidirGRU_model.add(Dropout(0.3))
BidirGRU_model.add(Bidirectional(GRU(100)))
BidirGRU_model.add(Dropout(0.1))
BidirGRU_model.add(BatchNormalization())
BidirGRU_model.add(Dense(2,activation=activations.softmax))


file_path = path+'BidirGRU_ptb.h5'
checkpoint = ModelCheckpoint(file_path, monitor='accuracy', verbose=1, save_best_only=True, mode='max')
early = EarlyStopping(monitor="accuracy", mode="max", patience=5, verbose=1)
red = ReduceLROnPlateau(monitor="accuracy", mode="max", patience=3, verbose=2)
callbacks_list = [checkpoint,early, red]

BidirGRU_model.compile(optimizer=optimizers.Adam(0.001), loss='sparse_categorical_crossentropy', metrics=['accuracy'])
BidirGRU_model.summary()

#BidirGRU_model.fit(X, Y, epochs=100,batch_size=256, verbose=2, callbacks=callbacks_list)
BidirGRU_model.load_weights(file_path)
pred_test = BidirGRU_model.predict(X_test)
pred_test = np.argmax(pred_test, axis=-1)

acc = accuracy_score(Y_test, pred_test)
f1 = f1_score(Y_test, pred_test, average='weighted')
roc_auc = roc_auc_score(Y_test, pred_test)
precision, recall, thresholds = precision_recall_curve(Y_test, pred_test)
prc_auc = auc(recall, precision)

print("Test accuracy score : %s "% acc)
print("Test f1 score : %s "% f1)
print("AUROC score : %s "% roc_auc)
print("AUPRC score : %s "% prc_auc)<jupyter_output><empty_output><jupyter_text>Bidirectional LSTM<jupyter_code>BiLSTM_model = Sequential()
BiLSTM_model.add(Masking(mask_value=0.,input_shape=(187,1)))
BiLSTM_model.add(BatchNormalization())
BiLSTM_model.add(Bidirectional(LSTM(100, return_sequences=True)))
BiLSTM_model.add(Dropout(0.3))
BiLSTM_model.add(Bidirectional(LSTM(100)))
BiLSTM_model.add(BatchNormalization())
BiLSTM_model.add(Dense(64, activation='relu'))
BiLSTM_model.add(Dense(2, activation='softmax'))

file_path = path+ "BILSTM187_ptb.h5"
checkpoint = ModelCheckpoint(file_path, monitor='val_accuracy', verbose=1, save_best_only=True, mode='max')
early = EarlyStopping(monitor="val_accuracy", mode="max", patience=5, verbose=1)
red = ReduceLROnPlateau(monitor="val_accuracy", mode="max", patience=3, verbose=2)
callbacks_list = [checkpoint,early, red]

BiLSTM_model.compile(optimizer=optimizers.Adam(0.001), loss='sparse_categorical_crossentropy', metrics=['accuracy'])
BiLSTM_model.summary()

BiLSTM_model.fit(X, Y, epochs=100,batch_size=256, verbose=2, callbacks=callbacks_list, validation_split=0.1)
BiLSTM_model.load_weights(file_path)


.fit(train,y_train)
pred_test = BiLSTM_model.predict(X_test)
pred_test = np.argmax(pred_test, axis=-1)

acc = accuracy_score(Y_test, pred_test)
f1 = f1_score(Y_test, pred_test, average='weighted')
roc_auc = roc_auc_score(Y_test, pred_test)
precision, recall, thresholds = precision_recall_curve(Y_test, pred_test)
prc_auc = auc(recall, precision)

print("Test accuracy score : %s "% acc)
print("Test f1 score : %s "% f1)
print("AUROC score : %s "% roc_auc)
print("AUPRC score : %s "% prc_auc)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.13318298183591368, 'fraction_numerical': 0.03145190783859564, 'mean_word_length': 5.893697478991597, 'pattern_counts': {'":': 0, '<': 35, '<?xml version=': 0, '>': 36, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 12, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '9', 'dir': 'jupyter-structured-clean-dedup', 'id': '356350', 'license': 'Apache-2.0', 'n_tokens_mistral': 6289, 'n_tokens_neox': 5923, 'n_words': 1045, 'path': 'notebooks/colab/PTB.ipynb', 'repo_name': 'pszemraj/ml4hc-s22-project01', 'size': '16406'}