<jupyter_start><jupyter_text>Setting Up All ArchitecturesHere, I want to setup the different architectures that will be implemented and trained on SEEG data to generate predictions of seizure.We may assume the FFT data is averaged into their electrophysiological frequency bands:* dalpha = 0-15* beta = 15-30* gamma = 30-90* high = 90-200* hfo = 200-500or without high/hfo if we choose to just analyze the lower frequency range activities.1. 2D CNN on images of FFT data projected into 2D space2. 2D CNN+LSTM on images of FFT data projected onto 2D space3. 2D CNN+biLSTM on images of FFT data projected onto 2D space4. 2D CNN+biLSTM+mix on images of FFt data projected onto 2D space5. 3D CNN on images of FFT data rendered in 3D mesh6. 3D CNN+LSTM, 3D CNN+biLSTM, 3D CNN+biLSTM+mix on images of FFT data rendered in 3D mesh<jupyter_code>import sys
sys.path.append('../../dnn/')
import model.ieeg_cnn_rnn
import model.train

import processing.util as util

import time
import numpy as np
# np.random.seed(1234)
from functools import reduce
import math as m
import os

# import DNN frameworks
import tensorflow as tf
import keras

# import high level optimizers, models and layers
from keras.optimizers import SGD
from keras.models import Sequential, Model
from keras.layers import InputLayer

# for CNN
from keras.layers import Conv2D, Conv3D, MaxPooling2D, GlobalAveragePooling2D
# for RNN
from keras.layers import LSTM, TimeDistributed

# for utility functionality
from keras.layers import Input, Dense, Dropout, Flatten

from keras.preprocessing import sequence
from keras.layers.embeddings import Embedding

# utility for datasets and training
# from keras.utils.training_utils import multi_gpu_model
from keras.datasets import imdb
from keras.preprocessing.image import ImageDataGenerator
from keras.callbacks import LearningRateScheduler
# imports tensorflow
# from keras import backend as K

from keras.utils import plot_model
from IPython.display import SVG
from keras.utils.vis_utils import model_to_dot


%load_ext autoreload
%autoreload 2
from keras.callbacks import TensorBoard

tbCallBack = TensorBoard(log_dir='Graph',
                         histogram_freq=10,
                         write_graph=True,
                         write_images=True)

test = (5,)+(5,6)
print(test)
print(vgg12cnn.input_shape)
test = vgg12cnn.weights
buff = 
# test.pop()
print(test)
print(vgg12cnn.summary())<jupyter_output>[<tf.Variable 'conv2d_1/kernel:0' shape=(3, 3, 4, 32) dtype=float32_ref>, <tf.Variable 'conv2d_1/bias:0' shape=(32,) dtype=float32_ref>, <tf.Variable 'conv2d_2/kernel:0' shape=(3, 3, 32, 32) dtype=float32_ref>, <tf.Variable 'conv2d_2/bias:0' shape=(32,) dtype=float32_ref>, <tf.Variable 'conv2d_3/kernel:0' shape=(3, 3, 32, 32) dtype=float32_ref>, <tf.Variable 'conv2d_3/bias:0' shape=(32,) dtype=float32_ref>, <tf.Variable 'conv2d_4/kernel:0' shape=(3, 3, 32, 32) dtype=float32_ref>, <tf.Variable 'conv2d_4/bias:0' shape=(32,) dtype=float32_ref>, <tf.Variable 'conv2d_5/kernel:0' shape=(3, 3, 32, 64) dtype=float32_ref>, <tf.Variable 'conv2d_5/bias:0' shape=(64,) dtype=float32_ref>, <tf.Variable 'conv2d_6/kernel:0' shape=(3, 3, 64, 64) dtype=float32_ref>, <tf.Variable 'conv2d_6/bias:0' shape=(64,) dtype=float32_ref>, <tf.Variable 'conv2d_7/kernel:0' shape=(3, 3, 64, 128) dtype=float32_ref>, <tf.Variable 'conv2d_7/bias:0' shape=(128,) dtype=float32_ref>]
_______________________________________[...]<jupyter_text>1. Build and Prototype all 2D CNN ArchitecturesHere, I want to build all the models that will be trained on the FFT images dataset projected into 2D space.<jupyter_code>imsize=32
numfreqs = 4
numclasses = 2
ieegdnn = model.ieeg_cnn_rnn.IEEGdnn(imsize=imsize, 
                                     n_colors=numfreqs,
                                    num_classes=numclasses)

w_init = None
n_layers = (4,2,1)
poolsize = (2,2)
filtersize = (3,3)

size_fc = 1024
DROPOUT = False #True

# VGG-12 style later
vgg12cnn = ieegdnn._build_2dcnn(w_init=w_init, n_layers=n_layers, 
                              poolsize=poolsize, filter_size=filtersize)
vgg12cnn = ieegdnn._build_seq_output(vgg12cnn, size_fc, DROPOUT)
print(vgg12cnn.summary())

plot_model(vgg12cnn, to_file='vgg12_plot.png', show_shapes=True, show_layer_names=True)
SVG(model_to_dot(vgg12cnn).create(prog='dot', format='svg'))
# load in CNN/LSTM
num_timewins = 5
size_mem = 50
size_fc = 1024
DROPOUT = False
cnn_lstm = ieegdnn.build_cnn_lstm(num_timewins=num_timewins, 
                                  size_mem=size_mem, 
                                  size_fc=size_fc, 
                                  BIDIRECT=True,
                                  DROPOUT=DROPOUT)

print(cnn_lstm.summary())
SVG(model_to_dot(cnn_lstm).create(prog='dot', format='svg'))
num_timewins = 5
size_mem = 128
size_fc = 1024
DROPOUT = False
cnn_same_lstm = ieegdnn.build_same_cnn_lstm(num_timewins=num_timewins, 
                                  size_mem=size_mem, 
                                  size_fc=size_fc,
                                  BIDIRECT=True,
                                  DROPOUT=DROPOUT)

print(cnn_same_lstm.input_shape)
print(cnn_same_lstm.summary())
# SVG(model_to_dot(cnn_same_lstm).create(prog='dot', format='svg'))
cnn_lstm_mix = ieegdnn.build_cnn_lstm_mix(num_timewins=num_timewins, 
                                  size_mem=size_mem, 
                                  size_fc=size_fc, 
                                  BIDIRECT=True,
                                  DROPOUT=DROPOUT)

print(cnn_lstm_mix.get_shape)
# print(cnn_lstm_mix.summary())
# SVG(model_to_dot(cnn_lstm_mix).create(prog='dot', format='svg'))
# load in CNN/LSTM
imsize=32
num_timewins = 5
size_mem = 50
size_fc = 1024
DROPOUT = False
n_colors = 4

finaldir = '/Volumes/<NAME>/pydata/dnn/output/train_v6/'
weightsfile = os.path.join(finaldir, 'final_weights.h5')
modelfile = os.path.join(finaldir, '2dcnn_model.json')

# load json and create model
json_file = open(modelfile, 'r')
loaded_model_json = json_file.read()
json_file.close()

# fixed_cnn_model = ieegdnn.load_model(weightsfile, freeze=True)
fixed_cnn_model = keras.models.model_from_json(loaded_model_json)
fixed_cnn_model.load_weights(weightsfile)

# remove the last 2 dense FC layers and freeze it
fixed_cnn_model.pop()
fixed_cnn_model.pop()
fixed_cnn_model.trainable = False

print(fixed_cnn_model.summary())
print(fixed_cnn_model.output_shape)
imsize=32
numfreqs = 4
numclasses = 2
ieegdnn = model.ieeg_cnn_rnn.IEEGdnn(imsize=imsize, 
                                     n_colors=numfreqs,
                                    num_classes=numclasses)

# create sequential model to get this all before the LSTM
currmodel = Sequential()
currmodel.add(TimeDistributed(fixed_cnn_model, input_shape=(num_timewins, imsize, imsize, n_colors)))
currmodel.add(LSTM(units=size_mem, 
            activation='relu', 
            return_sequences=False))
currmodel = ieegdnn._build_seq_output(currmodel, size_fc, DROPOUT=True)

currmodel = Model(inputs=currmodel.input, outputs = currmodel.output)
currmodel, config = ieegdnn.compile_model(currmodel, loss='binary_crossentropy', metrics=['accuracy']) 
print(currmodel.summary())
print(currmodel.input_shape)
print(currmodel.output_shape)
input_sequences = Input(shape=(num_timewins, imsize, imsize, n_colors))
test = TimeDistributed(fixed_cnn_model)(input_sequences)
test = LSTM(units=size_mem, 
            activation='relu', 
            return_sequences=False)(test)
predictions = Dense(2)(test)
test = Model(inputs=input_sequences, outputs=predictions)

print(test.summary())
test = Sequential()
test = fixed_cnn_model
test.add(Dense(10025))
print(test.summary())<jupyter_output>_________________________________________________________________
Layer (type)                 Output Shape              Param #   
=================================================================
input_1 (InputLayer)         (None, 32, 32, 4)         0         
_________________________________________________________________
conv2d_1 (Conv2D)            (None, 30, 30, 32)        1184      
_________________________________________________________________
conv2d_2 (Conv2D)            (None, 28, 28, 32)        9248      
_________________________________________________________________
conv2d_3 (Conv2D)            (None, 26, 26, 32)        9248      
_________________________________________________________________
conv2d_4 (Conv2D)            (None, 24, 24, 32)        9248      
_________________________________________________________________
max_pooling2d_1 (MaxPooling2 (None, 12, 12, 32)        0         
_________________________________________________________________
conv2d_5 ([...]<jupyter_text>a) Prepare Data For a CNN-LSTM and Misc. Functionality- test how to load model and freeze the layers- how to feed data into that model into the lstm<jupyter_code># load in CNN/LSTM
imsize=32
num_timewins = 5
size_mem = 50
size_fc = 1024
DROPOUT = False
n_colors = 4

finaldir = '/Volumes/ADAM LI/pydata/dnn/output/train_v6/'
weightsfile = os.path.join(finaldir, 'final_weights.h5')
modelfile = os.path.join(finaldir, '2dcnn_model.json')

# load json and create model
json_file = open(modelfile, 'r')
loaded_model_json = json_file.read()
json_file.close()

# fixed_cnn_model = ieegdnn.load_model(weightsfile, freeze=True)
fixed_cnn_model = keras.models.model_from_json(loaded_model_json)
fixed_cnn_model.load_weights(weightsfile)

# remove the last 2 dense FC layers and freeze it
fixed_cnn_model.pop()
fixed_cnn_model.pop()
fixed_cnn_model.trainable = False

print(fixed_cnn_model.summary())
print(fixed_cnn_model.output_shape)
# create sequential model to get this all before the LSTM
currmodel = Sequential()
currmodel.add(TimeDistributed(fixed_cnn_model, input_shape=(num_timewins, imsize, imsize, n_colors)))
currmodel.add(LSTM(units=size_mem, 
            activation='relu', 
            return_sequences=False))
currmodel = ieegdnn._build_seq_output(currmodel, size_fc, DROPOUT=True)

currmodel = Model(inputs=currmodel.input, outputs = currmodel.output)
currmodel, config = ieegdnn.compile_model(currmodel, loss='binary_crossentropy', metrics=['accuracy']) 
print(currmodel.summary())
print(currmodel.input_shape)
print(currmodel.output_shape)
##################### INPUT DATA FOR NN ####################
traindatadir ='/Volumes/ADAM LI/pydata/dnn/traindata/'
imagedir = os.path.join(traindatadir, 'image_2d')
# get all the separate files to use for training:
datafiles = []
for root, dirs, files in os.walk(imagedir):
    for file in files:
        datafiles.append(os.path.join(root, file))
print('\nTraining on ' + str(len(datafiles)) + ' datasets!\n')
import processing.util 

# Parameters
params = {'dim_x': imsize,
      'dim_y': imsize,
      'dim_z': numfreqs,
      'batch_size': 24,
      'shuffle': True}
datagen = processing.util.DataGenerator(**params)
training_generator = datagen.generate_fromdir(datafiles)

idx=0
for i,j in training_generator:
    
    print(i.shape)
    print(j.shape)
    
    if idx==2:
        break
    idx+=1
    break
import seaborn as sns
import matplotlib
import matplotlib.pyplot as plt

sns.heatmap(i[0,:,:,0].squeeze(), cmap='jet')
plt.show()
from sklearn.model_selection import train_test_split
# (X_train, y_train), (X_val, y_val), (X_test, y_test) = datahandler.reformatinput(images, labels)
X_train, X_test = train_test_split(datafiles, test_size=0.33, random_state=42)
  
print(len(datafiles))
print(len(X_train))
print(len(X_test))<jupyter_output>223
149
74<jupyter_text>b) Load Sample Data And See How That Fits Into InputNeed to test how image tensors are fed into the neural networks.<jupyter_code>import os
datadir = '/Volumes/<NAME>/pydata/dnn/traindata/image_2d/'

datafiles = []
for root, dirs, files in os.walk(datadir):
    for file in files:
        datafiles.append(os.path.join(root, file))
print('\nTraining on ' + str(len(datafiles)) + ' datasets!\n')
print(datafiles[0:5])
print("1st dim = batches/samples\n\
       2nd dim = time dim\n\
       3rd dim = W\n\
       4th dim = H\n\
       5th dim = band")
print(cnn_same_lstm.input)

maxlen = 0
for datafile in datafiles:
    data = np.load(datafile)
    image_tensor = data['image_tensor']
    metadata = data['metadata'].item()
    
    maxlen = max(maxlen, image_tensor.shape[0])
    
print(image_tensor.shape)
print(metadata.keys())
print(maxlen)
maxlen = 500
newimage_seq = keras.preprocessing.sequence.pad_sequences(test, maxlen=500, 
        dtype='float64', padding='pre', truncating='pre', value=0.)

# the training data dir
traindatadir = '/Volumes/<NAME>/pydata/dnn/traindata/finaldata/'
filename = os.path.join(traindatadir, 'allimages_2d.npz')
data = np.load(filename)
images = data['images']
print(images.shape)
test = images.reshape((-1, 4, 32, 32))
print(test.shape)
print(x_train.shape)
print(y_train.shape)
print(x_train[0])
print(y_train[0])

print('Pad sequences (samples x time)')
x_train = sequence.pad_sequences(x_train, maxlen=maxlen)
x_test = sequence.pad_sequences(x_test, maxlen=maxlen)
print('x_train shape:', x_train.shape)
print('x_test shape:', x_test.shape)

print('Build model...')
model = Sequential()
model.add(Embedding(max_features, 128))
model.add(LSTM(128, dropout=0.2, recurrent_dropout=0.2))
model.add(Dense(1, activation='sigmoid'))<jupyter_output><empty_output><jupyter_text>2. Building Architectures for 3D CNNBuild the CNN, CNN+LSTM, CNN+BidirectLSTM, CNN+BidirectLSTM+mix<jupyter_code>w_init = None
n_layers = (4,2,1)
poolsize = (2,2,2)
filtersize = (3,3,3)
size_fc = 1024
DROPOUT = False #True

# VGG-12 style later
vgg12cnn = ieegdnn._build_3dcnn(w_init=w_init, n_layers=n_layers, 
                              poolsize=poolsize, filter_size=filtersize)
vgg12cnn = ieegdnn._build_seq_output(vgg12cnn, size_fc, DROPOUT)
print(vgg12cnn.summary())
SVG(model_to_dot(vgg12cnn).create(prog='dot', format='svg'))
# load in CNN/LSTM
num_timewins = 5
size_mem = 128
size_fc = 1024
DROPOUT = False
# code up 3D CNN+LSTM, CNN+LSTMmix and get outputs
cnn_same_lstm = ieegdnn.build_same_cnn_lstm(num_timewins=num_timewins, 
                                  size_mem=size_mem, 
                                  size_fc=size_fc,
                                  dim=3,
                                  BIDIRECT=True,
                                  DROPOUT=DROPOUT)

print(cnn_same_lstm.summary())
SVG(model_to_dot(cnn_same_lstm).create(prog='dot', format='svg'))

cnn_lstm_mix = ieegdnn.build_cnn_lstm_mix(num_timewins=num_timewins, 
                                  size_mem=size_mem, 
                                  size_fc=size_fc, 
                                  dim=3,
                                  BIDIRECT=True,
                                  DROPOUT=DROPOUT)

print(type(cnn_lstm_mix))
print

print(cnn_lstm_mix.summary())
SVG(model_to_dot(cnn_lstm_mix).create(prog='dot', format='svg'))<jupyter_output>__________________________________________________________________________________________________
Layer (type)                    Output Shape         Param #     Connected to                     
==================================================================================================
input_15 (InputLayer)           (None, 32, 32, 32, 4 0                                            
__________________________________________________________________________________________________
input_16 (InputLayer)           (None, 32, 32, 32, 4 0                                            
__________________________________________________________________________________________________
input_17 (InputLayer)           (None, 32, 32, 32, 4 0                                            
__________________________________________________________________________________________________
input_18 (InputLayer)           (None, 32, 32, 32, 4 0                                            
__________[...]<jupyter_text>3. Build Up Some PreBuilt Keras ModelsImport VGG and certain architectures that already have pretrained weights.I will test say... VGG16, InceptionV3 and Xception<jupyter_code>from keras.applications import VGG16, VGG19, MobileNet, Xception, InceptionV3
from keras.models import Model
# create the base pre-trained model
base_model = VGG16(weights='imagenet', include_top=False)
# add a global spatial average pooling layer
x = base_model.output
x = GlobalAveragePooling2D()(x)
# let's add a fully-connected layer
x = Dense(1024, activation='relu')(x)
# and a logistic layer -- let's say we have 200 classes
predictions = Dense(200, activation='softmax')(x)
# this is the model we will train
model = Model(inputs=base_model.input, outputs=predictions)

print(model.summary())
# create the base pre-trained model
base_model = VGG19(weights='imagenet', include_top=False)
# add a global spatial average pooling layer
x = base_model.output
x = GlobalAveragePooling2D()(x)
# let's add a fully-connected layer
x = Dense(1024, activation='relu')(x)
# and a logistic layer -- let's say we have 200 classes
predictions = Dense(200, activation='softmax')(x)
# this is the model we will train
model = Model(inputs=base_model.input, outputs=predictions)

print(model.summary())
# create the base pre-trained model
base_model = MobileNet(input_shape=(128,128,3), weights='imagenet', include_top=False)
# add a global spatial average pooling layer
x = base_model.output
x = GlobalAveragePooling2D()(x)
# let's add a fully-connected layer
x = Dense(1024, activation='relu')(x)
# and a logistic layer -- let's say we have 200 classes
predictions = Dense(200, activation='softmax')(x)
# this is the model we will train
model = Model(inputs=base_model.input, outputs=predictions)

print(model.summary())
# create the base pre-trained model
base_model = Xception(weights='imagenet', include_top=False)
# add a global spatial average pooling layer
x = base_model.output
x = GlobalAveragePooling2D()(x)
# let's add a fully-connected layer
x = Dense(1024, activation='relu')(x)
# and a logistic layer -- let's say we have 200 classes
predictions = Dense(200, activation='softmax')(x)
# this is the model we will train
model = Model(inputs=base_model.input, outputs=predictions)

print(model.summary())
# create the base pre-trained model
base_model = InceptionV3(weights='imagenet', include_top=False)
# add a global spatial average pooling layer
x = base_model.output
x = GlobalAveragePooling2D()(x)
# let's add a fully-connected layer
x = Dense(1024, activation='relu')(x)
# and a logistic layer -- let's say we have 200 classes
predictions = Dense(200, activation='softmax')(x)
# this is the model we will train
model = Model(inputs=base_model.input, outputs=predictions)

print(model.summary())<jupyter_output>Downloading data from https://github.com/fchollet/deep-learning-models/releases/download/v0.5/inception_v3_weights_tf_dim_ordering_tf_kernels_notop.h5
87916544/87910968 [==============================] - 53s 1us/step
__________________________________________________________________________________________________
Layer (type)                    Output Shape         Param #     Connected to                     
==================================================================================================
input_23 (InputLayer)           (None, None, None, 3 0                                            
__________________________________________________________________________________________________
conv2d_12 (Conv2D)              (None, None, None, 3 864         input_23[0][0]                   
__________________________________________________________________________________________________
batch_normalization_5 (BatchNor (None, None, None, 3 96          conv2d_12[0][0]          [...]
{'fraction_non_alphanumeric': 0.1162413110761581, 'fraction_numerical': 0.031001065503069664, 'mean_word_length': 3.9547511312217196, 'pattern_counts': {'":': 0, '<': 37, '<?xml version=': 0, '>': 37, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 38, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '6', 'dir': 'jupyter-structured-clean-dedup', 'id': '369634', 'license': 'Apache-2.0', 'n_tokens_mistral': 6730, 'n_tokens_neox': 6230, 'n_words': 1449, 'path': 'notebooks/Testing Architectures/DNN Architecture Setups - 2DCNN, 3DCNN, +LSTM, +bidirectionLSTM, +mix.ipynb', 'repo_name': 'adam2392/dnn-unsupervised', 'size': '19712'}