<jupyter_start><jupyter_text>Make final prediciton results for Kaggle submission (Deep_Sith)Use  PyTorch 1.4.0 Py3.7 Kernal/ Container on RivannaTry to use V100 GPU, since it is much faster than the others  pip install the necessary packages and download the [SITH_Layer_master] folder<jupyter_code>#!pip install --user mne
#!pip install --user seaborn
#### pytorch and Cuda should be set up correctly on the Pytorch kernal or pytorch container
from src.eval import evaluation
from src.train_util import *
from models.Deep_isith_EEG_model import *
from models.LSTM_EEG_model import *
import pandas as pd

import warnings
warnings.filterwarnings('ignore')
from src.train_util import *
from models.Deep_isith_EEG_model import *
from models.LSTM_EEG_model import *

# read config file
import configparser
import argparse

# preprocessing
import mne
import numpy as np
import math

# pytorch
import torch
import torch.nn
import torch.nn.functional as F
ttype = torch.cuda.DoubleTensor if torch.cuda.is_available() else torch.DoubleTensor
labeltype = torch.cuda.LongTensor if torch.cuda.is_available() else torch.LongTensor
print(ttype)
from torch.utils.data import Dataset,DataLoader 

import matplotlib.pyplot as plt

# training 
from torch import nn as nn
from math import factorial
import random
import seaborn as sn
import os 
from os.path import join
import glob

# validation
from sklearn.metrics import roc_curve, auc, roc_auc_score, matthews_corrcoef,confusion_matrix,plot_roc_curve
from sklearn.metrics import precision_score, recall_score, f1_score<jupyter_output><class 'torch.cuda.DoubleTensor'><jupyter_text>Predict the test dataset<jupyter_code>def predict(model, test_loader):
    """
    
    Iterate through each batch and make prediciton and calculate performance metrics
    Use **matthews correlation coeefficient** since the data are imbanlanced
    Again 
    Signals need to be in correct format. validation input: [nbatch x 1 x nFeutures x time] tensor.

    The target has dimension of [time] tensor, in which each entry should be one of the numbers in 
    {0,1,2, ... K} at any time point.  
    
    """
    inferenced_y = np.empty(0)
    for _, (val_x) in enumerate(test_loader):
        #print(val_x.shape)
        # the actual tensor
        out_val = model(val_x)
        #print(out_val.shape)
        # pass through a softmax to tansform to probability on the third dimention (nbatch, seq, outFeature)
        res = torch.nn.functional.softmax(out_val, dim=2)
        #print(res.shape)
        # predict should also be the second dimension [1] to clauclate AUC
        y_pred = res[:,:,1]

        # flatten the predicted result 
        y_score = np.ndarray.flatten(y_pred.detach().cpu().numpy())
        
        inferenced_y = np.concatenate((inferenced_y, y_score), axis=0)

    return inferenced_y<jupyter_output><empty_output><jupyter_text>Sith parameters. Have to be exactly the same with the training DeepSITH parameters<jupyter_code>sith_params1 = {"in_features":32, 
                        "tau_min":1, "tau_max":50, 
                        "k":23, 'dt':1,
                        "ntau":10, 'g':0.0,  
                        "ttype":ttype, 
                        "hidden_size":20, "act_func":nn.ReLU()}

sith_params2 = {"in_features":sith_params1['hidden_size'], 
                        "tau_min":1, "tau_max":200.0,  
                        "k":12, 'dt':1,
                        "ntau":10, 'g':0.0, 
                        "ttype":ttype, 
                        "hidden_size":20, "act_func":nn.ReLU()}
sith_params3 = {"in_features":sith_params2['hidden_size'], 
                    "tau_min":1, "tau_max":800.0,  
                    "k":7, 'dt':1,
                    "ntau":10, 'g':0.0, 
                    "ttype":ttype, 
                    "hidden_size":20, "act_func":nn.ReLU()}
layer_params = [sith_params1, sith_params2,sith_params3]<jupyter_output><empty_output><jupyter_text>Deep_SITH dataset class<jupyter_code># override EEGDataset in src
class EEGDataset_pred(Dataset):
    """
    A pytorch dataset
    input shapes:
        train_x: [nbatch, channels, sequence]
        train_y: [nbatch,  sequence]
    
    Output shape:
        Need to add a magic second dimension in order for Deep_sith
        to work properly
        train_x: [nbatch, 1, channels, sequence]
        train_y: [nbatch,  sequence]
    """
    def __init__(self, test_x):
        self.test_x = test_x


    def __len__(self):
        return self.test_x.shape[0]
    
    def __getitem__(self, idx):
                
        return (self.test_x[idx].unsqueeze(0))
        
def load_test_data(test_x_t ,batch_size = 1):
    # batch_size is a hyper parameter to tune 
    dataset = EEGDataset_pred(test_x_t)

    # get the entire length of the dataset
    dataset_size = len(dataset)

    test_loader = DataLoader(dataset= dataset, batch_size=batch_size, 
                             shuffle=False)

    return (test_loader)<jupyter_output><empty_output><jupyter_text>Read config file from ./config<jupyter_code>import warnings
warnings.filterwarnings('ignore')
from src.train_util import *
from models.Deep_isith_EEG_model import *
from models.LSTM_EEG_model import *
import pandas as pd
# read config file
import configparser
import argparse

# enable use of command line
parser = argparse.ArgumentParser(description='Input config files')
parser.add_argument('--config', default = 'config/testing_config_Deep_isith.ini', type=str,
                    help='an integer for the accumulator')
opt, _ = parser.parse_known_args()

# parser to read parameters
config = configparser.ConfigParser()
config.sections()

# parameters from config file
results = []
config.read(opt.config)
dir = config['data']['directory']
subject_num = int(config['data']['subject #'])
kernel_size = int(config['training']['kernel_size'])# sliding window size to use
step = int(config['training']['step']) #  --the step between each slice. means overlap between batches is 1- step 
modelName = config['training']['model']
# num of epochs to train
nepochs = int(config['training']['nepochs'])
loss_func =  torch.nn.CrossEntropyLoss()
batch_size = int(config['training']['batch_size']) # batch_size is a hyper parameter to tune 
train_split = float(config['training']['train_split'])
lr = float(config['training']['lr'])<jupyter_output><empty_output><jupyter_text>Override the function *filter_standardization* in src. Add zero paddings for training and predicitons<jupyter_code># currently override the function in src
def filter_standardization(raw,window_size = 1000,
                          l_freq = 0,h_freq = 30, verbose = False, read_event = True):
    """
    raw: raw object from mnew
    window_size: rolling window_size for standardization,
    l_freq, h_freq: frequency filters
    nClass: the number of event channel to use 
    """

    filtered_X = raw.filter(l_freq=l_freq, h_freq= h_freq, method='fir',phase="minimum",
                            verbose= verbose, picks = [x for x in range(32)])
    filtered_X = filtered_X.to_data_frame().drop(['time'],axis=1)
    #print(filtered_X)
    # insert a padding dataframe to add to the beginning of the dataframe
    # this is to help sliding window standardization
    
    colNames = filtered_X.columns
    padding_df = pd.DataFrame(np.zeros((window_size-1, 32)), columns = colNames)
    filtered_X = padding_df.append(filtered_X, ignore_index=True)
    #print(filtered_X)
    # only the first 32 channels to standardize
    filtered_X = filtered_X.iloc[:,0:32]
    filtered_standardized = ((filtered_X - filtered_X.rolling(window_size).mean()) / filtered_X.rolling(window_size).std()).dropna()
    # filtered and stardardized training data
    input_signal = filtered_standardized.to_numpy()
    input_signal = np.swapaxes(input_signal,0,1)

    data = raw.get_data()
    
    if read_event:
        # strip the first 99 data points due to rolling window implementation
        target_signal = target_signal_val =data[32:38,window_size-1:] # export all channels, use only one channel eventually
        #print(input_signal.shape,target_signal.shape)
    else:
        target_signal = None
    # reformatt into tensor
#     input_tensor = ttype(input_signal.reshape(1,1,input_signal.shape[0],-1))
#     target_tensor = labeltype(target_signal.reshape(-1))

    #print(input_tensor.shape, target_tensor.shape)
    return (input_signal, target_signal)

# make predictions
colName = ['id','HandStart', 'FirstDigitTouch', 'BothStartLoadPhase',
          'LiftOff', 'Replace', 'BothReleased']
#final dataframe
df = pd.DataFrame(columns = colName)
for j in range(1,13): # out loop train all subjects
    results = []
    subject_num = j # ignore the config subject num
    # load data and do preprocessing
    test_x_list = []
    #test_y_list = []
    print(f"Starting to load Subject{subject_num} Data.")
    # loop for each series
    for file in os.listdir(dir):
        sub_idx = file.find('_')
        if file[:-4].endswith('_data') & (file[4:sub_idx] == str(subject_num)):
            # get the name
            sub_idx_all = [i for i in range(len(file)) if file.startswith('_', i)]
            id_s = file[:sub_idx_all[1]]
            print(file[:sub_idx_all[1]])
            # No events file for the test datset
            raw = creat_mne_raw_object(dir+file,read_events=False)
            # filter all channels
            input_signal,_ = filter_standardization(raw,window_size = 1000,
                                l_freq = 0,h_freq = 30, read_event = False)

            input_tensor = ttype(input_signal.reshape(1,1,input_signal.shape[0],-1))
            #target_tensor = labeltype(target_signal.reshape(6,-1)) # should be six channels
            input_tensor = input_tensor.squeeze(0)
            print(input_tensor.shape)
            ###########for testing do not patch data ###########
            #patches_test = input_tensor.unfold(dimension = 1, 
                                                #size = kernel_size, 
                                                #step = step).permute(1,0,2)
            #patches_label = target_tensor.unfold(1, kernel_size, step).permute(1,0,2)
            
            #print(patches_train.shape, patches_label.shape)

            # append to a list
            #test_x_list.append(patches_test)
            #train_y_list.append(patches_label)  

            # make prediction one by one
            test_x_t = input_tensor
            #test_y_t = torch.cat(train_y_list, dim=0)
            print(test_x_t.shape)
    
            # make predictions
            df_pred = pd.DataFrame(columns = colName)
            # another loop for each event
            for i in range(1,7): # There are six events 1 - 6
                # make a copy of every dict don't want to change them
                layer_params_l = [sith_params1.copy(), 
                                  sith_params2.copy(),sith_params3.copy()]
                nClass = i - 1
                # j is the current subject
                file = f'Deep_isith_Subject{j}_numEvent{nClass}.pth'
                PATH = 'saved_NNs/' + file
                model1 = DeepSITH_Tracker(out=2,
                                         layer_params=layer_params_l, dropout=0.1).double()
                model1.load_state_dict(torch.load(PATH))
                model1.eval()
                device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
                model1.to(device)

                #train_y_t_nClass = train_y_t[:,nClass,:]
                # create dataloader class
                #print(f'batch_size is {batch_size}')
                test_loader = load_test_data(test_x_t,
                                batch_size = batch_size)
                y_pred = predict(model1, test_loader)
                
                # +1 beacuse we have id
                df_pred[colName[nClass+1]] = y_pred

                df_pred['id'] = [f'{id_s}_{data_num}' for data_num in range(0, len(df_pred))]

                # end of loop
            # append to the final df
            df = df.append(df_pred,ignore_index=True)
    
df<jupyter_output><empty_output><jupyter_text>Save final DataFrame to csv<jupyter_code>if not os.path.exists('prediction'):
    os.makedirs('prediction')
df.to_csv('prediction/' + 'Deep_isith_FinalResult_padding.csv', index = False)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.10058120762027768, 'fraction_numerical': 0.012431385211494995, 'mean_word_length': 2.762222897054358, 'pattern_counts': {'":': 24, '<': 29, '<?xml version=': 0, '>': 29, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 8, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '7', 'dir': 'jupyter-structured-clean-dedup', 'id': '156397', 'license': 'MIT', 'n_tokens_mistral': 3922, 'n_tokens_neox': 3765, 'n_words': 950, 'path': 'test_DeepSITH.ipynb', 'repo_name': 'gauvand/SIF_Capstone', 'size': '12388'}