<jupyter_start><jupyter_text>Ensemble decoder on full dataset  User Options<jupyter_code># dataset='s1'
# dataset='m1'
dataset='hc'
#What folder to save the ensemble results to
#Note that the data we are loading are in this same folder (since they are the results from the other decoders)
save_folder=''
# save_folder='/home/jglaser/Files/Neural_Decoding/Results/'<jupyter_output><empty_output><jupyter_text>Import packages<jupyter_code>import numpy as np
import matplotlib.pyplot as plt
%matplotlib inline
from scipy import io
from scipy import stats
import pickle
import sys

#Add the main folder to the path, so we have access to the files there.
#Note that if your working directory is not the Paper_code folder, you may need to manually specify the path to the main folder. For example: sys.path.append('/home/jglaser/GitProj/Neural_Decoding')
sys.path.append('..') 

#Import metrics
from metrics import get_R2

from decoders import DenseNNDecoder

from bayes_opt import BayesianOptimization
#Turn off deprecation warnings

import warnings
warnings.filterwarnings("ignore", category=DeprecationWarning)<jupyter_output><empty_output><jupyter_text>Load results from other decodersNote we do not use the Kalman filter results in our ensemble due to slightly different formatting<jupyter_code>with open(save_folder+dataset+'_ground_truth.pickle','rb') as f:
    [y_test_all,y_train_all,y_valid_all]=pickle.load(f)

with open(save_folder+dataset+'_results_wf2.pickle','rb') as f:
    [mean_r2_wf,y_pred_wf_all,y_train_pred_wf_all,y_valid_pred_wf_all]=pickle.load(f)

with open(save_folder+dataset+'_results_wc2.pickle','rb') as f:
    [mean_r2_wc,y_pred_wc_all,y_train_pred_wc_all,y_valid_pred_wc_all]=pickle.load(f)    
    
with open(save_folder+dataset+'_results_xgb2.pickle','rb') as f:
    [mean_r2_xgb,y_pred_xgb_all,y_train_pred_xgb_all,y_valid_pred_xgb_all,time_elapsed]=pickle.load(f)

with open(save_folder+dataset+'_results_svr2.pickle','rb') as f:
    [mean_r2_svr,y_pred_svr_all,y_train_pred_svr_all,y_valid_pred_svr_all,time_elapsed]=pickle.load(f)

with open(save_folder+dataset+'_results_dnn2.pickle','rb') as f:
    [mean_r2_dnn,y_pred_dnn_all,y_train_pred_dnn_all,y_valid_pred_dnn_all,time_elapsed]=pickle.load(f)    

with open(save_folder+dataset+'_results_rnn2.pickle','rb') as f:
    [mean_r2_rnn,y_pred_rnn_all,y_train_pred_rnn_all,y_valid_pred_rnn_all,time_elapsed]=pickle.load(f)

with open(save_folder+dataset+'_results_gru2.pickle','rb') as f: 
        [mean_r2_gru,y_pred_gru_all,y_train_pred_gru_all,y_valid_pred_gru_all,time_elapsed]=pickle.load(f)

with open(save_folder+dataset+'_results_lstm2.pickle','rb') as f:
    [mean_r2_lstm,y_pred_lstm_all,y_train_pred_lstm_all,y_valid_pred_lstm_all,time_elapsed]=pickle.load(f)<jupyter_output><empty_output><jupyter_text>Run ensemble method1. We loop through each CV fold and both out (x and y position/velocities).2. We create the matrix of covariates (the predictions from the other methods)3. We optimize the hyperparameters for the fully connected (dense) neural network we are using, based on validation set R2 values4. We fit the neural net on training data w/ the optimal hyperparameters5. We make test set predictions and get test set R2 values<jupyter_code>##Initialize
y_pred_ensemble_all=[] #List where test set predictions are put (for saving and plotting)
mean_r2_dnn=np.empty([10,2]) #Where the R2 values are saved (matrix of 10 CV folds x 2 outputs)


for i in range(10): #Loop through the cross validation folds
    for j in range(2): #Loop through the 2 output predictions (x and y positions/velocities)

        
        
        ###CREATE COVARIATES###
        
        #Make matrix of covariates, where each feature is the predictions from one of the other decoders
        #Do this for training, validation, and testing data
        X_train=np.concatenate((y_train_pred_wf_all[i][:,j:j+1], y_train_pred_wc_all[i][:,j:j+1], 
                                y_train_pred_svr_all[i][:,j:j+1],y_train_pred_xgb_all[i][:,j:j+1],
                                y_train_pred_dnn_all[i][:,j:j+1], y_train_pred_rnn_all[i][:,j:j+1],
                                y_train_pred_gru_all[i][:,j:j+1], y_train_pred_lstm_all[i][:,j:j+1]),axis=1)
        X_valid=np.concatenate((y_valid_pred_wf_all[i][:,j:j+1], y_valid_pred_wc_all[i][:,j:j+1], 
                                y_valid_pred_svr_all[i][:,j:j+1],y_valid_pred_xgb_all[i][:,j:j+1],
                                y_valid_pred_dnn_all[i][:,j:j+1], y_valid_pred_rnn_all[i][:,j:j+1],
                                y_valid_pred_gru_all[i][:,j:j+1], y_valid_pred_lstm_all[i][:,j:j+1]),axis=1)
        X_test=np.concatenate((y_pred_wf_all[i][:,j:j+1], y_pred_wc_all[i][:,j:j+1], 
                               y_pred_svr_all[i][:,j:j+1],y_pred_xgb_all[i][:,j:j+1],
                               y_pred_dnn_all[i][:,j:j+1], y_pred_rnn_all[i][:,j:j+1],
                               y_pred_gru_all[i][:,j:j+1], y_pred_lstm_all[i][:,j:j+1]),axis=1)
        
        #Get outputs (training/validation/testing) for this CV fold and output
        y_train=y_train_all[i][:,j:j+1]
        y_valid=y_valid_all[i][:,j:j+1]
        y_test=y_test_all[i][:,j:j+1]

      
        ###HYPERPARAMETER OPTIMIZATION###
        
        #Define a function that returns the metric we are trying to optimize (R2 value of the validation set)
        #as a function of the hyperparameter we are fitting (num_units, frac_dropout, n_epochs)
        def dnn_evaluate(num_units,frac_dropout,n_epochs):
            num_units=int(num_units) #Put in proper format (Bayesian optimization uses floats, and we just want to test the integer)
            frac_dropout=float(frac_dropout) #Put in proper format
            n_epochs=int(n_epochs) #Put in proper format
            model_dnn=DenseNNDecoder(units=[num_units,num_units],dropout=frac_dropout,num_epochs=n_epochs) #Define model
            model_dnn.fit(X_train,y_train) #Fit model
            y_valid_predicted_dnn=model_dnn.predict(X_valid) #Get validation set predictions
            return np.mean(get_R2(y_valid,y_valid_predicted_dnn)) #Return mean validation set R2

        #Do bayesian optimization
        dnnBO = BayesianOptimization(dnn_evaluate, {'num_units': (3, 50), 'frac_dropout': (0,.5), 'n_epochs': (2,10)}, verbose=0) #Define Bayesian optimization, and set limits of hyperparameters
        #Set number of initial runs and subsequent tests, and do the optimization. Also, we set kappa=10 (greater than the default) so there is more exploration when there are more hyperparameters
        dnnBO.maximize(init_points=10, n_iter=15, kappa=10)
        best_params=dnnBO.res['max']['max_params'] #Get the hyperparameters that give rise to the best fit
        num_units=np.int(best_params['num_units']) #We want the integer value associated with the best "num_units" parameter (which is what the xgb_evaluate function does above)
        frac_dropout=float(best_params['frac_dropout'])
        n_epochs=np.int(best_params['n_epochs']) 

        # Run model w/ above hyperparameters
        
        model_dnn=DenseNNDecoder(units=[num_units,num_units],dropout=frac_dropout,num_epochs=n_epochs) #Declare model w/ fit hyperparameters
        model_dnn.fit(X_train,y_train) #Fit model
        y_test_predicted_dnn=model_dnn.predict(X_test) #Get test set predictions
        mean_r2_dnn[i,j]=np.mean(get_R2(y_test,y_test_predicted_dnn))  #Get test set R2  
        #Print R2 values
        R2s_dnn=get_R2(y_test,y_test_predicted_dnn)
        print('R2s:', R2s_dnn)            
                
        
        y_pred_ensemble_all.append(y_test_predicted_dnn) #  #Add test set predictions to list (for saving)    

mean_r2_ensemble=np.mean(mean_r2_dnn,axis=1) #Get mean R2 value for each fold (across x and y predictions)

#Save data
with open(save_folder+dataset+'_results_ensemble_dnn2.pickle','wb') as f:
    pickle.dump([mean_r2_ensemble,y_pred_ensemble_all],f)<jupyter_output>/opt/anaconda/anaconda2/lib/python2.7/site-packages/keras/models.py:826: UserWarning: The `nb_epoch` argument in `fit` has been renamed `epochs`.
  warnings.warn('The `nb_epoch` argument in `fit` '
{'fraction_non_alphanumeric': 0.10603088992400098, 'fraction_numerical': 0.013238538857563127, 'mean_word_length': 4.3854785478547855, 'pattern_counts': {'":': 0, '<': 16, '<?xml version=': 0, '>': 16, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '632892', 'license': 'BSD-3-Clause', 'n_tokens_mistral': 3017, 'n_tokens_neox': 2965, 'n_words': 611, 'path': 'Paper_code/Ensemble_FullData.ipynb', 'repo_name': 'horsto/Neural_Decoding', 'size': '8158'}