# Path: echotorch/nn/BDESNCell.py
# class BDESNCell(nn.Module):
#     """
#     Bi-directional Echo State Network module
#     """
# 
#     # Constructor
#     def __init__(self, input_dim, hidden_dim, spectral_radius=0.9, bias_scaling=0, input_scaling=1.0,
#                  w=None, w_in=None, w_bias=None, sparsity=None, input_set=[1.0, -1.0], w_sparsity=None,
#                  nonlin_func=torch.tanh,  leaky_rate=1.0, create_cell=True):
#         """
#         Constructor
#         :param input_dim: Inputs dimension.
#         :param hidden_dim: Hidden layer dimension
#         :param spectral_radius: Reservoir's spectral radius
#         :param bias_scaling: Scaling of the bias, a constant input to each neuron (default: 0, no bias)
#         :param input_scaling: Scaling of the input weight matrix, default 1.
#         :param w: Internal weights matrix
#         :param w_in: Input-reservoir weights matrix
#         :param w_bias: Bias weights matrix
#         :param sparsity:
#         :param input_set:
#         :param w_sparsity:
#         :param nonlin_func: Reservoir's activation function (tanh, sig, relu)
#         """
#         super(BDESNCell, self).__init__()
# 
#         # Recurrent layer
#         if create_cell:
#             self.esn_cell = LiESNCell(leaky_rate, False, input_dim, hidden_dim, spectral_radius, bias_scaling,
#                                       input_scaling, w, w_in, w_bias, None, sparsity, input_set, w_sparsity,
#                                       nonlin_func)
#         # end if
#     # end __init__
# 
#     ###############################################
#     # PROPERTIES
#     ###############################################
# 
#     # Hidden weight matrix
#     @property
#     def w(self):
#         """
#         Hidden weight matrix
#         :return:
#         """
#         return self.esn_cell.w
#     # end w
# 
#     # Input matrix
#     @property
#     def w_in(self):
#         """
#         Input matrix
#         :return:
#         """
#         return self.esn_cell.w_in
#     # end w_in
# 
#     ###############################################
#     # PUBLIC
#     ###############################################
# 
#     # Reset learning
#     def reset(self):
#         """
#         Reset learning
#         :return:
#         """
#         # Reset output layer
#         self.output.reset()
# 
#         # Training mode again
#         self.train(True)
#     # end reset
# 
#     # Output matrix
#     def get_w_out(self):
#         """
#         Output matrix
#         :return:
#         """
#         return self.output.w_out
#     # end get_w_out
# 
#     # Set W
#     def set_w(self, w):
#         """
#         Set W
#         :param w:
#         :return:
#         """
#         self.esn_cell.w = w
#     # end set_w
# 
#     # Forward
#     def forward(self, u, y=None):
#         """
#         Forward
#         :param u: Input signal.
#         :param y: Target outputs
#         :return: Output or hidden states
#         """
#         # Forward compute hidden states
#         forward_hidden_states = self.esn_cell(u)
# 
#         # Backward compute hidden states
#         backward_hidden_states = self.esn_cell(Variable(torch.from_numpy(np.flip(u.data.numpy(), 1).copy())))
#         backward_hidden_states = Variable(torch.from_numpy(np.flip(backward_hidden_states.data.numpy(), 1).copy()))
# 
#         return torch.cat((forward_hidden_states, backward_hidden_states), dim=2)
#     # end forward
# 
#     # Finish training
#     def finalize(self):
#         """
#         Finalize training with LU factorization
#         """
#         # Finalize output training
#         self.output.finalize()
# 
#         # Not in training mode anymore
#         self.train(False)
#     # end finalize
# 
#     # Reset hidden layer
#     def reset_hidden(self):
#         """
#         Reset hidden layer
#         :return:
#         """
#         self.esn_cell.reset_hidden()
#     # end reset_hidden
# 
#     # Get W's spectral radius
#     def get_spectral_radius(self):
#         """
#         Get W's spectral radius
#         :return: W's spectral radius
#         """
#         return self.esn_cell.get_spectral_radius()
#     # end spectral_radius


import torch
import torch.nn as nn
import torch.nn.functional as F
import matplotlib.pyplot as plt
from .BDESNCell import BDESNCell
from sklearn.decomposition import IncrementalPCA
from torch.autograd import Variable
    # Constructor
    def __init__(self, input_dim, hidden_dim, output_dim, pca_dim, linear_dim, leaky_rate=1.0, spectral_radius=0.9, bias_scaling=0,
                 input_scaling=1.0, w=None, w_in=None, w_bias=None, sparsity=None, input_set=[1.0, -1.0],
                 w_sparsity=None, nonlin_func=torch.tanh, learning_algo='inv', ridge_param=0.0, create_cell=True,
                 pca_batch_size=10):
        """
        Constructor
        :param input_dim: Inputs dimension.
        :param hidden_dim: Hidden layer dimension
        :param output_dim: Reservoir size
        :param spectral_radius: Reservoir's spectral radius
        :param bias_scaling: Scaling of the bias, a constant input to each neuron (default: 0, no bias)
        :param input_scaling: Scaling of the input weight matrix, default 1.
        :param w: Internal weights matrix
        :param w_in: Input-reservoir weights matrix
        :param w_bias: Bias weights matrix
        :param sparsity:
        :param input_set:
        :param w_sparsity:
        :param nonlin_func: Reservoir's activation function (tanh, sig, relu)
        :param learning_algo: Which learning algorithm to use (inv, LU, grad)
        """
        super(BDESNPCA, self).__init__()

        # Properties
        self.output_dim = output_dim
        self.pca_dim = pca_dim

        # Recurrent layer
        if create_cell:

# Path: echotorch/nn/BDESNCell.py
# class BDESNCell(nn.Module):
#     """
#     Bi-directional Echo State Network module
#     """
# 
#     # Constructor
#     def __init__(self, input_dim, hidden_dim, spectral_radius=0.9, bias_scaling=0, input_scaling=1.0,
#                  w=None, w_in=None, w_bias=None, sparsity=None, input_set=[1.0, -1.0], w_sparsity=None,
#                  nonlin_func=torch.tanh,  leaky_rate=1.0, create_cell=True):
#         """
#         Constructor
#         :param input_dim: Inputs dimension.
#         :param hidden_dim: Hidden layer dimension
#         :param spectral_radius: Reservoir's spectral radius
#         :param bias_scaling: Scaling of the bias, a constant input to each neuron (default: 0, no bias)
#         :param input_scaling: Scaling of the input weight matrix, default 1.
#         :param w: Internal weights matrix
#         :param w_in: Input-reservoir weights matrix
#         :param w_bias: Bias weights matrix
#         :param sparsity:
#         :param input_set:
#         :param w_sparsity:
#         :param nonlin_func: Reservoir's activation function (tanh, sig, relu)
#         """
#         super(BDESNCell, self).__init__()
# 
#         # Recurrent layer
#         if create_cell:
#             self.esn_cell = LiESNCell(leaky_rate, False, input_dim, hidden_dim, spectral_radius, bias_scaling,
#                                       input_scaling, w, w_in, w_bias, None, sparsity, input_set, w_sparsity,
#                                       nonlin_func)
#         # end if
#     # end __init__
# 
#     ###############################################
#     # PROPERTIES
#     ###############################################
# 
#     # Hidden weight matrix
#     @property
#     def w(self):
#         """
#         Hidden weight matrix
#         :return:
#         """
#         return self.esn_cell.w
#     # end w
# 
#     # Input matrix
#     @property
#     def w_in(self):
#         """
#         Input matrix
#         :return:
#         """
#         return self.esn_cell.w_in
#     # end w_in
# 
#     ###############################################
#     # PUBLIC
#     ###############################################
# 
#     # Reset learning
#     def reset(self):
#         """
#         Reset learning
#         :return:
#         """
#         # Reset output layer
#         self.output.reset()
# 
#         # Training mode again
#         self.train(True)
#     # end reset
# 
#     # Output matrix
#     def get_w_out(self):
#         """
#         Output matrix
#         :return:
#         """
#         return self.output.w_out
#     # end get_w_out
# 
#     # Set W
#     def set_w(self, w):
#         """
#         Set W
#         :param w:
#         :return:
#         """
#         self.esn_cell.w = w
#     # end set_w
# 
#     # Forward
#     def forward(self, u, y=None):
#         """
#         Forward
#         :param u: Input signal.
#         :param y: Target outputs
#         :return: Output or hidden states
#         """
#         # Forward compute hidden states
#         forward_hidden_states = self.esn_cell(u)
# 
#         # Backward compute hidden states
#         backward_hidden_states = self.esn_cell(Variable(torch.from_numpy(np.flip(u.data.numpy(), 1).copy())))
#         backward_hidden_states = Variable(torch.from_numpy(np.flip(backward_hidden_states.data.numpy(), 1).copy()))
# 
#         return torch.cat((forward_hidden_states, backward_hidden_states), dim=2)
#     # end forward
# 
#     # Finish training
#     def finalize(self):
#         """
#         Finalize training with LU factorization
#         """
#         # Finalize output training
#         self.output.finalize()
# 
#         # Not in training mode anymore
#         self.train(False)
#     # end finalize
# 
#     # Reset hidden layer
#     def reset_hidden(self):
#         """
#         Reset hidden layer
#         :return:
#         """
#         self.esn_cell.reset_hidden()
#     # end reset_hidden
# 
#     # Get W's spectral radius
#     def get_spectral_radius(self):
#         """
#         Get W's spectral radius
#         :return: W's spectral radius
#         """
#         return self.esn_cell.get_spectral_radius()
#     # end spectral_radius


# Path: echotorch/nn/BDESNPCA.py
import torch
import torch.nn as nn
import torch.nn.functional as F
import matplotlib.pyplot as plt
from .BDESNCell import BDESNCell
from sklearn.decomposition import IncrementalPCA
from torch.autograd import Variable

    # Constructor
    def __init__(self, input_dim, hidden_dim, output_dim, pca_dim, linear_dim, leaky_rate=1.0, spectral_radius=0.9, bias_scaling=0,
                 input_scaling=1.0, w=None, w_in=None, w_bias=None, sparsity=None, input_set=[1.0, -1.0],
                 w_sparsity=None, nonlin_func=torch.tanh, learning_algo='inv', ridge_param=0.0, create_cell=True,
                 pca_batch_size=10):
        """
        Constructor
        :param input_dim: Inputs dimension.
        :param hidden_dim: Hidden layer dimension
        :param output_dim: Reservoir size
        :param spectral_radius: Reservoir's spectral radius
        :param bias_scaling: Scaling of the bias, a constant input to each neuron (default: 0, no bias)
        :param input_scaling: Scaling of the input weight matrix, default 1.
        :param w: Internal weights matrix
        :param w_in: Input-reservoir weights matrix
        :param w_bias: Bias weights matrix
        :param sparsity:
        :param input_set:
        :param w_sparsity:
        :param nonlin_func: Reservoir's activation function (tanh, sig, relu)
        :param learning_algo: Which learning algorithm to use (inv, LU, grad)
        """
        super(BDESNPCA, self).__init__()

        # Properties
        self.output_dim = output_dim
        self.pca_dim = pca_dim

        # Recurrent layer
        if create_cell:

            self.esn_cell = BDESNCell(