# Path: mozi/utils/theano_utils.py
# def shared_zeros(shape, dtype=floatX, name=None, **kwargs):
#     return sharedX(np.zeros(shape), dtype=dtype, name=name, **kwargs)
# 
# Path: mozi/weight_init.py
# class GaussianWeight(WeightInitialization):
#     def __init__(self, mean=0, std=0.1):
#         self.mean = mean
#         self.std = std
# 
#     def __call__(self, dim, name='W', **kwargs):
#         W_values = np.random.normal(loc=self.mean, scale=self.std, size=dim)
#         return sharedX(name=name, value=W_values, borrow=True, **kwargs)
# 
# Path: mozi/layers/template.py
# class Template(object):
#     """
#     DESCRIPTION:
#         The interface to be implemented by any layer.
#     """
#     def __init__(self):
#         '''
#         FIELDS:
#             self.params: any params from the layer that needs to be updated
#                          by backpropagation can be put inside self.params
#             self.updates: use for updating any shared variables
#         '''
#         self.params = []
#         self.updates = []
# 
#     def _test_fprop(self, state_below):
#         '''
#         DESCRIPTION:
#             This is called during validating/testing of the model.
#         PARAM:
#             state_below: the input to layer
#         '''
#         return self._train_fprop(state_below)
# 
#     def _train_fprop(self, state_below):
#         '''
#         DESCRIPTION:
#             This is called during every training batch whereby the output from the
#             model will be used to update the parameters during error backpropagation.
#         PARAM:
#             state_below: the input to layer
#         '''
#         raise NotImplementedError()
# 
# 
#     def _layer_stats(self, state_below, layer_output):
#         """
#         DESCRIPTION:
#             Layer stats is used for debugging the layer by allowing user to peek
#             at the weight values or the layer output or any parameters of interest
#             during training. By computing the values of parameter of interest,
#             for example T.max(self.W) and put in the return list, the training will
#             print the maximum of the weight in the layer after every epoch.
#         PARAM:
#             state_below: the input to layer
#             layer_output: the output from the layer
#         RETURN:
#             A list of tuples of [('name_a', var_a), ('name_b', var_b)] whereby var is scalar
#         """
#         return []


import numpy as np
import theano
import theano.tensor as T
from theano.sandbox.rng_mrg import MRG_RandomStreams
from mozi.utils.theano_utils import shared_zeros
from mozi.weight_init import GaussianWeight
from mozi.layers.template import Template

floatX = theano.config.floatX
theano_rand = MRG_RandomStreams()


class Linear(Template):

    def __init__(self, prev_dim=None, this_dim=None, W=None, b=None,
                 weight_init=GaussianWeight(mean=0, std=0.1)):
        """
        DESCRIPTION:
            This is a fully connected layer
        PARAM:
            prev_dim(int): dimension of previous layer
            this_dim(int): dimension of this layer
            name(string): name of the layer
            W(tensor variable): Weight of 2D tensor matrix
            b(tensor variable): bias of 2D tensor matrix
            params(list): a list of params in layer that can be updated
        """

        self.prev_dim = prev_dim
        self.this_dim = this_dim

        self.W = W
        if self.W is None:
            self.W = weight_init((prev_dim, this_dim), name='W')

        self.b = b
        if self.b is None:

# Path: mozi/utils/theano_utils.py
# def shared_zeros(shape, dtype=floatX, name=None, **kwargs):
#     return sharedX(np.zeros(shape), dtype=dtype, name=name, **kwargs)
# 
# Path: mozi/weight_init.py
# class GaussianWeight(WeightInitialization):
#     def __init__(self, mean=0, std=0.1):
#         self.mean = mean
#         self.std = std
# 
#     def __call__(self, dim, name='W', **kwargs):
#         W_values = np.random.normal(loc=self.mean, scale=self.std, size=dim)
#         return sharedX(name=name, value=W_values, borrow=True, **kwargs)
# 
# Path: mozi/layers/template.py
# class Template(object):
#     """
#     DESCRIPTION:
#         The interface to be implemented by any layer.
#     """
#     def __init__(self):
#         '''
#         FIELDS:
#             self.params: any params from the layer that needs to be updated
#                          by backpropagation can be put inside self.params
#             self.updates: use for updating any shared variables
#         '''
#         self.params = []
#         self.updates = []
# 
#     def _test_fprop(self, state_below):
#         '''
#         DESCRIPTION:
#             This is called during validating/testing of the model.
#         PARAM:
#             state_below: the input to layer
#         '''
#         return self._train_fprop(state_below)
# 
#     def _train_fprop(self, state_below):
#         '''
#         DESCRIPTION:
#             This is called during every training batch whereby the output from the
#             model will be used to update the parameters during error backpropagation.
#         PARAM:
#             state_below: the input to layer
#         '''
#         raise NotImplementedError()
# 
# 
#     def _layer_stats(self, state_below, layer_output):
#         """
#         DESCRIPTION:
#             Layer stats is used for debugging the layer by allowing user to peek
#             at the weight values or the layer output or any parameters of interest
#             during training. By computing the values of parameter of interest,
#             for example T.max(self.W) and put in the return list, the training will
#             print the maximum of the weight in the layer after every epoch.
#         PARAM:
#             state_below: the input to layer
#             layer_output: the output from the layer
#         RETURN:
#             A list of tuples of [('name_a', var_a), ('name_b', var_b)] whereby var is scalar
#         """
#         return []


# Path: mozi/layers/linear.py
import numpy as np
import theano
import theano.tensor as T
from theano.sandbox.rng_mrg import MRG_RandomStreams
from mozi.utils.theano_utils import shared_zeros
from mozi.weight_init import GaussianWeight
from mozi.layers.template import Template


floatX = theano.config.floatX
theano_rand = MRG_RandomStreams()


class Linear(Template):

    def __init__(self, prev_dim=None, this_dim=None, W=None, b=None,
                 weight_init=GaussianWeight(mean=0, std=0.1)):
        """
        DESCRIPTION:
            This is a fully connected layer
        PARAM:
            prev_dim(int): dimension of previous layer
            this_dim(int): dimension of this layer
            name(string): name of the layer
            W(tensor variable): Weight of 2D tensor matrix
            b(tensor variable): bias of 2D tensor matrix
            params(list): a list of params in layer that can be updated
        """

        self.prev_dim = prev_dim
        self.this_dim = this_dim

        self.W = W
        if self.W is None:
            self.W = weight_init((prev_dim, this_dim), name='W')

        self.b = b
        if self.b is None:

            self.b = shared_zeros(shape=this_dim, name='b')