# Path: mozi/layers/template.py
# class Template(object):
#     """
#     DESCRIPTION:
#         The interface to be implemented by any layer.
#     """
#     def __init__(self):
#         '''
#         FIELDS:
#             self.params: any params from the layer that needs to be updated
#                          by backpropagation can be put inside self.params
#             self.updates: use for updating any shared variables
#         '''
#         self.params = []
#         self.updates = []
# 
#     def _test_fprop(self, state_below):
#         '''
#         DESCRIPTION:
#             This is called during validating/testing of the model.
#         PARAM:
#             state_below: the input to layer
#         '''
#         return self._train_fprop(state_below)
# 
#     def _train_fprop(self, state_below):
#         '''
#         DESCRIPTION:
#             This is called during every training batch whereby the output from the
#             model will be used to update the parameters during error backpropagation.
#         PARAM:
#             state_below: the input to layer
#         '''
#         raise NotImplementedError()
# 
# 
#     def _layer_stats(self, state_below, layer_output):
#         """
#         DESCRIPTION:
#             Layer stats is used for debugging the layer by allowing user to peek
#             at the weight values or the layer output or any parameters of interest
#             during training. By computing the values of parameter of interest,
#             for example T.max(self.W) and put in the return list, the training will
#             print the maximum of the weight in the layer after every epoch.
#         PARAM:
#             state_below: the input to layer
#             layer_output: the output from the layer
#         RETURN:
#             A list of tuples of [('name_a', var_a), ('name_b', var_b)] whereby var is scalar
#         """
#         return []
# 
# Path: mozi/utils/theano_utils.py
# def sharedX(value, dtype=floatX, name=None, borrow=False, **kwargs):
#     return theano.shared(np.asarray(value, dtype=dtype), name=name, borrow=borrow, **kwargs)


import numpy as np
import theano
import theano.tensor as T
from theano.sandbox.rng_mrg import MRG_RandomStreams
from mozi.layers.template import Template
from mozi.utils.theano_utils import sharedX

floatX = theano.config.floatX
theano_rand = MRG_RandomStreams()


class Sigmoid(Template):
    def _train_fprop(self, state_below):
        return T.nnet.sigmoid(state_below)


class RELU(Template):
    def _train_fprop(self, state_below):
        return state_below * (state_below > 0.)


class PRELU(Template):
    def __init__(self, dim, alpha=0.2):
        '''
        y = wx + b
        if y > 0 then z = y else z = alpha * y
        return z
        alpha: the gradient of the slope which is updated by backpropagation
        '''

# Path: mozi/layers/template.py
# class Template(object):
#     """
#     DESCRIPTION:
#         The interface to be implemented by any layer.
#     """
#     def __init__(self):
#         '''
#         FIELDS:
#             self.params: any params from the layer that needs to be updated
#                          by backpropagation can be put inside self.params
#             self.updates: use for updating any shared variables
#         '''
#         self.params = []
#         self.updates = []
# 
#     def _test_fprop(self, state_below):
#         '''
#         DESCRIPTION:
#             This is called during validating/testing of the model.
#         PARAM:
#             state_below: the input to layer
#         '''
#         return self._train_fprop(state_below)
# 
#     def _train_fprop(self, state_below):
#         '''
#         DESCRIPTION:
#             This is called during every training batch whereby the output from the
#             model will be used to update the parameters during error backpropagation.
#         PARAM:
#             state_below: the input to layer
#         '''
#         raise NotImplementedError()
# 
# 
#     def _layer_stats(self, state_below, layer_output):
#         """
#         DESCRIPTION:
#             Layer stats is used for debugging the layer by allowing user to peek
#             at the weight values or the layer output or any parameters of interest
#             during training. By computing the values of parameter of interest,
#             for example T.max(self.W) and put in the return list, the training will
#             print the maximum of the weight in the layer after every epoch.
#         PARAM:
#             state_below: the input to layer
#             layer_output: the output from the layer
#         RETURN:
#             A list of tuples of [('name_a', var_a), ('name_b', var_b)] whereby var is scalar
#         """
#         return []
# 
# Path: mozi/utils/theano_utils.py
# def sharedX(value, dtype=floatX, name=None, borrow=False, **kwargs):
#     return theano.shared(np.asarray(value, dtype=dtype), name=name, borrow=borrow, **kwargs)


# Path: mozi/layers/activation.py
import numpy as np
import theano
import theano.tensor as T
from theano.sandbox.rng_mrg import MRG_RandomStreams
from mozi.layers.template import Template
from mozi.utils.theano_utils import sharedX


floatX = theano.config.floatX
theano_rand = MRG_RandomStreams()


class Sigmoid(Template):
    def _train_fprop(self, state_below):
        return T.nnet.sigmoid(state_below)


class RELU(Template):
    def _train_fprop(self, state_below):
        return state_below * (state_below > 0.)


class PRELU(Template):
    def __init__(self, dim, alpha=0.2):
        '''
        y = wx + b
        if y > 0 then z = y else z = alpha * y
        return z
        alpha: the gradient of the slope which is updated by backpropagation
        '''

        self.alpha = sharedX(np.ones(dim) * alpha, name='PRELU_gradient')