# Path: mozi/layers/template.py
# class Template(object):
#     """
#     DESCRIPTION:
#         The interface to be implemented by any layer.
#     """
#     def __init__(self):
#         '''
#         FIELDS:
#             self.params: any params from the layer that needs to be updated
#                          by backpropagation can be put inside self.params
#             self.updates: use for updating any shared variables
#         '''
#         self.params = []
#         self.updates = []
# 
#     def _test_fprop(self, state_below):
#         '''
#         DESCRIPTION:
#             This is called during validating/testing of the model.
#         PARAM:
#             state_below: the input to layer
#         '''
#         return self._train_fprop(state_below)
# 
#     def _train_fprop(self, state_below):
#         '''
#         DESCRIPTION:
#             This is called during every training batch whereby the output from the
#             model will be used to update the parameters during error backpropagation.
#         PARAM:
#             state_below: the input to layer
#         '''
#         raise NotImplementedError()
# 
# 
#     def _layer_stats(self, state_below, layer_output):
#         """
#         DESCRIPTION:
#             Layer stats is used for debugging the layer by allowing user to peek
#             at the weight values or the layer output or any parameters of interest
#             during training. By computing the values of parameter of interest,
#             for example T.max(self.W) and put in the return list, the training will
#             print the maximum of the weight in the layer after every epoch.
#         PARAM:
#             state_below: the input to layer
#             layer_output: the output from the layer
#         RETURN:
#             A list of tuples of [('name_a', var_a), ('name_b', var_b)] whereby var is scalar
#         """
#         return []
# 
# Path: mozi/utils/theano_utils.py
# def shared_zeros(shape, dtype=floatX, name=None, **kwargs):
#     return sharedX(np.zeros(shape), dtype=dtype, name=name, **kwargs)
# 
# def sharedX(value, dtype=floatX, name=None, borrow=False, **kwargs):
#     return theano.shared(np.asarray(value, dtype=dtype), name=name, borrow=borrow, **kwargs)
# 
# def shared_ones(shape, dtype=floatX, name=None, **kwargs):
#     return sharedX(np.ones(shape), dtype=dtype, name=name, **kwargs)
# 
# Path: mozi/weight_init.py
# class UniformWeight(WeightInitialization):
#     def __init__(self, scale=0.05):
#         self.scale = scale
# 
#     def __call__(self, dim, name='W', **kwargs):
#         W_values = np.random.uniform(low=-self.scale, high=self.scale, size=dim)
#         return sharedX(name=name, value=W_values, borrow=True, **kwargs)


from mozi.layers.template import Template
from mozi.utils.theano_utils import shared_zeros, sharedX, shared_ones
from mozi.weight_init import UniformWeight
import theano.tensor as T
import theano
    def __init__(self, dim, layer_type, gamma_init=UniformWeight(), short_memory=0.01):
        '''
        REFERENCE:
            Batch Normalization: Accelerating Deep Network Training by Reducing Internal Covariate Shift
        PARAMS:
            short_memory: short term memory
                y_t is the latest value, the moving average x_tp1 is calculated as
                x_tp1 = memory * y_t + (1-memory) * x_t, the larger the short term
                memory, the more weight is put on contempory.
            layer_type: fc or conv
            epsilon:
                denominator min value for preventing division by zero in computing std
            dim: for fc layers, shape is the layer dimension, for conv layers,
                shape is the number of feature maps
        '''

        assert layer_type in ['fc', 'conv']
        self.layer_type = layer_type
        self.epsilon = 1e-6
        self.dim = dim
        self.mem = short_memory

        if self.layer_type == 'fc':
            input_shape = (1, dim)
            self.broadcastable = (True, False)
        elif self.layer_type == 'conv':
            input_shape = (1, dim, 1, 1)
            self.broadcastable = (True, False, True, True)

        self.gamma = gamma_init(input_shape, name='gamma')

# Path: mozi/layers/template.py
# class Template(object):
#     """
#     DESCRIPTION:
#         The interface to be implemented by any layer.
#     """
#     def __init__(self):
#         '''
#         FIELDS:
#             self.params: any params from the layer that needs to be updated
#                          by backpropagation can be put inside self.params
#             self.updates: use for updating any shared variables
#         '''
#         self.params = []
#         self.updates = []
# 
#     def _test_fprop(self, state_below):
#         '''
#         DESCRIPTION:
#             This is called during validating/testing of the model.
#         PARAM:
#             state_below: the input to layer
#         '''
#         return self._train_fprop(state_below)
# 
#     def _train_fprop(self, state_below):
#         '''
#         DESCRIPTION:
#             This is called during every training batch whereby the output from the
#             model will be used to update the parameters during error backpropagation.
#         PARAM:
#             state_below: the input to layer
#         '''
#         raise NotImplementedError()
# 
# 
#     def _layer_stats(self, state_below, layer_output):
#         """
#         DESCRIPTION:
#             Layer stats is used for debugging the layer by allowing user to peek
#             at the weight values or the layer output or any parameters of interest
#             during training. By computing the values of parameter of interest,
#             for example T.max(self.W) and put in the return list, the training will
#             print the maximum of the weight in the layer after every epoch.
#         PARAM:
#             state_below: the input to layer
#             layer_output: the output from the layer
#         RETURN:
#             A list of tuples of [('name_a', var_a), ('name_b', var_b)] whereby var is scalar
#         """
#         return []
# 
# Path: mozi/utils/theano_utils.py
# def shared_zeros(shape, dtype=floatX, name=None, **kwargs):
#     return sharedX(np.zeros(shape), dtype=dtype, name=name, **kwargs)
# 
# def sharedX(value, dtype=floatX, name=None, borrow=False, **kwargs):
#     return theano.shared(np.asarray(value, dtype=dtype), name=name, borrow=borrow, **kwargs)
# 
# def shared_ones(shape, dtype=floatX, name=None, **kwargs):
#     return sharedX(np.ones(shape), dtype=dtype, name=name, **kwargs)
# 
# Path: mozi/weight_init.py
# class UniformWeight(WeightInitialization):
#     def __init__(self, scale=0.05):
#         self.scale = scale
# 
#     def __call__(self, dim, name='W', **kwargs):
#         W_values = np.random.uniform(low=-self.scale, high=self.scale, size=dim)
#         return sharedX(name=name, value=W_values, borrow=True, **kwargs)


# Path: mozi/layers/normalization.py
from mozi.layers.template import Template
from mozi.utils.theano_utils import shared_zeros, sharedX, shared_ones
from mozi.weight_init import UniformWeight
import theano.tensor as T
import theano

    def __init__(self, dim, layer_type, gamma_init=UniformWeight(), short_memory=0.01):
        '''
        REFERENCE:
            Batch Normalization: Accelerating Deep Network Training by Reducing Internal Covariate Shift
        PARAMS:
            short_memory: short term memory
                y_t is the latest value, the moving average x_tp1 is calculated as
                x_tp1 = memory * y_t + (1-memory) * x_t, the larger the short term
                memory, the more weight is put on contempory.
            layer_type: fc or conv
            epsilon:
                denominator min value for preventing division by zero in computing std
            dim: for fc layers, shape is the layer dimension, for conv layers,
                shape is the number of feature maps
        '''

        assert layer_type in ['fc', 'conv']
        self.layer_type = layer_type
        self.epsilon = 1e-6
        self.dim = dim
        self.mem = short_memory

        if self.layer_type == 'fc':
            input_shape = (1, dim)
            self.broadcastable = (True, False)
        elif self.layer_type == 'conv':
            input_shape = (1, dim, 1, 1)
            self.broadcastable = (True, False, True, True)

        self.gamma = gamma_init(input_shape, name='gamma')

        self.beta = shared_zeros(input_shape, name='beta')