# Path: deeppy/base.py
# class ParamMixin(object):
#     @property
#     def params(self):
#         """ List of Parameter objects. """
#         raise NotImplementedError()
# 
#     @params.setter
#     def params(self, params):
#         raise NotImplementedError()
# 
# Path: deeppy/parameter.py
# class Parameter(PickleMixin):
#     def __init__(self, fill, name='', learn_rate=1.0, weight_decay=0.0,
#                  monitor=False):
#         self.filler = Filler.from_any(fill)
#         self.name = name
#         self.learn_rate = learn_rate
#         self._monitor = monitor
#         self.weight_decay = weight_decay
#         self._array = None
#         self._tmp_grad_array = None
#         self._tmp_step = None
#         self.shares = []
# 
#     @classmethod
#     def from_any(cls, arg):
#         if isinstance(arg, Parameter):
#             return arg
#         elif isinstance(arg, (int, float, np.ndarray, ca.ndarray, Filler)):
#             return cls(arg)
#         raise ValueError('Invalid parameter arguments')
# 
#     def setup(self, shape):
#         if self._array is None:
#             self._array = self.filler.array(shape)
#         else:
#             if isinstance(shape, int):
#                 shape = (shape,)
#             if self._array.shape != shape:
#                 raise ValueError('Shape %s does not match existing shape %s' %
#                                  (shape, self._array.shape))
# 
#     @property
#     def array(self):
#         return self._array
# 
#     @property
#     def grad_array(self):
#         ''' Returns the gradient array. '''
#         if self._tmp_grad_array is None:
#             if self.array is None:
#                 raise ValueError('Parameter not setup')
#             self._tmp_grad_array = ca.zeros_like(self.array)
#         return self._tmp_grad_array
# 
#     def grad(self):
#         ''' Returns a parameter step calculated from the gradient.
#         This differs from grad_array() as the parameter may be shared such
#         that its gradient has multiple sources. '''
#         grad = self.grad_array
#         for param in self.shares:
#             grad += param.grad_array
#         grad = self._add_penalty(grad)
#         return grad
# 
#     def step(self, step):
#         ''' Update the parameter values according to the given step. '''
#         if self._monitor:
#             self._tmp_step = step
#         self._array += step
# 
#     def _add_penalty(self, grad):
#         if self.weight_decay == 0.0:
#             return grad
#         else:
#             grad += 2*self.weight_decay * self._array
#             return grad
# 
#     def monitor(self):
#         if not self._monitor:
#             return
#         val_mean_abs = np.array(ca.mean(ca.fabs(self._array)))
#         grad_mean_abs = np.array(ca.mean(ca.fabs(self._tmp_grad_array)))
#         step_mean_abs = np.array(ca.mean(ca.fabs(self._tmp_step)))
#         log.info('%s:\t%.1e  [%.1e, %.1e]', self.name, val_mean_abs,
#                  grad_mean_abs, step_mean_abs)
# 
#     def share(self):
#         param = SharedParameter(self)
#         self.shares.append(param)
#         return param
# 
# Path: deeppy/feedforward/layers.py
# class Layer(PickleMixin):
#     bprop_to_x = True
# 
#     def setup(self, x_shape):
#         """ Setup layer with parameters that are unknown at __init__(). """
#         pass
# 
#     def fprop(self, x):
#         """ Calculate layer output for given input (forward propagation). """
#         raise NotImplementedError()
# 
#     def bprop(self, y_grad):
#         """ Calculate input gradient. """
#         raise NotImplementedError()
# 
#     def y_shape(self, x_shape):
#         """ Calculate shape of this layer's output.
#         x_shape[0] is the number of samples in the input.
#         x_shape[1:] is the shape of the feature.
#         """
#         raise NotImplementedError()


import cudarray as ca
from ..base import ParamMixin
from ..parameter import Parameter
from .layers import Layer
            return Softplus(*args)
        if method == 'tanh':
            return Tanh(*args)
        raise ValueError('Invalid input arguments')

    def y_shape(self, x_shape):
        return x_shape


class LeakyReLU(Activation):
    def __init__(self, a=0.25):
        self.a = a

    def fprop(self, x):
        self._tmp_x = x
        pos = ca.maximum(x, 0)
        neg = self.a * ca.minimum(x, 0)
        return pos + neg

    def bprop(self, y_grad):
        pos = ca.nnet.relu_d(self._tmp_x)
        neg = self.a * (self._tmp_x < 0)
        return (pos + neg) * y_grad


class ParametricReLU(Activation, ParamMixin):
    def __init__(self, a=0.25):
        self.a = Parameter.from_any(a)

    def setup(self, x_shape):

# Path: deeppy/base.py
# class ParamMixin(object):
#     @property
#     def params(self):
#         """ List of Parameter objects. """
#         raise NotImplementedError()
# 
#     @params.setter
#     def params(self, params):
#         raise NotImplementedError()
# 
# Path: deeppy/parameter.py
# class Parameter(PickleMixin):
#     def __init__(self, fill, name='', learn_rate=1.0, weight_decay=0.0,
#                  monitor=False):
#         self.filler = Filler.from_any(fill)
#         self.name = name
#         self.learn_rate = learn_rate
#         self._monitor = monitor
#         self.weight_decay = weight_decay
#         self._array = None
#         self._tmp_grad_array = None
#         self._tmp_step = None
#         self.shares = []
# 
#     @classmethod
#     def from_any(cls, arg):
#         if isinstance(arg, Parameter):
#             return arg
#         elif isinstance(arg, (int, float, np.ndarray, ca.ndarray, Filler)):
#             return cls(arg)
#         raise ValueError('Invalid parameter arguments')
# 
#     def setup(self, shape):
#         if self._array is None:
#             self._array = self.filler.array(shape)
#         else:
#             if isinstance(shape, int):
#                 shape = (shape,)
#             if self._array.shape != shape:
#                 raise ValueError('Shape %s does not match existing shape %s' %
#                                  (shape, self._array.shape))
# 
#     @property
#     def array(self):
#         return self._array
# 
#     @property
#     def grad_array(self):
#         ''' Returns the gradient array. '''
#         if self._tmp_grad_array is None:
#             if self.array is None:
#                 raise ValueError('Parameter not setup')
#             self._tmp_grad_array = ca.zeros_like(self.array)
#         return self._tmp_grad_array
# 
#     def grad(self):
#         ''' Returns a parameter step calculated from the gradient.
#         This differs from grad_array() as the parameter may be shared such
#         that its gradient has multiple sources. '''
#         grad = self.grad_array
#         for param in self.shares:
#             grad += param.grad_array
#         grad = self._add_penalty(grad)
#         return grad
# 
#     def step(self, step):
#         ''' Update the parameter values according to the given step. '''
#         if self._monitor:
#             self._tmp_step = step
#         self._array += step
# 
#     def _add_penalty(self, grad):
#         if self.weight_decay == 0.0:
#             return grad
#         else:
#             grad += 2*self.weight_decay * self._array
#             return grad
# 
#     def monitor(self):
#         if not self._monitor:
#             return
#         val_mean_abs = np.array(ca.mean(ca.fabs(self._array)))
#         grad_mean_abs = np.array(ca.mean(ca.fabs(self._tmp_grad_array)))
#         step_mean_abs = np.array(ca.mean(ca.fabs(self._tmp_step)))
#         log.info('%s:\t%.1e  [%.1e, %.1e]', self.name, val_mean_abs,
#                  grad_mean_abs, step_mean_abs)
# 
#     def share(self):
#         param = SharedParameter(self)
#         self.shares.append(param)
#         return param
# 
# Path: deeppy/feedforward/layers.py
# class Layer(PickleMixin):
#     bprop_to_x = True
# 
#     def setup(self, x_shape):
#         """ Setup layer with parameters that are unknown at __init__(). """
#         pass
# 
#     def fprop(self, x):
#         """ Calculate layer output for given input (forward propagation). """
#         raise NotImplementedError()
# 
#     def bprop(self, y_grad):
#         """ Calculate input gradient. """
#         raise NotImplementedError()
# 
#     def y_shape(self, x_shape):
#         """ Calculate shape of this layer's output.
#         x_shape[0] is the number of samples in the input.
#         x_shape[1:] is the shape of the feature.
#         """
#         raise NotImplementedError()


# Path: deeppy/feedforward/activation_layers.py
import cudarray as ca
from ..base import ParamMixin
from ..parameter import Parameter
from .layers import Layer

            return Softplus(*args)
        if method == 'tanh':
            return Tanh(*args)
        raise ValueError('Invalid input arguments')

    def y_shape(self, x_shape):
        return x_shape


class LeakyReLU(Activation):
    def __init__(self, a=0.25):
        self.a = a

    def fprop(self, x):
        self._tmp_x = x
        pos = ca.maximum(x, 0)
        neg = self.a * ca.minimum(x, 0)
        return pos + neg

    def bprop(self, y_grad):
        pos = ca.nnet.relu_d(self._tmp_x)
        neg = self.a * (self._tmp_x < 0)
        return (pos + neg) * y_grad


class ParametricReLU(Activation, ParamMixin):
    def __init__(self, a=0.25):
        self.a = Parameter.from_any(a)

    def setup(self, x_shape):

        self.a.setup((1, 1))