# Path: seya/regularizers.py
# class GaussianKL(Regularizer):
#     """ KL-divergence between two gaussians.
#     Useful for Variational AutoEncoders.
#     Use this as an activation regularizer
# 
#     Parameters:
#     -----------
#     mean, logsigma: parameters of the input distributions
#     prior_mean, prior_logsigma: paramaters of the desired distribution (note the
#     log on logsigma)
#     regularizer_scale: Rescales the regularization cost. Keep this 1 for most cases.
# 
#     Notes:
#     ------
#     See seya.layers.variational.VariationalDense for usage example
# 
#     """
#     def __init__(self, mean, logsigma, prior_mean=0, prior_logsigma=0,
#                  regularizer_scale=1):
#         self.regularizer_scale = regularizer_scale
#         self.mean = mean
#         self.logsigma = logsigma
#         self.prior_mean = prior_mean
#         self.prior_logsigma = prior_logsigma
#         super(GaussianKL, self).__init__()
# 
#     def __call__(self, loss):
#         # See Variational Auto-Encoding Bayes by Kingma and Welling.
#         mean, logsigma = self.mean, self.logsigma
#         kl = (self.prior_logsigma - logsigma +
#               0.5 * (-1 + K.exp(2 * logsigma) + (mean - self.prior_mean) ** 2) /
#               K.exp(2 * self.prior_logsigma))
#         loss += K.mean(kl) * self.regularizer_scale
#         return loss
# 
#     def get_config(self):
#         return {"name": self.__class__.__name__}


from keras import backend as K
from keras.layers.core import Layer
from keras import initializations, activations
from seya.regularizers import GaussianKL
        self.regularizer_scale = regularizer_scale
        self.batch_size = batch_size
        self.init = initializations.get(init)
        self.activation = activations.get(activation)
        self.output_dim = output_dim
        self.initial_weights = weights
        self.input_dim = input_dim
        if self.input_dim:
            kwargs['input_shape'] = (self.input_dim,)
        self.input = K.placeholder(ndim=2)
        super(VariationalDense, self).__init__(**kwargs)

    def build(self):
        input_dim = self.input_shape[-1]

        self.W_mean = self.init((input_dim, self.output_dim))
        self.b_mean = K.zeros((self.output_dim,))
        self.W_logsigma = self.init((input_dim, self.output_dim))
        self.b_logsigma = K.zeros((self.output_dim,))

        self.trainable_weights = [self.W_mean, self.b_mean, self.W_logsigma,
                       self.b_logsigma]

        self.regularizers = []
        reg = self.get_variational_regularization(self.get_input())
        self.regularizers.append(reg)

    def get_variational_regularization(self, X):
        mean = self.activation(K.dot(X, self.W_mean) + self.b_mean)
        logsigma = self.activation(K.dot(X, self.W_logsigma) + self.b_logsigma)

# Path: seya/regularizers.py
# class GaussianKL(Regularizer):
#     """ KL-divergence between two gaussians.
#     Useful for Variational AutoEncoders.
#     Use this as an activation regularizer
# 
#     Parameters:
#     -----------
#     mean, logsigma: parameters of the input distributions
#     prior_mean, prior_logsigma: paramaters of the desired distribution (note the
#     log on logsigma)
#     regularizer_scale: Rescales the regularization cost. Keep this 1 for most cases.
# 
#     Notes:
#     ------
#     See seya.layers.variational.VariationalDense for usage example
# 
#     """
#     def __init__(self, mean, logsigma, prior_mean=0, prior_logsigma=0,
#                  regularizer_scale=1):
#         self.regularizer_scale = regularizer_scale
#         self.mean = mean
#         self.logsigma = logsigma
#         self.prior_mean = prior_mean
#         self.prior_logsigma = prior_logsigma
#         super(GaussianKL, self).__init__()
# 
#     def __call__(self, loss):
#         # See Variational Auto-Encoding Bayes by Kingma and Welling.
#         mean, logsigma = self.mean, self.logsigma
#         kl = (self.prior_logsigma - logsigma +
#               0.5 * (-1 + K.exp(2 * logsigma) + (mean - self.prior_mean) ** 2) /
#               K.exp(2 * self.prior_logsigma))
#         loss += K.mean(kl) * self.regularizer_scale
#         return loss
# 
#     def get_config(self):
#         return {"name": self.__class__.__name__}


# Path: seya/layers/variational.py
from keras import backend as K
from keras.layers.core import Layer
from keras import initializations, activations
from seya.regularizers import GaussianKL

        self.regularizer_scale = regularizer_scale
        self.batch_size = batch_size
        self.init = initializations.get(init)
        self.activation = activations.get(activation)
        self.output_dim = output_dim
        self.initial_weights = weights
        self.input_dim = input_dim
        if self.input_dim:
            kwargs['input_shape'] = (self.input_dim,)
        self.input = K.placeholder(ndim=2)
        super(VariationalDense, self).__init__(**kwargs)

    def build(self):
        input_dim = self.input_shape[-1]

        self.W_mean = self.init((input_dim, self.output_dim))
        self.b_mean = K.zeros((self.output_dim,))
        self.W_logsigma = self.init((input_dim, self.output_dim))
        self.b_logsigma = K.zeros((self.output_dim,))

        self.trainable_weights = [self.W_mean, self.b_mean, self.W_logsigma,
                       self.b_logsigma]

        self.regularizers = []
        reg = self.get_variational_regularization(self.get_input())
        self.regularizers.append(reg)

    def get_variational_regularization(self, X):
        mean = self.activation(K.dot(X, self.W_mean) + self.b_mean)
        logsigma = self.activation(K.dot(X, self.W_logsigma) + self.b_logsigma)

        return GaussianKL(mean, logsigma,