<commit_before>import numpy
import theano
import theano.tensor as tt
import Mariana.settings as MSET
from Mariana.abstraction import Abstraction_ABC
import Mariana.initializations as MI

__all__= ["Decorator_ABC", "BatchNormalization", "Center", "Normalize", "Mask", "RandomMask", "BinomialDropout", "Clip", "AdditiveGaussianNoise", "MultiplicativeGaussianNoise"]

def iCast(thing) :
    if thing.dtype.find("int") > -1 :
        return tt.cast(thing, MSET.INTX)
    else :
        return tt.cast(thing, theano.config.floatX)

class Decorator_ABC(Abstraction_ABC) :
    """A decorator is a modifier that is applied on a layer. They are always the last the abstraction to be applied and they can transform a layer in anyway they want."""

    def __call__(self, **kwargs) :
        self.decorate(**kwargs)

    def apply(self, layer) :
        """Apply to a layer and update networks's log"""
        hyps = {}
        for k in self.hyperParameters :
            hyps[k] = getattr(self, k)

        message = "%s is decorated by %s" % (layer.name, self.__class__.__name__)
        layer.network.logLayerEvent(layer, message, hyps)
        return self.decorate(layer)

    def decorate(self, layer) :
        """The function that all decorator_ABCs must implement"""
        raise NotImplemented("This one should be implemented in child")

class Mask(Decorator_ABC):
    """Applies a fixed mask to the outputs of the layer. This layers does this:

        .. math::

            outputs = outputs * mask

    If you want to remove some parts of the output use 0s, if you want to keep them as they are use 1s.
    Anything else will lower or increase the values by the given factors.

    :param array/list mask: It should have the same dimensions as the layer's outputs.

    """
    def __init__(self, mask, onTrain=True, onTest=False):
        Decorator_ABC.__init__(self)
        self.mask = tt.cast(mask, theano.config.floatX)
        self.onTrain = onTrain
        self.onTest = onTest
        self.hyperParameters.extend(["onTrain", "onTest"])

    def decorate(self, layer) :
        if self.onTrain :
            layer.outputs["train"] = layer.outputs["train"] * self.mask
        
        if self.onTest :
            layer.outputs["test"] = layer.outputs["test"] * self.mask

class RandomMask(Decorator_ABC):
    """
    This decorator takes a list of masks and will randomly apply them to the outputs of the layer it decorates.
    Could be used as a fast approximation for dropout. 
    """
    def __init__(self, masks, onTrain=True, onTest=False, *args, **kwargs):
        Decorator_ABC.__init__(self)
        self.nbMasks = len(masks)
        self.onTest = onTest
        self.onTrain = onTrain

        if self.nbMasks > 0 :
            self.masks = theano.shared(numpy.asarray(masks, dtype=theano.config.floatX))
            self.hyperParameters.extend([])

    def decorate(self, layer) :
        if self.nbMasks > 0 :
            rnd = tt.shared_randomstreams.RandomStreams()
            maskId = rnd.random_integers(low=0, high=self.nbMasks-1, ndim=1)
            mask = self.masks[maskId]

            if self.onTrain :
                layer.outputs["train"] = layer.outputs["train"] * mask
    
            if self.onTest :
                layer.outputs["test"] = layer.outputs["test"] * mask

class BinomialDropout(Decorator_ABC):
    """Stochastically mask some parts of the output of the layer. Use it to make things such as denoising autoencoders and dropout layers"""
    def __init__(self, ratio, onTrain=True, onTest=False):
        Decorator_ABC.__init__(self)

        assert (ratio >= 0 and ratio <= 1)
        self.ratio = ratio
        self.seed = MSET.RANDOM_SEED
        
        self.onTrain = onTrain
        self.onTest = onTest
        self.hyperParameters.extend(["ratio", "onTrain", "onTest"])

    def _decorate(self, outputs) :
        if self.ratio <= 0 :
            return outputs
        rnd = tt.shared_randomstreams.RandomStreams()
        mask = rnd.binomial(n = 1, p = (1-self.ratio), size = outputs.shape)
        # cast to stay in GPU float limit
        mask = iCast(mask)
        return (outputs * mask)

    def decorate(self, layer) :
        if self.onTrain :
            layer.outputs["train"] = self._decorate(layer.outputs["train"])
        
        if self.onTest :
            layer.outputs["test"] = self._decorate(layer.outputs["test"])

class Center(Decorator_ABC) :
    """Centers the outputs by substracting the mean"""
    def __init__(self, onTrain=True, onTest=True):
        Decorator_ABC.__init__(self)
        self.onTrain = onTrain
        self.onTest = onTest
        self.hyperParameters.extend(["onTrain", "onTest"])

    def decorate(self, layer) :
        if self.onTrain :
            layer.outputs["train"] = layer.outputs["train"]-tt.mean(layer.outputs["train"])
        if self.onTest :
            layer.outputs["test"] = layer.outputs["test"]-tt.mean(layer.outputs["test"])

class Normalize(Decorator_ABC) :
    """
    Normalizes the outputs by substracting the mean and deviding by the standard deviation

    :param float espilon: Actually it is not the std that is used but the approximation: sqrt(Variance + epsilon). Use this parameter to set the epsilon value
    """

    def __init__(self, espilon=1e-6, onTrain=True, onTest=True) :
        Decorator_ABC.__init__(self)
        self.espilon = espilon
        self.onTrain = onTrain
        self.onTest = onTest
        self.hyperParameters.extend(["onTrain", "onTest", "epsilon"])

    def decorate(self, layer) :
        if self.onTrain :
            std = tt.sqrt( tt.var(layer.outputs["train"]s) + self.espilon )
            layer.outputs["train"] = ( layer.outputs["train"]-tt.mean(layer.output) / std )

        if self.onTest :
            std = tt.sqrt( tt.var(layer.outputs["test"]) + self.espilon )
            layer.outputs["test"] = ( layer.outputs["test"]-tt.mean(layer.outputs["test"]) ) / std

class BatchNormalization(Decorator_ABC):
    """Applies Batch Normalization to the outputs of the layer.
    Implementation according to <NAME> and <NAME> (http://arxiv.org/abs/1502.03167)
    
        .. math::

            W * ( inputs - mean(mu) )/( std(inputs) ) + b

        Where W and b are learned and std stands for the standard deviation. The mean and the std are computed accross the whole minibatch.

        :param float epsilon: Actually it is not the std that is used but the approximation: sqrt(Variance + epsilon). Use this parameter to set the epsilon value
        :param initialization WInitialization: How to initizalise the weights. This decorator is smart enough to use layer initializations.
        :param initialization bInitialization: Same for bias
    """

    def __init__(self, WInitialization=MI.SmallUniformWeights(), bInitialization=MI.ZeroBias(), epsilon=1e-6, onTrain=True, onTest=True) :
        Decorator_ABC.__init__(self)
        self.epsilon = epsilon
        self.WInitialization = WInitialization
        self.bInitialization = bInitialization
        self.W = None
        self.b = None
        self.paramShape = None<commit_msg>batch norm layer?<commit_after>import numpy
import theano
import theano.tensor as tt
import Mariana.settings as MSET
from Mariana.abstraction import Abstraction_ABC
import Mariana.initializations as MI

__all__= ["Decorator_ABC", "BatchNormalization", "Center", "Normalize", "Mask", "RandomMask", "BinomialDropout", "Clip", "AdditiveGaussianNoise", "MultiplicativeGaussianNoise"]

def iCast(thing) :
    if thing.dtype.find("int") > -1 :
        return tt.cast(thing, MSET.INTX)
    else :
        return tt.cast(thing, theano.config.floatX)

class Decorator_ABC(Abstraction_ABC) :
    """A decorator is a modifier that is applied on a layer. They are always the last the abstraction to be applied and they can transform a layer in anyway they want."""

    def __call__(self, **kwargs) :
        self.decorate(**kwargs)

    def apply(self, layer) :
        """Apply to a layer and update networks's log"""
        hyps = {}
        for k in self.hyperParameters :
            hyps[k] = getattr(self, k)

        message = "%s is decorated by %s" % (layer.name, self.__class__.__name__)
        layer.network.logLayerEvent(layer, message, hyps)
        return self.decorate(layer)

    def decorate(self, layer) :
        """The function that all decorator_ABCs must implement"""
        raise NotImplemented("This one should be implemented in child")

class Mask(Decorator_ABC):
    """Applies a fixed mask to the outputs of the layer. This layers does this:

        .. math::

            outputs = outputs * mask

    If you want to remove some parts of the output use 0s, if you want to keep them as they are use 1s.
    Anything else will lower or increase the values by the given factors.

    :param array/list mask: It should have the same dimensions as the layer's outputs.

    """
    def __init__(self, mask, onTrain=True, onTest=False):
        Decorator_ABC.__init__(self)
        self.mask = tt.cast(mask, theano.config.floatX)
        self.onTrain = onTrain
        self.onTest = onTest
        self.hyperParameters.extend(["onTrain", "onTest"])

    def decorate(self, layer) :
        if self.onTrain :
            layer.outputs["train"] = layer.outputs["train"] * self.mask
        
        if self.onTest :
            layer.outputs["test"] = layer.outputs["test"] * self.mask

class RandomMask(Decorator_ABC):
    """
    This decorator takes a list of masks and will randomly apply them to the outputs of the layer it decorates.
    Could be used as a fast approximation for dropout. 
    """
    def __init__(self, masks, onTrain=True, onTest=False, *args, **kwargs):
        Decorator_ABC.__init__(self)
        self.nbMasks = len(masks)
        self.onTest = onTest
        self.onTrain = onTrain

        if self.nbMasks > 0 :
            self.masks = theano.shared(numpy.asarray(masks, dtype=theano.config.floatX))
            self.hyperParameters.extend([])

    def decorate(self, layer) :
        if self.nbMasks > 0 :
            rnd = tt.shared_randomstreams.RandomStreams()
            maskId = rnd.random_integers(low=0, high=self.nbMasks-1, ndim=1)
            mask = self.masks[maskId]

            if self.onTrain :
                layer.outputs["train"] = layer.outputs["train"] * mask
    
            if self.onTest :
                layer.outputs["test"] = layer.outputs["test"] * mask

class BinomialDropout(Decorator_ABC):
    """Stochastically mask some parts of the output of the layer. Use it to make things such as denoising autoencoders and dropout layers"""
    def __init__(self, ratio, onTrain=True, onTest=False):
        Decorator_ABC.__init__(self)

        assert (ratio >= 0 and ratio <= 1)
        self.ratio = ratio
        self.seed = MSET.RANDOM_SEED
        
        self.onTrain = onTrain
        self.onTest = onTest
        self.hyperParameters.extend(["ratio", "onTrain", "onTest"])

    def _decorate(self, outputs) :
        if self.ratio <= 0 :
            return outputs
        rnd = tt.shared_randomstreams.RandomStreams()
        mask = rnd.binomial(n = 1, p = (1-self.ratio), size = outputs.shape)
        # cast to stay in GPU float limit
        mask = iCast(mask)
        return (outputs * mask)

    def decorate(self, layer) :
        if self.onTrain :
            layer.outputs["train"] = self._decorate(layer.outputs["train"])
        
        if self.onTest :
            layer.outputs["test"] = self._decorate(layer.outputs["test"])

class Center(Decorator_ABC) :
    """Centers the outputs by substracting the mean"""
    def __init__(self, onTrain=True, onTest=True):
        Decorator_ABC.__init__(self)
        self.onTrain = onTrain
        self.onTest = onTest
        self.hyperParameters.extend(["onTrain", "onTest"])

    def decorate(self, layer) :
        if self.onTrain :
            layer.outputs["train"] = layer.outputs["train"]-tt.mean(layer.outputs["train"])
        if self.onTest :
            layer.outputs["test"] = layer.outputs["test"]-tt.mean(layer.outputs["test"])

class Normalize(Decorator_ABC) :
    """
    Normalizes the outputs by substracting the mean and deviding by the standard deviation

    :param float espilon: Actually it is not the std that is used but the approximation: sqrt(Variance + epsilon). Use this parameter to set the epsilon value
    """

    def __init__(self, espilon=1e-6, onTrain=True, onTest=True) :
        Decorator_ABC.__init__(self)
        self.espilon = espilon
        self.onTrain = onTrain
        self.onTest = onTest
        self.hyperParameters.extend(["onTrain", "onTest", "epsilon"])

    def decorate(self, layer) :
        if self.onTrain :
            std = tt.sqrt( tt.var(layer.outputs["train"]) + self.espilon )
            layer.outputs["train"] = ( layer.outputs["train"]-tt.mean(layer.output) / std )

        if self.onTest :
            std = tt.sqrt( tt.var(layer.outputs["test"]) + self.espilon )
            layer.outputs["test"] = ( layer.outputs["test"]-tt.mean(layer.outputs["test"]) ) / std

class BatchNormalization_deprectaded_to_become_layer(Decorator_ABC):
    """Applies Batch Normalization to the outputs of the layer.
    Implementation according to <NAME> and <NAME> (http://arxiv.org/abs/1502.03167)
    
        .. math::

            W * ( inputs - mean(mu) )/( std(inputs) ) + b

        Where W and b are learned and std stands for the standard deviation. The mean and the std are computed accross the whole minibatch.

        :param float epsilon: Actually it is not the std that is used but the approximation: sqrt(Variance + epsilon). Use this parameter to set the epsilon value
        :param initialization WInitialization: How to initizalise the weights. This decorator is smart enough to use layer initializations.
        :param initialization bInitialization: Same for bias
    """

    def __init__(self, WInitialization=MI.SmallUniformWeights(), bInitialization=MI.ZeroBias(), epsilon=1e-6, onTrain=True, onTest=True) :
        Decorator_ABC.__init__(self)
        self.epsilon = epsilon
        self.WInitialization = WInitialization
        self.bInitialization = bInitialization
        self.W = None
        self.b = None
        self.paramShape = None
{'fraction_non_alphanumeric': 0.10706533155941461, 'fraction_numerical': 0.0036412015965268537, 'mean_word_length': 2.716367421285454, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3266727', 'n_tokens_mistral': 4350, 'n_tokens_neox': 4150, 'n_words': 1225}