<commit_before>from __future__ import absolute_import
import theano.tensor as T


class Regularizer(object):

    def update_gradient(self, gradient, params):
        raise NotImplementedError

    def update_loss(self, loss):
        raise NotImplementedError


class WeightsL1(Regularizer):
    def __init__(self, l=0.01):
        self.l = l

    def update_gradient(self, gradient, params):
        gradient += T.sgn(params) * self.l
        return gradient


class WeightsL2(Regularizer):
    def __init__(self, l=0.01):
        self.l = l

    def update_gradient(self, gradient, params):
        gradient += params * self.l
        return gradient


class WeightsL1L2(Regularizer):
    def __init__(self, l1=0.01, l2=0.01):
        self.l1 = l1
        self.l2 = l2

    def update_gradient(self, gradient, params):
        gradient += params * self.l2
        gradient += T.sgn(params) * self.l1
        return gradient


class Identity(Regularizer):

    def update_gradient(self, gradient, params):
        return gradient


class ActivityL1(Regularizer):
    def __init__(self, l = 0.01):
        self.l = l
        self.layer = None

    def set_layer(self, layer):
        self.layer = layer

    def update_loss(self, loss):
        return loss + self.l * T.sum(T.mean(abs(self.layer.get_output(True)), axis=0))


class ActivityL2(Regularizer):
    def __init__(self, l = 0.01):
        self.l = l
        self.layer = None

    def set_layer(self, layer):
        self.layer = layer

    def update_loss(self, loss):
        return loss + self.l * T.sum(T.mean(self.layer.get_output(True) ** 2, axis=0))


#old style variables for backwards compatibility
l1 = weights_l1 = WeightsL1
l2 = weights_l2 = WeightsL2
l1l2 = weights_l1l2 = WeightsL1L2
identity = Identity()

activity_l1 = ActivityL1
activity_l2 = ActivityL2
<commit_msg>Simplify regularizers<commit_after>from __future__ import absolute_import
import theano.tensor as T

class Regularizer(object):
    def update_gradient(self, gradient, params):
        return gradient

    def update_loss(self, loss):
        return loss


class WeightsL1(Regularizer):
    def __init__(self, l=0.01):
        self.l = l

    def update_gradient(self, gradient, params):
        gradient += T.sgn(params) * self.l
        return gradient


class WeightsL2(Regularizer):
    def __init__(self, l=0.01):
        self.l = l

    def update_gradient(self, gradient, params):
        gradient += params * self.l
        return gradient


class WeightsL1L2(Regularizer):
    def __init__(self, l1=0.01, l2=0.01):
        self.l1 = l1
        self.l2 = l2

    def update_gradient(self, gradient, params):
        gradient += params * self.l2
        gradient += T.sgn(params) * self.l1
        return gradient


class ActivityL1(Regularizer):
    def __init__(self, l=0.01):
        self.l = l
        self.layer = None

    def set_layer(self, layer):
        self.layer = layer

    def update_loss(self, loss):
        return loss + self.l * T.sum(T.mean(abs(self.layer.get_output(True)), axis=0))


class ActivityL2(Regularizer):
    def __init__(self, l=0.01):
        self.l = l
        self.layer = None

    def set_layer(self, layer):
        self.layer = layer

    def update_loss(self, loss):
        return loss + self.l * T.sum(T.mean(self.layer.get_output(True) ** 2, axis=0))


identity = Regularizer

l1 = weights_l1 = WeightsL1
l2 = weights_l2 = WeightsL2
l1l2 = weights_l1l2 = WeightsL1L2

activity_l1 = ActivityL1
activity_l2 = ActivityL2

{'fraction_non_alphanumeric': 0.10557768924302789, 'fraction_numerical': 0.02902675014228799, 'mean_word_length': 2.7795698924731185, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6052779', 'n_tokens_mistral': 1221, 'n_tokens_neox': 1150, 'n_words': 287}