<commit_before>from .affine import Affine


class ReLu(Affine):
    name = 'relu'
    
    def predict_batch(self, X):
        output = Affine.predict_batch(self, X)
        return self.ops.clip_low(output, 0, inplace=True)

    def begin_update(self, input_BI, dropout=0.0):
        output_BO, finish_affine = Affine.begin_update(self, input_BI)
        def finish_update(gradient, *args, **kwargs):
            return finish_affine(gradient * (output_BO > 0), *args, **kwargs)
        return output_BO, finish_update
<commit_msg>Describe params in relu<commit_after>from .affine import Affine


class ReLu(Affine):
    name = 'relu'
    @property
    def describe_params(self):
        '''
        Yields (name, shape, initializer) triples describing the weights directly
        owned by the layer.
        '''
        #def init(W, **kwargs):
        #    W += self.ops.he_normal_init(W.shape, W.shape[-1])
        init = self.ops.xavier_uniform_init
        yield 'W-%s' % self.name, (self.nr_out, self.nr_in), init
        yield 'b-%s' % self.name, (self.nr_out,), None
    
    def predict_batch(self, X):
        output = Affine.predict_batch(self, X)
        return self.ops.clip_low(output, 0, inplace=True)

    def begin_update(self, input_BI, dropout=0.0):
        output_BO, finish_affine = Affine.begin_update(self, input_BI)
        def finish_update(gradient, *args, **kwargs):
            return finish_affine(gradient * (output_BO > 0), *args, **kwargs)
        output_BO = self.ops.clip_low(output_BO, 0, inplace=True)
        return output_BO, finish_update

{'fraction_non_alphanumeric': 0.11977186311787072, 'fraction_numerical': 0.0063371356147021544, 'mean_word_length': 2.9181141439205955, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4913258', 'n_tokens_mistral': 534, 'n_tokens_neox': 521, 'n_words': 120}