<commit_before>from .model import Model
from .batchnorm import BatchNormalization, ScaleShift
from ...api import layerize
from .affine import Affine


class ResBlock(Model):
    name = 'resblock'
    @property
    def input_shape(self):
        return (self.nr_out,)

    @property
    def output_shape(self):
        return (self.nr_out,)

    def __init__(self, width, **kwargs):
        self.nr_out = width
        self.nr_in = width
        self.name = kwargs.get('name')
        layers = [
            BatchNormalization(name='%s-bn1' % self.name),
            ScaleShift(width, name='%s-scaleshift1' % self.name),
            layerize(_relu, name='%s-relu1-func' % self.name),
            Affine(width, width, name='%s-weight1' % self.name),
            BatchNormalization(name='%s-bn2' % self.name),
            ScaleShift(width, name='%s-scaleshift2' % self.name),
            layerize(_relu, name='%s-relu2-func' % self.name),
            Affine(width, width, name='%s-weight2' % self.name),
        ]
        Model.__init__(self, *layers, **kwargs)


def _relu(X, dropout=0.0):
    mask = X > 0
    def finish_update(gradient, *args, **kwargs):
        return gradient * mask
    return X * mask, finish_update


class ReLuResBN(Model):
    name = 'resblock'
    @property
    def input_shape(self):
        return (self.nr_out,)

    @property
    def output_shape(self):
        return (self.nr_out,)

    def __init__(self, width, **kwargs):
        self.nr_out = width
        subkw = dict(kwargs)
        if 'name' in subkw:<commit_msg>Update resnet to use relu ops<commit_after>from .model import Model
from .batchnorm import BatchNormalization, ScaleShift
from ...api import layerize
from .affine import Affine

import cytoolz as toolz


class ResBlock(Model):
    name = 'resblock'
    @property
    def input_shape(self):
        return (self.nr_out,)

    @property
    def output_shape(self):
        return (self.nr_out,)

    def __init__(self, width, **kwargs):
        self.nr_out = width
        self.nr_in = width
        self.name = kwargs.get('name')
        Model.__init__(self, **kwargs)
        self.layers = [
            BatchNormalization(name='%s-bn1' % self.name),
            ScaleShift(width, name='%s-scaleshift1' % self.name),
            layerize(_relu(self.ops)),
            Affine(width, width, name='%s-weight1' % self.name),
            BatchNormalization(name='%s-bn2' % self.name),
            ScaleShift(width, name='%s-scaleshift2' % self.name),
            layerize(_relu(self.ops)),
            Affine(width, width, name='%s-weight2' % self.name),
        ]

@toolz.curry
def _relu(ops, X, dropout=0.0):
    x_copy = ops.xp.ascontiguousarray(X, dtype='f')
    ops.relu(x_copy, inplace=True)
    mask = X > 0
    def finish_update(gradient, *args, **kwargs):
        gradient = ops.xp.ascontiguousarray(gradient, dtype='f')
        ops.backprop_relu(gradient, x_copy)
        return gradient
    X[:] = x_copy
    return X, finish_update


class ReLuResBN(Model):
    name = 'resblock'
    Block = ResBlock

    @property
    def input_shape(self):
        return (self.nr_out,)

    @property
    def output_shape(self):
        return (self.nr_out,)

    def __init__(self, width, **kwargs):
        self.nr_out = width
        subkw = dict(kwargs)
        if 'name' in subkw:
{'fraction_non_alphanumeric': 0.13033033033033034, 'fraction_numerical': 0.006006006006006006, 'mean_word_length': 2.5137130801687766, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2624198', 'n_tokens_mistral': 1130, 'n_tokens_neox': 1087, 'n_words': 241}