<commit_before>import numpy

from chainer import cuda
from chainer import function
from chainer.functions import sigmoid


class SigmoidCrossEntropy(function.Function):

    """Sigmoid activation followed by a sigmoid cross entropy loss."""

    def __init__(self, use_cudnn=True):
        self.use_cudnn = use_cudnn

    def forward_cpu(self, inputs):
        x, t = inputs
        assert x.shape == t.shape
        self.y, = sigmoid.Sigmoid().forward_cpu((x,))
        # stable computation of the cross entropy.
        loss = -numpy.sum(
            x * (t - (x >= 0)) - numpy.log(1 + numpy.exp(-numpy.abs(x))))
        return numpy.array((loss / t.shape[0],), dtype=numpy.float32),

    def forward_gpu(self, inputs):
        x, t = inputs
        self.y, = sigmoid.Sigmoid(self.use_cudnn).forward_gpu((x,))
        loss = -cuda.reduce(
            'int* t, float* x',
            'x[i] * (t[i] - (x[i] >= 0))'
            ' - log(1 + exp(x[i] - 2 * x[i] * (x[i] >= 0)))',
            'a+b', '0', 'sigmoid_crossent_fwd', numpy.float32)(t, x)
        return loss / t.shape[0],

    def backward_cpu(self, inputs, grad_outputs):
        t, gloss = inputs[1], grad_outputs[0]
        gx = gloss * (self.y - t) / t.shape[0]
        return gx, None

    def backward_gpu(self, inputs, grad_outputs):
        t, gloss = inputs[1], grad_outputs[0]
        gx = cuda.empty_like(self.y)
        coeff = gloss / t.shape[0]
        cuda.elementwise(
            'float* gx, const float* y, const int* t, const float* coeff',
            'gx[i] = *coeff * (y[i] - t[i])',
            'sigmoid_crossent_bwd')(gx, self.y, t, coeff)
        return gx, None

<commit_msg>Use log1p() instead of log()<commit_after>import numpy

from chainer import cuda
from chainer import function
from chainer.functions import sigmoid


class SigmoidCrossEntropy(function.Function):

    """Sigmoid activation followed by a sigmoid cross entropy loss."""

    def __init__(self, use_cudnn=True):
        self.use_cudnn = use_cudnn

    def forward_cpu(self, inputs):
        x, t = inputs
        assert x.shape == t.shape
        self.y, = sigmoid.Sigmoid().forward_cpu((x,))
        # stable computation of the cross entropy.
        loss = -numpy.sum(
            x * (t - (x >= 0)) - numpy.log1p(numpy.exp(-numpy.abs(x))))
        return numpy.array((loss / t.shape[0],), dtype=numpy.float32),

    def forward_gpu(self, inputs):
        x, t = inputs
        self.y, = sigmoid.Sigmoid(self.use_cudnn).forward_gpu((x,))
        loss = -cuda.reduce(
            'int* t, float* x',
            'x[i] * (t[i] - (x[i] >= 0))'
            ' - log1pf(expf(x[i] - 2 * x[i] * (x[i] >= 0)))',
            'a+b', '0', 'sigmoid_crossent_fwd', numpy.float32)(t, x)
        return loss / t.shape[0],

    def backward_cpu(self, inputs, grad_outputs):
        t, gloss = inputs[1], grad_outputs[0]
        gx = gloss * (self.y - t) / t.shape[0]
        return gx, None

    def backward_gpu(self, inputs, grad_outputs):
        t, gloss = inputs[1], grad_outputs[0]
        gx = cuda.empty_like(self.y)
        coeff = gloss / t.shape[0]
        cuda.elementwise(
            'float* gx, const float* y, const int* t, const float* coeff',
            'gx[i] = *coeff * (y[i] - t[i])',
            'sigmoid_crossent_bwd')(gx, self.y, t, coeff)
        return gx, None


{'fraction_non_alphanumeric': 0.15310717502251575, 'fraction_numerical': 0.011708195737015911, 'mean_word_length': 2.2763028515240906, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '812452', 'n_tokens_mistral': 1227, 'n_tokens_neox': 1195, 'n_words': 288}