<commit_before>import itertools
import six
import unittest

import numpy

import chainer
from chainer import cuda
from chainer import functions
from chainer import gradient_check
from chainer import testing
from chainer.testing import attr


@testing.parameterize(*testing.product_dict(
    [
        {'lengths': [3, 3], 'batches': [2, 2, 2]},
        {'lengths': [3, 2, 1], 'batches': [3, 2, 1]},
        {'lengths': [3, 1, 1], 'batches': [3, 1, 1]},
        {'lengths': [1, 1], 'batches': [2]},
    ],
    [
        {'reduce': 'mean'},
        {'reduce': 'no'},
    ]
))
class TestCRF1d(unittest.TestCase):
    n_label = 3

    def setUp(self):
        self.cost = numpy.random.uniform(
            -1, 1, (self.n_label, self.n_label)).astype(numpy.float32)
        self.xs = [numpy.random.uniform(
            -1, 1, (b, 3)).astype(numpy.float32) for b in self.batches]
        self.ys = [
            numpy.random.randint(
                0, self.n_label, (b,)).astype(numpy.int32)
            for b in self.batches]
        self.g = numpy.random.uniform(
            -1, 1, (len(self.lengths))).astype(numpy.float32)

    def _calc_score(self, batch, ys):
        return sum(x[batch, y] for x, y in zip(self.xs, ys)) + \
            sum(self.cost[y1, y2] for y1, y2 in zip(ys[:-1], ys[1:]))

    def check_forward(self, cost_data, xs_data, ys_data):
        cost = chainer.Variable(cost_data)
        xs = [chainer.Variable(x) for x in xs_data]
        ys = [chainer.Variable(y) for y in ys_data]
        actual = functions.crf1d(cost, xs, ys, reduce=self.reduce)

        z = numpy.zeros((self.batches[0],), numpy.float32)
        for b, length in enumerate(self.lengths):
            for ys in itertools.product(range(self.n_label), repeat=length):
                z[b] += numpy.exp(self._calc_score(b, ys))

        score = numpy.zeros((self.batches[0],), numpy.float32)
        for b, length in enumerate(self.lengths):
            ys = [self.ys[i][b] for i in range(length)]
            score[b] = self._calc_score(b, ys)

        loss = -(score - numpy.log(z))
        if self.reduce == 'mean':
            expect = numpy.sum(loss) / self.batches[0]
            testing.assert_allclose(actual.data, expect)
        elif self.reduce == 'no':
            expect = loss
            self.assertEqual(len(expect), len(actual))
            for a, e in six.moves.zip(actual, expect):
                testing.assert_allclose(a.data, e)

    def test_forward_cpu(self):
        self.check_forward(self.cost, self.xs, self.ys)

    @attr.gpu
    def test_forward_gpu(self):
        self.check_forward(cuda.to_gpu(self.cost),
                           [cuda.to_gpu(x) for x in self.xs],
                           [cuda.to_gpu(y) for y in self.ys])

    def check_backward(self, cost_data, xs_data, ys_data, g_data):
        def f(cost, *args):
            xs = args[:len(args) // 2]
            ys = args[len(args) // 2:]
            return functions.crf1d(cost, xs, ys, reduce=self.reduce)

        args = [cost_data] + xs_data + ys_data
        if len(self.batches) == 1:
            # When each sequence only contains one element, cost matrix
            # is not used, and its gradient is not updated.<commit_msg>Simplify test<commit_after>import itertools
import six
import unittest

import numpy

import chainer
from chainer import cuda
from chainer import functions
from chainer import gradient_check
from chainer import testing
from chainer.testing import attr


@testing.parameterize(*testing.product_dict(
    [
        {'lengths': [3, 3], 'batches': [2, 2, 2]},
        {'lengths': [3, 2, 1], 'batches': [3, 2, 1]},
        {'lengths': [3, 1, 1], 'batches': [3, 1, 1]},
        {'lengths': [1, 1], 'batches': [2]},
    ],
    [
        {'reduce': 'mean'},
        {'reduce': 'no'},
    ]
))
class TestCRF1d(unittest.TestCase):
    n_label = 3

    def setUp(self):
        self.cost = numpy.random.uniform(
            -1, 1, (self.n_label, self.n_label)).astype(numpy.float32)
        self.xs = [numpy.random.uniform(
            -1, 1, (b, 3)).astype(numpy.float32) for b in self.batches]
        self.ys = [
            numpy.random.randint(
                0, self.n_label, (b,)).astype(numpy.int32)
            for b in self.batches]
        self.g = numpy.random.uniform(
            -1, 1, (len(self.lengths))).astype(numpy.float32)

    def _calc_score(self, batch, ys):
        return sum(x[batch, y] for x, y in zip(self.xs, ys)) + \
            sum(self.cost[y1, y2] for y1, y2 in zip(ys[:-1], ys[1:]))

    def check_forward(self, cost_data, xs_data, ys_data):
        cost = chainer.Variable(cost_data)
        xs = [chainer.Variable(x) for x in xs_data]
        ys = [chainer.Variable(y) for y in ys_data]
        actual = functions.crf1d(cost, xs, ys, reduce=self.reduce)

        z = numpy.zeros((self.batches[0],), numpy.float32)
        for b, length in enumerate(self.lengths):
            for ys in itertools.product(range(self.n_label), repeat=length):
                z[b] += numpy.exp(self._calc_score(b, ys))

        score = numpy.zeros((self.batches[0],), numpy.float32)
        for b, length in enumerate(self.lengths):
            ys = [self.ys[i][b] for i in range(length)]
            score[b] = self._calc_score(b, ys)

        loss = -(score - numpy.log(z))
        if self.reduce == 'mean':
            expect = numpy.sum(loss) / self.batches[0]
        elif self.reduce == 'no':
            expect = loss

        testing.assert_allclose(actual.data, expect)


    def test_forward_cpu(self):
        self.check_forward(self.cost, self.xs, self.ys)

    @attr.gpu
    def test_forward_gpu(self):
        self.check_forward(cuda.to_gpu(self.cost),
                           [cuda.to_gpu(x) for x in self.xs],
                           [cuda.to_gpu(y) for y in self.ys])

    def check_backward(self, cost_data, xs_data, ys_data, g_data):
        def f(cost, *args):
            xs = args[:len(args) // 2]
            ys = args[len(args) // 2:]
            return functions.crf1d(cost, xs, ys, reduce=self.reduce)

        args = [cost_data] + xs_data + ys_data
        if len(self.batches) == 1:
            # When each sequence only contains one element, cost matrix
            # is not used, and its gradient is not updated.
{'fraction_non_alphanumeric': 0.1440637450199203, 'fraction_numerical': 0.017848605577689244, 'mean_word_length': 2.2135176651305684, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6077749', 'n_tokens_mistral': 2253, 'n_tokens_neox': 2145, 'n_words': 524}