<commit_before>from __future__ import division

import os
import tempfile
import unittest

from chainer import serializers
from chainer import testing
from chainer import training


class DummyUpdater(training.Updater):

    def __init__(self, iters_per_epoch):
        self.iteration = 0
        self.iters_per_epoch = iters_per_epoch

    def finalize(self):
        pass

    def get_all_optimizers(self):
        return {}

    def update(self):
        self.iteration += 1

    @property
    def epoch(self):
        return self.iteration // self.iters_per_epoch

    @property
    def epoch_detail(self):
        return self.iteration / self.iters_per_epoch

    @property
    def previous_epoch_detail(self):
        if self.iteration == 0:
            return None
        return (self.iteration - 1) / self.iters_per_epoch

    @property
    def is_new_epoch(self):
        return 0 <= self.iteration % self.iters_per_epoch < 1

    def serialize(self, serializer):
        self.iteration = serializer('iteration', self.iteration)


@testing.parameterize(*testing.product_dict([
    # iteration
    {
        'iters_per_epoch': 5, 'interval': (2, 'iteration'),
        'expected': [False, True, False, True, False, True, False]},
    # basic epoch
    {
        'iters_per_epoch': 1, 'interval': (3, 'epoch'),
        'expected': [False, False, True, False, False, True, False]},
    # fractional epoch
    {
        'iters_per_epoch': 2, 'interval': (1.5, 'epoch'),
        'expected': [False, False, True, False, False, True, False]},
    # unaligned epoch
    {
        'iters_per_epoch': 2.5, 'interval': (1, 'epoch'),
        'expected': [False, False, True, False, True, False, False]},
    # tiny epoch
    {
        'iters_per_epoch': 0.5, 'interval': (1, 'epoch'),
        'expected': [True, True, True, True, True, True, True]},
]))
class TestTrigger(unittest.TestCase):

    def setUp(self):
        self.trigger = training.trigger.IntervalTrigger(*self.interval)

    def test_trigger(self):
        updater = DummyUpdater(self.iters_per_epoch)
        trainer = training.Trainer(updater)
        for expected in self.expected:
            updater.update()
            self.assertEqual(self.trigger(trainer), expected)

    def test_stop_trigger(self):
        updater = DummyUpdater(self.iters_per_epoch)
        trainer = training.Trainer(updater)
        for expected in [False] + self.expected:
            self.assertEqual(self.trigger(trainer), expected)
            if expected:
                break
            updater.update()

    def test_resumed_trigger(self):
        temp_dir = tempfile.mkdtemp()
        temp_file = os.path.join(temp_dir, 'temp.npz')

        updater = DummyUpdater(self.iters_per_epoch)
        serializers.save_npz(temp_file, updater)

        for expected in self.expected:
            updater = DummyUpdater(self.iters_per_epoch)
            serializers.load_npz(temp_file, updater)
            trainer = training.Trainer(updater)

            updater.update()
            self.assertEqual(self.trigger(trainer), expected)

            serializers.save_npz(temp_file, updater)


testing.run_module(__name__, __file__)
<commit_msg>resume once for each test<commit_after>from __future__ import division

import os
import tempfile
import unittest

from chainer import serializers
from chainer import testing
from chainer import training


class DummyUpdater(training.Updater):

    def __init__(self, iters_per_epoch):
        self.iteration = 0
        self.iters_per_epoch = iters_per_epoch

    def finalize(self):
        pass

    def get_all_optimizers(self):
        return {}

    def update(self):
        self.iteration += 1

    @property
    def epoch(self):
        return self.iteration // self.iters_per_epoch

    @property
    def epoch_detail(self):
        return self.iteration / self.iters_per_epoch

    @property
    def previous_epoch_detail(self):
        if self.iteration == 0:
            return None
        return (self.iteration - 1) / self.iters_per_epoch

    @property
    def is_new_epoch(self):
        return 0 <= self.iteration % self.iters_per_epoch < 1

    def serialize(self, serializer):
        self.iteration = serializer('iteration', self.iteration)


@testing.parameterize(*testing.product_dict([
    # iteration
    {
        'iters_per_epoch': 5, 'interval': (2, 'iteration'),
        'expected': [False, True, False, True, False, True, False]},
    # basic epoch
    {
        'iters_per_epoch': 1, 'interval': (3, 'epoch'),
        'expected': [False, False, True, False, False, True, False]},
    # fractional epoch
    {
        'iters_per_epoch': 2, 'interval': (1.5, 'epoch'),
        'expected': [False, False, True, False, False, True, False]},
    # unaligned epoch
    {
        'iters_per_epoch': 2.5, 'interval': (1, 'epoch'),
        'expected': [False, False, True, False, True, False, False]},
    # tiny epoch
    {
        'iters_per_epoch': 0.5, 'interval': (1, 'epoch'),
        'expected': [True, True, True, True, True, True, True]},
]))
class TestTrigger(unittest.TestCase):

    def setUp(self):
        self.trigger = training.trigger.IntervalTrigger(*self.interval)

    def test_trigger(self):
        updater = DummyUpdater(self.iters_per_epoch)
        trainer = training.Trainer(updater)
        for expected in self.expected:
            updater.update()
            self.assertEqual(self.trigger(trainer), expected)

    def test_stop_trigger(self):
        updater = DummyUpdater(self.iters_per_epoch)
        trainer = training.Trainer(updater)
        for expected in [False] + self.expected:
            self.assertEqual(self.trigger(trainer), expected)
            if expected:
                break
            updater.update()

    def test_resumed_trigger(self):
        temp_dir = tempfile.mkdtemp()
        temp_file = os.path.join(temp_dir, 'temp.npz')

        for resume in range(len(self.expected)):
            updater = DummyUpdater(self.iters_per_epoch)
            trainer = training.Trainer(updater)
            for expected in self.expected[:resume]:
                updater.update()
                self.assertEqual(self.trigger(trainer), expected)
            serializers.save_npz(temp_file, updater)

            updater = DummyUpdater(self.iters_per_epoch)
            serializers.load_npz(temp_file, updater)
            trainer = training.Trainer(updater)
            for expected in self.expected[resume:]:
                updater.update()
                self.assertEqual(self.trigger(trainer), expected)


testing.run_module(__name__, __file__)

{'fraction_non_alphanumeric': 0.11077481840193705, 'fraction_numerical': 0.0057506053268765135, 'mean_word_length': 2.6193866374589265, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2771849', 'n_tokens_mistral': 2051, 'n_tokens_neox': 2070, 'n_words': 451}