<commit_before>from bndl.compute.tests import DatasetTest
from bndl.execute.worker import current_worker
from collections import Counter


class SelectWorkersTest(DatasetTest):
    def test_require(self):
        executed_on = self.ctx.accumulator(Counter())
        def register_worker(i):
            nonlocal executed_on
            executed_on += Counter({current_worker().name:1})

        dset = self.ctx.range(10).map(register_worker)

        targeted = Counter()
        for _ in range(2):
            for worker in self.ctx.workers:
                worker_name = worker.name
                targeted[worker_name] += 10
                dset.require_workers(lambda w: w.name == worker_name).execute()
                self.assertEqual(executed_on.value, targeted)
<commit_msg>require_workers signature changed<commit_after>from bndl.compute.tests import DatasetTest
from bndl.execute.worker import current_worker
from collections import Counter


class SelectWorkersTest(DatasetTest):
    def test_require(self):
        executed_on = self.ctx.accumulator(Counter())
        def register_worker(i):
            nonlocal executed_on
            executed_on += Counter({current_worker().name:1})

        dset = self.ctx.range(10).map(register_worker)

        targeted = Counter()
        for _ in range(2):
            for worker in self.ctx.workers:
                worker_name = worker.name
                targeted[worker_name] += 10
                dset.require_workers(lambda workers: [w for w in workers if w.name == worker_name]).execute()
                self.assertEqual(executed_on.value, targeted)

{'fraction_non_alphanumeric': 0.0891089108910891, 'fraction_numerical': 0.007425742574257425, 'mean_word_length': 2.61744966442953, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3506418', 'n_tokens_mistral': 454, 'n_tokens_neox': 431, 'n_words': 98}