<commit_before>import asyncio
import time
import pytest

from searxstats.common.foreach import for_each


@pytest.mark.asyncio
@pytest.mark.parametrize('limit', [None, 4, 3, 2, 1])
async def test_for_each(limit):
    input_list = ['A', 'B', 'C', 'D']
    expected_output = set(input_list)
    output_set = set()
    parallel = [0, 0]

    def f_generic(i):
        parallel[0] += 1
        parallel[1] = max(parallel)
        yield
        if i in output_set:
            raise ValueError(f'{i} already seen')
        output_set.add(i)
        parallel[0] -= 1

    async def f_func(i):
        generic_routine = f_generic(i)
        next(generic_routine)
        time.sleep(0.5)
        next(generic_routine, None)

    async def f_async(i):
        generic_routine = f_generic(i)
        next(generic_routine)
        asyncio.sleep(0.5)
        time.sleep(0.5)
        next(generic_routine, None)

    await for_each(input_list, f_func, limit=limit)
    assert output_set == expected_output
    assert parallel[0] == 0
    assert parallel[1] == limit or 1

    output_set = set()
    parallel = [0, 0]
    await for_each(input_list, f_async, limit=limit)
    assert output_set == expected_output<commit_msg>[fix] tests<commit_after>import asyncio
import time
import pytest

from searxstats.common.foreach import for_each


@pytest.mark.asyncio
@pytest.mark.parametrize('limit', [None, 4, 3, 2, 1])
async def test_for_each(limit):
    input_list = ['A', 'B', 'C', 'D']
    expected_output = set(input_list)
    output_set = set()
    parallel = [0, 0]

    def f_generic(i):
        parallel[0] += 1
        parallel[1] = max(parallel)
        yield
        if i in output_set:
            raise ValueError(f'{i} already seen')
        output_set.add(i)
        parallel[0] -= 1

    async def f_func(i):
        generic_routine = f_generic(i)
        next(generic_routine)
        time.sleep(0.5)
        next(generic_routine, None)

    async def f_async(i):
        generic_routine = f_generic(i)
        next(generic_routine)
        await asyncio.sleep(0.5)
        next(generic_routine, None)

    await for_each(input_list, f_func, limit=limit)
    assert output_set == expected_output
    assert parallel[0] == 0
    assert parallel[1] == limit or 1

    output_set = set()
    parallel = [0, 0]
    await for_each(input_list, f_async, limit=limit)
    assert output_set == expected_output
{'fraction_non_alphanumeric': 0.11426188490408674, 'fraction_numerical': 0.01834862385321101, 'mean_word_length': 2.7484375, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1050381', 'n_tokens_mistral': 843, 'n_tokens_neox': 822, 'n_words': 192}