<commit_before>#ifndef NANOCV_THREAD_LOOP_H
#define NANOCV_THREAD_LOOP_H

#include "thread_pool.h"

namespace ncv
{
        ///
        /// \brief split a loop computation of the given size using multiple threads on a given thread pool
        ///
        template
        <
                typename tsize,
                class toperator
        >
        void thread_loop(tsize N, toperator op, thread_pool_t& pool)
        {
                const tsize n_workers = static_cast<tsize>(pool.n_workers());
                const tsize task_size = std::max(tsize(1), N / n_workers / 16);

                for (tsize i = 0; i < N; i += task_size)
                {
                        pool.enqueue([=,&op]()
                        {
                                for (size_t t = 0; t + i < N; t ++)
                                {
                                        op(t + i);
                                }
                        });
                }

                pool.wait();
        }

        ///
        /// \brief split a loop computation of the given size using multiple threads
        ///
        template
        <
                typename tsize,
                class toperator
        >
        void thread_loop(tsize N, toperator op, tsize threads = tsize(0))
        {
                thread_pool_t pool(threads);
                thread_loop(N, op, pool);
        }

        ///
        /// \brief split a loop computation of the given size using multiple threads and cumulate partial results
        ///
        template
        <
                typename tdata,
                typename tsize,
                class toperator_init,
                class toperator,
                class toperator_cumulate
        >
        void thread_loop_cumulate(
                tsize N, toperator_init op_init, toperator op, toperator_cumulate op_cumulate, tsize threads = tsize(0))
        {
                thread_pool_t pool(threads);

                const tsize n_tasks = static_cast<tsize>(pool.n_workers());

                std::vector<tdata> data(n_tasks);
                for (tsize t = 0; t < n_tasks; t ++)
                {
                        op_init(data[t]);
                }

                for (tsize t = 0; t < n_tasks; t ++)
                {
                        pool.enqueue([=,&data,&op]()
                        {
                                for (tsize i = t; i < N; i += n_tasks)
                                {
                                        op(i, data[t]);
                                }
                        });
                }

                pool.wait();

                for (tsize t = 0; t < n_tasks; t ++)
                {
                        op_cumulate(data[t]);
                }
        }
}

#endif // NANOCV_THREAD_LOOP_H

<commit_msg>reverted thread_loop implementation (still heavier than openmp)<commit_after>#ifndef NANOCV_THREAD_LOOP_H
#define NANOCV_THREAD_LOOP_H

#include "thread_pool.h"

namespace ncv
{
        ///
        /// \brief split a loop computation of the given size using multiple threads on a given thread pool
        ///
        template
        <
                typename tsize,
                class toperator
        >
        void thread_loop(tsize N, toperator op, thread_pool_t& pool)
        {
                const tsize n_tasks = static_cast<tsize>(pool.n_workers());

                for (tsize t = 0; t < n_tasks; t ++)
                {
                        pool.enqueue([=,&op]()
                        {
                                for (tsize i = t; i < N; i += n_tasks)
                                {
                                        op(i);
                                }
                        });
                }

                pool.wait();
        }

        ///
        /// \brief split a loop computation of the given size using multiple threads
        ///
        template
        <
                typename tsize,
                class toperator
        >
        void thread_loop(tsize N, toperator op, tsize threads = tsize(0))
        {
                thread_pool_t pool(threads);
                thread_loop(N, op, pool);
        }

        ///
        /// \brief split a loop computation of the given size using multiple threads and cumulate partial results
        ///
        template
        <
                typename tdata,
                typename tsize,
                class toperator_init,
                class toperator,
                class toperator_cumulate
        >
        void thread_loop_cumulate(
                tsize N, toperator_init op_init, toperator op, toperator_cumulate op_cumulate, tsize threads = tsize(0))
        {
                thread_pool_t pool(threads);

                const tsize n_tasks = static_cast<tsize>(pool.n_workers());

                std::vector<tdata> data(n_tasks);
                for (tsize t = 0; t < n_tasks; t ++)
                {
                        op_init(data[t]);
                }

                for (tsize t = 0; t < n_tasks; t ++)
                {
                        pool.enqueue([=,&data,&op]()
                        {
                                for (tsize i = t; i < N; i += n_tasks)
                                {
                                        op(i, data[t]);
                                }
                        });
                }

                pool.wait();

                for (tsize t = 0; t < n_tasks; t ++)
                {
                        op_cumulate(data[t]);
                }
        }
}

#endif // NANOCV_THREAD_LOOP_H


{'fraction_non_alphanumeric': 0.08568387440127727, 'fraction_numerical': 0.0028383892141209863, 'mean_word_length': 0.920954003407155, 'pattern_counts': {'":': 0, '<': 27, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 9, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2982232', 'n_tokens_mistral': 1560, 'n_tokens_neox': 1419, 'n_words': 363}