<commit_before>/***********************************************
 * Author: <NAME> - <EMAIL>
 * Create: 2017-07-26 13:56
 * Last modified : 2017-09-01 10:55
 * Filename      : ParallelOperator.h
 * Description   : Parallel Operate by multithread 
 **********************************************/
#pragma once

#include <cmath>
#include <vector>
#include <thread>
#include <functional>
#include "utils/TypeDef.h"
#include "utils/Log.h"

namespace abcdl{
namespace utils{

class ParallelOperator{
public:
    ParallelOperator(){
        _num_thread = std::thread::hardware_concurrency();
    }

    ParallelOperator(const size_t num_thread){
        _num_thread = num_thread;
    }

    template<class T>
    void parallel_mul2one(T* op1,
                          const size_t num_op1,
                          const std::function<void(T*)> &f){
        size_t block_size = get_block_size(num_op1);
        size_t num_thread = get_num_thread(num_op1, block_size);
        std::vector<std::thread> threads(num_thread);

        for(size_t i = 0; i != num_thread; i++){
            threads[i] = std::thread(
                [&op1, &f](size_t start_idx, size_t end_idx){
                    for(size_t ti = start_idx; ti != end_idx; ti++){
                        f(&op1[ti]);
                    }
                }, i * block_size, std::min(num_op1, (i + 1) * block_size)
            );
        }

        for(auto& thread : threads){
            thread.join();
        }
    }

    template<class T>
    void parallel_mul2one(T* op1,
                          const size_t num_op1,
                          const T& op2,
                          const std::function<void(T*, const T&)> &f){
        size_t block_size = get_block_size(num_op1);
        size_t num_thread = get_num_thread(num_op1, block_size);
        std::vector<std::thread> threads(num_thread);

        for(size_t i = 0; i != num_thread; i++){
            threads[i] = std::thread(
                [&op1, &op2, &f](size_t start_idx, size_t end_idx){
                    for(size_t ti = start_idx; ti != end_idx; ti++){
                        f(&op1[ti], op2);
                    }
                }, i * block_size, std::min(num_op1, (i + 1) * block_size)
            );
        }

        for(auto& thread : threads){
            thread.join();
        }
    }

    template<class T>
    void parallel_mul2one_copy(T* result_data,
                               const T* op1,
                               const size_t num_op1,
                               const std::function<void(T*, const T&)> &f){
        size_t block_size = get_block_size(num_op1);
        size_t num_thread = get_num_thread(num_op1, block_size);
        std::vector<std::thread> threads(num_thread);

        for(size_t i = 0; i != num_thread; i++){
            threads[i] = std::thread(
                [&result_data, &op1, &f](size_t start_idx, size_t end_idx){
                    for(size_t ti = start_idx; ti != end_idx; ti++){
                        f(&result_data[ti], op1[ti]);
                    }
                }, i * block_size, std::min(num_op1, (i + 1) * block_size)
            );
        }

        for(auto& thread : threads){
            thread.join();
        }
    }

    template<class T>
    void parallel_mul2one_copy(T* result_data,
                               const T* op1,
                               const size_t num_op1,
                               const T& op2,
                               const std::function<void(T*, const T&, const T&)> &f){
        size_t block_size = get_block_size(num_op1);
        size_t num_thread = get_num_thread(num_op1, block_size);
        std::vector<std::thread> threads(num_thread);

        for(size_t i = 0; i != num_thread; i++){
            threads[i] = std::thread(
                [&result_data, &op1, &f, &op2](size_t start_idx, size_t end_idx){
                    for(size_t ti = start_idx; ti != end_idx; ti++){
                        f(&result_data[ti], op1[ti], op2);
                    }
                }, i * block_size, std::min(num_op1, (i + 1) * block_size)
            );
        }

        for(auto& thread : threads){
            thread.join();
        }
    }

    template<class T>
    void parallel_mul2mul(T* op1,
                          const size_t num_op1,
                          const T* op2,
                          const size_t num_op2,
                          const std::function<void(T*, const T&)> &f){
        CHECK(num_op1 == num_op2);
        size_t block_size = get_block_size(num_op1);
        size_t num_thread = get_num_thread(num_op1, block_size);
        std::vector<std::thread> threads(num_thread);

        for(size_t i = 0; i != num_thread; i++){
            threads[i] = std::thread(
                [&op1, &op2, &f](size_t start_idx, size_t end_idx){
                    for(size_t ti = start_idx; ti != end_idx; ti++){
                        f(&op1[ti], op2[ti]);
                    }
                }, i * block_size, std::min(num_op1, (i + 1) * block_size)
            );
        }

        for(auto& thread : threads){
            thread.join();
        }
    }

    template<class T>
    void parallel_mul2mul_repeat(T* op1,
                                 const size_t num_op1,
                                 const T* op2,
                                 const size_t num_op2,
                                 const std::function<void(T*, const T&)> &f){
        CHECK(num_op1 % num_op2 == 0);
        size_t block_size = get_block_size(num_op1);
        size_t num_thread = get_num_thread(num_op1, block_size);
        std::vector<std::thread> threads(num_thread);

        for(size_t i = 0; i != num_thread; i++){
            threads[i] = std::thread(
                [&op1, &op2, &num_op2, &f](size_t start_idx, size_t end_idx){
                    for(size_t ti = start_idx; ti != end_idx; ti++){
                        f(&op1[ti], op2[ti % num_op2]);
                    }
                }, i * block_size, std::min(num_op1, (i + 1) * block_size)
            );
        }

        for(auto& thread : threads){
            thread.join();
        }
    }

    template<class T>
    void parallel_mul2mul_cross(T* result_data,
                                const T* op1,
                                const size_t num_op1,
                                const T* op2,
                                const size_t num_op2,
                                const std::function<void(T*, const T&, const T&)> &f){
        CHECK(num_op1 == num_op2);
        size_t block_size = get_block_size(num_op1 * num_op2);
        size_t num_thread = get_num_thread(num_op1 * num_op2, block_size);
        std::vector<std::thread> threads(num_thread);
        block_size = num_op1 / num_thread;
        if(num_op1 % num_thread != 0){
            block_size += 1;
        }<commit_msg>initialize<commit_after>/***********************************************
 * Author: <NAME> - <EMAIL>
 * Create: 2017-07-26 13:56
 * Last modified : 2017-09-01 10:55
 * Filename      : ParallelOperator.h
 * Description   : Parallel Operate by multithread 
 **********************************************/
#pragma once

#include <cmath>
#include <vector>
#include <thread>
#include <functional>
#include "utils/TypeDef.h"
#include "utils/Log.h"

namespace abcdl{
namespace utils{

class ParallelOperator{
public:
    ParallelOperator(){
        _num_thread = std::thread::hardware_concurrency();
    }

    ParallelOperator(const size_t num_thread){
        _num_thread = num_thread;
    }

    template<class T>
    void parallel_mul2one(T* op1,
                          const size_t num_op1,
                          const std::function<void(T*)> &f){
        size_t block_size = get_block_size(num_op1);
        size_t num_thread = get_num_thread(num_op1, block_size);
        std::vector<std::thread> threads(num_thread);

        for(size_t i = 0; i != num_thread; i++){
            threads[i] = std::thread(
                [&op1, &f](size_t start_idx, size_t end_idx){
                    for(size_t ti = start_idx; ti != end_idx; ti++){
                        f(&op1[ti]);
                    }
                }, i * block_size, std::min(num_op1, (i + 1) * block_size)
            );
        }

        for(auto& thread : threads){
            thread.join();
        }
    }

    template<class T>
    void parallel_mul2one(T* op1,
                          const size_t num_op1,
                          const T& op2,
                          const std::function<void(T*, const T&)> &f){
        size_t block_size = get_block_size(num_op1);
        size_t num_thread = get_num_thread(num_op1, block_size);
        std::vector<std::thread> threads(num_thread);

        for(size_t i = 0; i != num_thread; i++){
            threads[i] = std::thread(
                [&op1, &op2, &f](size_t start_idx, size_t end_idx){
                    for(size_t ti = start_idx; ti != end_idx; ti++){
                        f(&op1[ti], op2);
                    }
                }, i * block_size, std::min(num_op1, (i + 1) * block_size)
            );
        }

        for(auto& thread : threads){
            thread.join();
        }
    }

    template<class T>
    void parallel_mul2one_copy(T* result_data,
                               const T* op1,
                               const size_t num_op1,
                               const std::function<void(T*, const T&)> &f){
        size_t block_size = get_block_size(num_op1);
        size_t num_thread = get_num_thread(num_op1, block_size);
        std::vector<std::thread> threads(num_thread);

        for(size_t i = 0; i != num_thread; i++){
            threads[i] = std::thread(
                [&result_data, &op1, &f](size_t start_idx, size_t end_idx){
                    for(size_t ti = start_idx; ti != end_idx; ti++){
                        f(&result_data[ti], op1[ti]);
                    }
                }, i * block_size, std::min(num_op1, (i + 1) * block_size)
            );
        }

        for(auto& thread : threads){
            thread.join();
        }
    }

    template<class T>
    void parallel_mul2one_copy(T* result_data,
                               const T* op1,
                               const size_t num_op1,
                               const T& op2,
                               const std::function<void(T*, const T&, const T&)> &f){
        size_t block_size = get_block_size(num_op1);
        size_t num_thread = get_num_thread(num_op1, block_size);
        std::vector<std::thread> threads(num_thread);

        for(size_t i = 0; i != num_thread; i++){
            threads[i] = std::thread(
                [&result_data, &op1, &f, &op2](size_t start_idx, size_t end_idx){
                    for(size_t ti = start_idx; ti != end_idx; ti++){
                        f(&result_data[ti], op1[ti], op2);
                    }
                }, i * block_size, std::min(num_op1, (i + 1) * block_size)
            );
        }

        for(auto& thread : threads){
            thread.join();
        }
    }

    template<class T>
    void parallel_mul2mul(T* op1,
                          const size_t num_op1,
                          const T* op2,
                          const size_t num_op2,
                          const std::function<void(T*, const T&)> &f){
        CHECK(num_op1 == num_op2);
        size_t block_size = get_block_size(num_op1);
        size_t num_thread = get_num_thread(num_op1, block_size);
        std::vector<std::thread> threads(num_thread);

        for(size_t i = 0; i != num_thread; i++){
            threads[i] = std::thread(
                [&op1, &op2, &f](size_t start_idx, size_t end_idx){
                    for(size_t ti = start_idx; ti != end_idx; ti++){
                        f(&op1[ti], op2[ti]);
                    }
                }, i * block_size, std::min(num_op1, (i + 1) * block_size)
            );
        }

        for(auto& thread : threads){
            thread.join();
        }
    }

    template<class T>
    void parallel_mul2mul_repeat(T* op1,
                                 const size_t num_op1,
                                 const T* op2,
                                 const size_t num_op2,
                                 const std::function<void(T*, const T&)> &f){
        CHECK(num_op1 % num_op2 == 0);
        size_t block_size = get_block_size(num_op1);
        size_t num_thread = get_num_thread(num_op1, block_size);
        std::vector<std::thread> threads(num_thread);

        for(size_t i = 0; i != num_thread; i++){
            threads[i] = std::thread(
                [&op1, &op2, &num_op2, &f](size_t start_idx, size_t end_idx){
                    for(size_t ti = start_idx; ti != end_idx; ti++){
                        f(&op1[ti], op2[ti % num_op2]);
                    }
                }, i * block_size, std::min(num_op1, (i + 1) * block_size)
            );
        }

        for(auto& thread : threads){
            thread.join();
        }
    }

    template<class T>
    void parallel_mul2mul_cross(T* result_data,
                                const T* op1,
                                const size_t num_op1,
                                const T* op2,
                                const size_t num_op2,
                                const std::function<void(T*, const T&, const T&)> &f){
        CHECK(num_op1 > 0 && num_op2 > 0);
        size_t block_size = get_block_size(num_op1 * num_op2);
        size_t num_thread = get_num_thread(num_op1 * num_op2, block_size);
        std::vector<std::thread> threads(num_thread);
        block_size = num_op1 / num_thread;
        if(num_op1 % num_thread != 0){
            block_size += 1;
        }
{'fraction_non_alphanumeric': 0.13269384492406075, 'fraction_numerical': 0.017585931254996003, 'mean_word_length': 1.5461609620721555, 'pattern_counts': {'":': 0, '<': 57, '<?xml version=': 0, '>': 59, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 8, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1188557', 'n_tokens_mistral': 4449, 'n_tokens_neox': 4265, 'n_words': 896}