<commit_before>#include <iostream>
#include <cstdint>
#include <random>
#include <chrono>
#include <vector>
#include <boost/dynamic_bitset.hpp>

/**
 * Lessons learned so far:
 * - SIMD registers are used as soon as we compile with -03
 * - If we want to get the real speed up (and use all SIMD registers by unfolding both loops,
 *   we have to know both, input-size and number of comparisons, at compile time.
 * - Otherwise, only some of the SIMD registers are used, and speedup is 0, compared to
 *   code optimized with 02.
 * - Example compile command: g++ -std=c++11 -march=native -O3 simd.cpp -o simd
 * - Command for getting assembly code: g++ -std=c++11 -march=native -O3 -S simd.cpp
 */

typedef std::mt19937 Engine;
typedef std::uniform_int_distribution<unsigned> Intdistr;


template <typename T, typename U>
void smaller (const T *input, U outputs, const T *comparison_values, const unsigned array_size, const unsigned comparisons) {

    for (unsigned i = 0; i < array_size; ++i)
    {
        for (unsigned m = 0; m < comparisons; ++m) {
            outputs[m*array_size+i] = input[i] < comparison_values[m];
        }
    }
}

//template <typename T>
//void pretty_print (T *arr, unsigned size, std::string s = "Pretty Print") {
//    std::cout << s << ":" << std::endl;
//    for (auto r = arr; r < arr+size; ++r ) {
//        std::cout << *r << std::endl;
//    }
//}

template <typename T>
void fill (T *arr, unsigned size) {
    Engine engine (0);
    Intdistr distr (0, 100000);
    for (auto r = arr; r < arr+size; ++r ) {
        *r = distr(engine);
    }
}

int main (int argc, char *argv[]) {
    typedef unsigned TestType;
    static constexpr unsigned repetitions = 100;

    constexpr unsigned input_size = 1000000;
    constexpr unsigned comparisons = 3;

//    if (argc != 3)
//    {
//        std::cout << "Usage: ./simd <input-size> <comparisons>" << std::endl;
//        return -1;
//    }
//    unsigned long input_size = std::stoi(argv[1]);
//    unsigned comparisons = std::stoi(argv[2]);

    std::cout << "input size: " << input_size << std::endl;
    std::cout << "comparisons :" << comparisons<< std::endl;

    TestType test_input [input_size];
    fill(test_input, input_size);
//    pretty_print(test_input, input_size, "Input");

    TestType comparison_values [comparisons];
    for (unsigned c = 0; c < comparisons; ++c) {
        comparison_values[c] = test_input[c];
    }
//    pretty_print(comparison_values, comparisons, "Comparison values");

    bool results [comparisons * input_size];
//    std::vector<bool> results (comparisons * input_size);
//    boost::dynamic_bitset<> results(comparisons * input_size);

    auto start = std::chrono::high_resolution_clock::now();
    for (unsigned i = 0; i < repetitions; ++i)
        smaller(test_input, results, comparison_values, input_size, comparisons);
    auto end = std::chrono::high_resolution_clock::now();
    std::cout << "Avg Time [microsecs]: "
              << (std::chrono::duration_cast<std::chrono::microseconds>(end-start).count())
                 /((double)repetitions)
              << std::endl;

//    for (unsigned c = 0; c < comparisons; ++c) {
//        pretty_print(&results[c*input_size], input_size, "Result");
//    }
<commit_msg>finished simd experiments for all-same-operator (smaller)<commit_after>#include <iostream>
#include <cstdint>
#include <random>
#include <chrono>
#include <vector>
#include <boost/dynamic_bitset.hpp>
#include <numeric>

/**
 * Lessons learned so far:
 * - SIMD registers are used as soon as we compile with -03
 * - If we want to get the real speed up (and use all SIMD registers by unfolding both loops,
 *   we have to know both, input-size and number of comparisons, at compile time.
 * - Otherwise, only some of the SIMD registers are used, and speedup is 0, compared to
 *   code optimized with 02.
 * - Example compile command: g++ -std=c++11 -march=native -O3 simd.cpp -o simd
 * - Command for getting assembly code: g++ -std=c++11 -march=native -O3 -S simd.cpp
 */

typedef std::mt19937 Engine;
typedef std::uniform_int_distribution<unsigned> Intdistr;


template <typename T, typename U>
void smaller (const T *input, U outputs, const T *comparison_values, const unsigned array_size, const unsigned comparisons) {

    for (unsigned i = 0; i < array_size; ++i)
    {
        for (unsigned m = 0; m < comparisons; ++m) {
            outputs[m*array_size+i] = (float) (input[i] < comparison_values[m]);
        }
    }
}

template <typename T>
void pretty_print (T *arr, unsigned size, std::string s = "Pretty Print") {
    std::cout << s << ":" << std::endl;
    for (auto r = arr; r < arr+size; ++r ) {
        std::cout << *r << std::endl;
    }
}

template <typename T>
void compute_stats(const std::vector<T> stats, double &mean, double &stdev) {
    double sum = std::accumulate(stats.begin(), stats.end(), 0.0);
    mean = sum / stats.size();

    std::vector<double> diff(stats.size());
    std::transform(stats.begin(), stats.end(), diff.begin(),
                   std::bind2nd(std::minus<double>(), mean));
    double sq_sum = std::inner_product(diff.begin(), diff.end(), diff.begin(), 0.0);
    stdev = std::sqrt(sq_sum / stats.size());
}

template <typename T>
void fill (T *arr, unsigned size) {
    Engine engine (0);
    Intdistr distr (0, 100000);
    for (auto r = arr; r < arr+size; ++r ) {
        *r = distr(engine);
    }
}

int main (int argc, char *argv[]) {
    //**** PARAMS ****/
    typedef unsigned TestType;
    static constexpr unsigned repetitions = 10000;

//    constexpr unsigned input_size = 500000;
    constexpr unsigned comparisons = 1;

//    if (argc != 3)
//    {
//        std::cout << "Usage: ./simd <input-size> <comparisons>" << std::endl;
//        return -1;
//    }
    unsigned long input_size = std::stoi(argv[1]);
//    unsigned comparisons = std::stoi(argv[2]);

    std::cout << "input size: " << input_size << std::endl;
    std::cout << "comparisons: " << comparisons<< std::endl;


    //**** INPUT ****/
    TestType test_input [input_size];
    fill(test_input, input_size);
//    pretty_print(test_input, input_size, "Input");

    TestType comparison_values [comparisons];
    for (unsigned c = 0; c < comparisons; ++c) {
        comparison_values[c] = test_input[c];
    }
//    pretty_print(comparison_values, comparisons, "Comparison values");


    //**** COMPUTE ****/
    std::vector<unsigned long> stats (repetitions);
    bool results [comparisons * input_size];
//    std::vector<bool> results (comparisons * input_size);
//    boost::dynamic_bitset<> results(comparisons * input_size);

    for (unsigned i = 0; i < repetitions; ++i) {
        auto start = std::chrono::high_resolution_clock::now();
        smaller(test_input, results, comparison_values, input_size, comparisons);
        auto end = std::chrono::high_resolution_clock::now();
        stats [i] =
            std::chrono::duration_cast<std::chrono::microseconds>(end-start).count();
    }

    //**** REPORT ****/
    double mean, stdev;
    compute_stats(stats, mean, stdev);
    std::cout
//              << "Avg Time [microsecs]: "
              << mean
              << "\t"
//              << "(+/- "
              << stdev
//              << ")"
              << std::endl;
//    pretty_print(stats.data(), repetitions, "Stats");

//    for (unsigned c = 0; c < comparisons; ++c) {
//        pretty_print(&results[c*input_size], input_size, "Result");
//    }

{'fraction_non_alphanumeric': 0.15745079662605435, 'fraction_numerical': 0.012585352791538358, 'mean_word_length': 3.0686274509803924, 'pattern_counts': {'":': 2, '<': 127, '<?xml version=': 0, '>': 39, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6835355', 'n_tokens_mistral': 2495, 'n_tokens_neox': 2356, 'n_words': 688}