<commit_before>/*******************************************************************************
 * tests/api/groupby_node_test.cpp
 *
 * Part of Project Thrill.
 *
 * Copyright (C) 2015 <NAME> <<EMAIL>>
 *
 * This file has no license. Only Chuck Norris can compile it.
 ******************************************************************************/

#include <thrill/api/allgather.hpp>
#include <thrill/api/generate.hpp>
#include <thrill/api/groupby.hpp>
#include <thrill/api/groupby_index.hpp>
#include <thrill/api/sum.hpp>
#include <thrill/common/logger.hpp>
#include <thrill/data/file.hpp>
#include <gtest/gtest.h>
#include <thrill/core/multiway_merge.hpp>

#include <algorithm>
#include <cstdlib>
#include <limits>
#include <string>
#include <vector>

using namespace thrill; // NOLINT

static const bool debug = false;

TEST(GroupByNode, CompileAndSum) {

    std::function<void(Context&)> start_func =
        [](Context& ctx) {
            unsigned n = 999999;
            static const unsigned m = 4;

            auto sizets = Generate(
                ctx,
                [](const size_t& index) {
                    return static_cast<std::size_t>(index + 1);
                },
                n);

            auto modulo_keyfn = [](size_t in) { return (in % m); };

            auto sum_fn =
                [](api::GroupByIterator<std::size_t, decltype(modulo_keyfn)>& r,
                   std::size_t key) {
                    auto res = 0;
                    int k = 0;
                    while (r.HasNext()) {
                        auto n = r.Next();
                        assert(n%m == key);
                        k = n % m;
                        res += n;
                    }
                    return static_cast<int>(res);
                };

            // group by to compute sum and gather results
            auto reduced = sizets.GroupBy(modulo_keyfn, sum_fn);
            std::vector<int> out_vec = reduced.AllGather();

            // compute vector with expected results
            std::vector<unsigned> res_vec(m, 0);
            for (unsigned t = 0; t <= n; ++t) {
                res_vec[t % m] += t;
            }

            std::sort(out_vec.begin(), out_vec.end());
            std::sort(res_vec.begin(), res_vec.end());

            LOG << "res_vec " << "/" << " out_vec";
            for (std::size_t i = 0; i < res_vec.size(); ++i) {
                LOG << res_vec[i] << " / " << out_vec[i];
                ASSERT_EQ(res_vec[i], out_vec[i]);
            }
        };

    api::RunLocalTests(start_func);
}

TEST(GroupByNode, Median) {

    std::function<void(Context&)> start_func =
        [](Context& ctx) {
            unsigned n = 999999;
            static const unsigned m = 4;

            auto sizets = Generate(
                ctx,
                [](const size_t& index) {
                    return index + 1;
                },
                n);

            auto modulo_keyfn = [](size_t in) { return (in % m); };

            auto median_fn =
                [](api::GroupByIterator<std::size_t, decltype(modulo_keyfn)>& r,
                    std::size_t) {
                    std::vector<std::size_t> all;
                    while (r.HasNext()) {
                        all.push_back(r.Next());
                    }
                    std::sort(std::begin(all), std::end(all));
                    for (auto c : all) {
                        LOG << c;
                    }
                    return static_cast<int>(all[all.size() / 2 - 1]);<commit_msg>make tests smaller and faster<commit_after>/*******************************************************************************
 * tests/api/groupby_node_test.cpp
 *
 * Part of Project Thrill.
 *
 * Copyright (C) 2015 <NAME> <<EMAIL>>
 *
 * This file has no license. Only Chuck Norris can compile it.
 ******************************************************************************/

#include <thrill/api/allgather.hpp>
#include <thrill/api/generate.hpp>
#include <thrill/api/groupby.hpp>
#include <thrill/api/groupby_index.hpp>
#include <thrill/api/sum.hpp>
#include <thrill/common/logger.hpp>
#include <thrill/data/file.hpp>
#include <gtest/gtest.h>
#include <thrill/core/multiway_merge.hpp>

#include <algorithm>
#include <cstdlib>
#include <limits>
#include <string>
#include <vector>

using namespace thrill; // NOLINT

static const bool debug = false;

TEST(GroupByNode, CompileAndSum) {

    std::function<void(Context&)> start_func =
        [](Context& ctx) {
            unsigned n = 99999;
            static const unsigned m = 4;

            auto sizets = Generate(
                ctx,
                [](const size_t& index) {
                    return static_cast<std::size_t>(index + 1);
                },
                n);

            auto modulo_keyfn = [](size_t in) { return (in % m); };

            auto sum_fn =
                [](api::GroupByIterator<std::size_t, decltype(modulo_keyfn)>& r,
                   std::size_t key) {
                    auto res = 0;
                    int k = 0;
                    while (r.HasNext()) {
                        auto n = r.Next();
                        assert(n%m == key);
                        k = n % m;
                        res += n;
                    }
                    return static_cast<int>(res);
                };

            // group by to compute sum and gather results
            auto reduced = sizets.GroupBy(modulo_keyfn, sum_fn);
            std::vector<int> out_vec = reduced.AllGather();

            // compute vector with expected results
            std::vector<unsigned> res_vec(m, 0);
            for (unsigned t = 0; t <= n; ++t) {
                res_vec[t % m] += t;
            }

            std::sort(out_vec.begin(), out_vec.end());
            std::sort(res_vec.begin(), res_vec.end());

            LOG << "res_vec " << "/" << " out_vec";
            for (std::size_t i = 0; i < res_vec.size(); ++i) {
                LOG << res_vec[i] << " / " << out_vec[i];
                ASSERT_EQ(res_vec[i], out_vec[i]);
            }
        };

    api::RunLocalTests(start_func);
}

TEST(GroupByNode, Median) {

    std::function<void(Context&)> start_func =
        [](Context& ctx) {
            unsigned n = 99999;
            static const unsigned m = 4;

            auto sizets = Generate(
                ctx,
                [](const size_t& index) {
                    return index + 1;
                },
                n);

            auto modulo_keyfn = [](size_t in) { return (in % m); };

            auto median_fn =
                [](api::GroupByIterator<std::size_t, decltype(modulo_keyfn)>& r,
                    std::size_t) {
                    std::vector<std::size_t> all;
                    while (r.HasNext()) {
                        all.push_back(r.Next());
                    }
                    std::sort(std::begin(all), std::end(all));
                    for (auto c : all) {
                        LOG << c;
                    }
                    return static_cast<int>(all[all.size() / 2 - 1]);
{'fraction_non_alphanumeric': 0.17333709131905298, 'fraction_numerical': 0.007328072153325817, 'mean_word_length': 1.5630191404839293, 'pattern_counts': {'":': 0, '<': 89, '<?xml version=': 0, '>': 57, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1056314', 'n_tokens_mistral': 2183, 'n_tokens_neox': 1996, 'n_words': 455}