<commit_before>#define NONIUS_RUNNER
#include <nonius/main.h++>

#include "../include/dict/dict.hpp"

#include <unordered_map>
#include <random>
#include <functional>

#ifdef WITH_GOOGLE_BENCH
#include <sparsehash/dense_hash_map>
#endif

template <typename Map>
void insert_test(Map& map) {
    for (int i = 0; i != 10000; ++i) {
        map[i] = i;
    }
}

NONIUS_BENCHMARK("dict insert", [](nonius::chronometer meter) {
    boost::dict<int, int> d;
    d.reserve(1000);
    meter.measure([&] { insert_test(d); });
})

NONIUS_BENCHMARK("umap insert", [](nonius::chronometer meter) {
    std::unordered_map<int, int> d;
    d.reserve(1000);
    meter.measure([&] { insert_test(d); });
})

#ifdef WITH_GOOGLE_BENCH
NONIUS_BENCHMARK("google insert", [](nonius::chronometer meter) {
    google::dense_hash_map<int, int> d(1000);
    d.set_empty_key(0);
    meter.measure([&] { insert_test(d); });
})
#endif

#ifdef WITH_GOOGLE_BENCH
template <typename Map>
Map build_map_google(int size) {
    Map d;
    d.set_empty_key(0);

    for (int i = 0; i != size; ++i) {
        d[i] = i;
    }

    return d;
}

template <typename Map>
Map build_map_google_with_reserve(int size) {
    Map d(size);
    d.set_empty_key(0);

    for (int i = 0; i != size; ++i) {
        d[i] = i;
    }

    return d;
}
#endif

template <typename Map>
Map build_map(int size) {
    Map d;

    for (int i = 0; i != size; ++i) {
        d[i] = i;
    }

    return d;
}

template <typename Map>
Map build_map_with_reserve(int size) {
    Map d(size);

    for (int i = 0; i != size; ++i) {
        d[i] = i;
    }

    return d;
}

template <typename Map, typename Generator>
int lookup_test(Map& map, Generator& gen) {
    int res = 0;
    for (std::size_t i = 0; i < 100; i += 1) {
        res += map[gen()];
    }

    return res;
}

const int lookup_test_size = 1000000;

NONIUS_BENCHMARK("dict lookup", [](nonius::chronometer meter) {
    auto d = build_map<boost::dict<int, int>>(lookup_test_size);

    std::uniform_int_distribution<std::size_t> normal(0, d.size() - 1);
    std::mt19937 engine;
    auto gen = std::bind(std::ref(normal), std::ref(engine));

    meter.measure([&, gen] { return lookup_test(d, gen); });
})

NONIUS_BENCHMARK("umap lookup", [](nonius::chronometer meter) {
    auto d = build_map<std::unordered_map<int, int>>(lookup_test_size);

    std::uniform_int_distribution<std::size_t> normal(0, d.size() - 1);
    std::mt19937 engine;
    auto gen = std::bind(std::ref(normal), std::ref(engine));<commit_msg>added small dict look up test<commit_after>#define NONIUS_RUNNER
#include <nonius/main.h++>

#include "../include/dict/dict.hpp"

#include <unordered_map>
#include <random>
#include <functional>

#ifdef WITH_GOOGLE_BENCH
#include <sparsehash/dense_hash_map>
#endif

template <typename Map>
void insert_test(Map& map) {
    for (int i = 0; i != 10000; ++i) {
        map[i] = i;
    }
}

NONIUS_BENCHMARK("dict insert", [](nonius::chronometer meter) {
    boost::dict<int, int> d;
    d.reserve(1000);
    meter.measure([&] { insert_test(d); });
})

NONIUS_BENCHMARK("umap insert", [](nonius::chronometer meter) {
    std::unordered_map<int, int> d;
    d.reserve(1000);
    meter.measure([&] { insert_test(d); });
})

#ifdef WITH_GOOGLE_BENCH
NONIUS_BENCHMARK("google insert", [](nonius::chronometer meter) {
    google::dense_hash_map<int, int> d(1000);
    d.set_empty_key(0);
    meter.measure([&] { insert_test(d); });
})
#endif

#ifdef WITH_GOOGLE_BENCH
template <typename Map>
Map build_map_google(int size) {
    Map d;
    d.set_empty_key(0);

    for (int i = 0; i != size; ++i) {
        d[i] = i;
    }

    return d;
}

template <typename Map>
Map build_map_google_with_reserve(int size) {
    Map d(size);
    d.set_empty_key(0);

    for (int i = 0; i != size; ++i) {
        d[i] = i;
    }

    return d;
}
#endif

template <typename Map>
Map build_map(int size) {
    Map d;

    for (int i = 0; i != size; ++i) {
        d[i] = i;
    }

    return d;
}

template <typename Map>
Map build_map_with_reserve(int size) {
    Map d(size);

    for (int i = 0; i != size; ++i) {
        d[i] = i;
    }

    return d;
}

template <typename Map, typename Generator>
int lookup_test(Map& map, Generator& gen) {
    int res = 0;
    for (std::size_t i = 0; i < 100; i += 1) {
        res += map[gen()];
    }

    return res;
}

const int small_lookup_test_size = 10;

NONIUS_BENCHMARK("small dict lookup", [](nonius::chronometer meter) {
    auto d = build_map<boost::dict<int, int>>(small_lookup_test_size);

    std::uniform_int_distribution<std::size_t> normal(0, d.size() - 1);
    std::mt19937 engine;
    auto gen = std::bind(std::ref(normal), std::ref(engine));

    meter.measure([&, gen] { return lookup_test(d, gen); });
})

NONIUS_BENCHMARK("small umap lookup", [](nonius::chronometer meter) {
    auto d = build_map<std::unordered_map<int, int>>(small_lookup_test_size);

    std::uniform_int_distribution<std::size_t> normal(0, d.size() - 1);
    std::mt19937 engine;
    auto gen = std::bind(std::ref(normal), std::ref(engine));

    meter.measure([&, gen] { return lookup_test(d, gen); });
})

#ifdef WITH_GOOGLE_BENCH
NONIUS_BENCHMARK("small google lookup", [](nonius::chronometer meter) {
    auto d =
        build_map_google<google::dense_hash_map<int, int>>(small_lookup_test_size);

    std::uniform_int_distribution<std::size_t> normal(0, d.size() - 1);
    std::mt19937 engine;
    auto gen = std::bind(std::ref(normal), std::ref(engine));

    meter.measure([&, gen] { return lookup_test(d, gen); });
})
#endif

const int lookup_test_size = 1000000;

NONIUS_BENCHMARK("dict lookup", [](nonius::chronometer meter) {
    auto d = build_map<boost::dict<int, int>>(lookup_test_size);

    std::uniform_int_distribution<std::size_t> normal(0, d.size() - 1);
    std::mt19937 engine;
    auto gen = std::bind(std::ref(normal), std::ref(engine));

    meter.measure([&, gen] { return lookup_test(d, gen); });
})

NONIUS_BENCHMARK("umap lookup", [](nonius::chronometer meter) {
    auto d = build_map<std::unordered_map<int, int>>(lookup_test_size);

    std::uniform_int_distribution<std::size_t> normal(0, d.size() - 1);
    std::mt19937 engine;
    auto gen = std::bind(std::ref(normal), std::ref(engine));
{'fraction_non_alphanumeric': 0.17189500640204866, 'fraction_numerical': 0.020326504481434058, 'mean_word_length': 3.844186046511628, 'pattern_counts': {'":': 0, '<': 54, '<?xml version=': 0, '>': 52, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2051807', 'n_tokens_mistral': 2526, 'n_tokens_neox': 2376, 'n_words': 515}