<commit_before>/***************************************************************************
 *  containers/test_map.cpp
 *
 *  Part of the STXXL. See http://stxxl.sourceforge.net
 *
 *  Copyright (C) 2005, 2006 <NAME> <<EMAIL>>
 *
 *  Distributed under the Boost Software License, Version 1.0.
 *  (See accompanying file LICENSE_1_0.txt or copy at
 *  http://www.boost.org/LICENSE_1_0.txt)
 **************************************************************************/

#include <algorithm>
#include <cmath>
#include <stxxl/map>

typedef unsigned int key_type;
typedef unsigned int data_type;

struct cmp : public std::less<key_type>
{
    static key_type min_value()
    {
        return (std::numeric_limits<key_type>::min)();
    }
    static key_type max_value()
    {
        return (std::numeric_limits<key_type>::max)();
    }
};

#define BLOCK_SIZE (32 * 1024)
#define CACHE_SIZE (2 * 1024 * 1024 / BLOCK_SIZE)

#define CACHE_ELEMENTS (BLOCK_SIZE * CACHE_SIZE / (sizeof(key_type) + sizeof(data_type)))

typedef stxxl::map<key_type, data_type, cmp, BLOCK_SIZE, BLOCK_SIZE> map_type;

int main(int argc, char ** argv)
{
    stxxl::stats * bm = stxxl::stats::get_instance();
    STXXL_MSG(*bm);

    STXXL_MSG("Block size " << BLOCK_SIZE / 1024 << " kb");
    STXXL_MSG("Cache size " << (CACHE_SIZE * BLOCK_SIZE) / 1024 << " kb");
    int max_mult = 256;
    if (argc > 1)
        max_mult = atoi(argv[1]);
    for (int mult = 1; mult < max_mult; mult *= 2)
    {
        const unsigned el = mult * (CACHE_ELEMENTS / 8);
        STXXL_MSG("Elements to insert " << el << " volume =" <<
                  (el * (sizeof(key_type) + sizeof(data_type))) / 1024 << " kb");
        map_type * DMap = new map_type(CACHE_SIZE * BLOCK_SIZE / 2, CACHE_SIZE * BLOCK_SIZE / 2);
        //map_type  Map(CACHE_SIZE*BLOCK_SIZE/2,CACHE_SIZE*BLOCK_SIZE/2);
        map_type & Map = *DMap;
        for (unsigned i = 0; i < el; ++i)
        {
            Map[i] = i + 1;
        }

        double writes = double(bm->get_writes()) / double(el);
        double logel = log(double(el)) / log(double(BLOCK_SIZE));
        STXXL_MSG("Logs: writes " << writes << " logel " << logel << " writes/logel " << (writes / logel));
        STXXL_MSG(*bm);
        bm->reset();
        STXXL_MSG("Doing search");
        unsigned queries = el;
        const map_type & ConstMap = Map;
        stxxl::random_number32 myrandom;
        for (unsigned i = 0; i < queries; ++i)
        {
            key_type key = myrandom() % el;
            map_type::const_iterator result = ConstMap.find(key);
            assert((*result).second == key + 1);
        }
        double reads = double(bm->get_reads()) / logel;
        double readsperq = double(bm->get_reads()) / queries;
        STXXL_MSG("reads/logel " << reads << " readsperq " << readsperq);
        STXXL_MSG(*bm);
        bm->reset();
    }

    return 0;
}<commit_msg>fix another memory leak found by valgrind<commit_after>/***************************************************************************
 *  containers/test_map.cpp
 *
 *  Part of the STXXL. See http://stxxl.sourceforge.net
 *
 *  Copyright (C) 2005, 2006 <NAME> <<EMAIL>>
 *
 *  Distributed under the Boost Software License, Version 1.0.
 *  (See accompanying file LICENSE_1_0.txt or copy at
 *  http://www.boost.org/LICENSE_1_0.txt)
 **************************************************************************/

#include <algorithm>
#include <cmath>
#include <stxxl/map>

typedef unsigned int key_type;
typedef unsigned int data_type;

struct cmp : public std::less<key_type>
{
    static key_type min_value()
    {
        return (std::numeric_limits<key_type>::min)();
    }
    static key_type max_value()
    {
        return (std::numeric_limits<key_type>::max)();
    }
};

#define BLOCK_SIZE (32 * 1024)
#define CACHE_SIZE (2 * 1024 * 1024 / BLOCK_SIZE)

#define CACHE_ELEMENTS (BLOCK_SIZE * CACHE_SIZE / (sizeof(key_type) + sizeof(data_type)))

typedef stxxl::map<key_type, data_type, cmp, BLOCK_SIZE, BLOCK_SIZE> map_type;

int main(int argc, char ** argv)
{
    stxxl::stats * bm = stxxl::stats::get_instance();
    STXXL_MSG(*bm);

    STXXL_MSG("Block size " << BLOCK_SIZE / 1024 << " kb");
    STXXL_MSG("Cache size " << (CACHE_SIZE * BLOCK_SIZE) / 1024 << " kb");
    int max_mult = 256;
    if (argc > 1)
        max_mult = atoi(argv[1]);
    for (int mult = 1; mult < max_mult; mult *= 2)
    {
        const unsigned el = mult * (CACHE_ELEMENTS / 8);
        STXXL_MSG("Elements to insert " << el << " volume =" <<
                  (el * (sizeof(key_type) + sizeof(data_type))) / 1024 << " kb");
        map_type * DMap = new map_type(CACHE_SIZE * BLOCK_SIZE / 2, CACHE_SIZE * BLOCK_SIZE / 2);
        //map_type  Map(CACHE_SIZE*BLOCK_SIZE/2,CACHE_SIZE*BLOCK_SIZE/2);
        map_type & Map = *DMap;
        for (unsigned i = 0; i < el; ++i)
        {
            Map[i] = i + 1;
        }

        double writes = double(bm->get_writes()) / double(el);
        double logel = log(double(el)) / log(double(BLOCK_SIZE));
        STXXL_MSG("Logs: writes " << writes << " logel " << logel << " writes/logel " << (writes / logel));
        STXXL_MSG(*bm);
        bm->reset();
        STXXL_MSG("Doing search");
        unsigned queries = el;
        const map_type & ConstMap = Map;
        stxxl::random_number32 myrandom;
        for (unsigned i = 0; i < queries; ++i)
        {
            key_type key = myrandom() % el;
            map_type::const_iterator result = ConstMap.find(key);
            assert((*result).second == key + 1);
        }
        double reads = double(bm->get_reads()) / logel;
        double readsperq = double(bm->get_reads()) / queries;
        STXXL_MSG("reads/logel " << reads << " readsperq " << readsperq);
        STXXL_MSG(*bm);
        bm->reset();
        delete DMap;
    }

    return 0;
}
{'fraction_non_alphanumeric': 0.19218241042345277, 'fraction_numerical': 0.020572604148808505, 'mean_word_length': 2.8105813193990854, 'pattern_counts': {'":': 0, '<': 93, '<?xml version=': 0, '>': 35, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4975215', 'n_tokens_mistral': 2083, 'n_tokens_neox': 1913, 'n_words': 446}