<commit_before>#include "tl_hashmap.h"

#include <stdlib.h>
#include <string.h>
#include <limits.h>



#ifdef TL_ALLIGN_MEMORY
    #define PADDING sizeof(void*)
    #define ALLIGN( ptr )\
            if( ((size_t)(ptr)) % PADDING )\
                (ptr) += PADDING - (((size_t)(ptr)) % PADDING)
#else
    #define PADDING 0
    #define ALLIGN( ptr )
#endif



typedef struct
{
    size_t binsize;
    size_t index;
    int used;
    tl_hashmap_entry* ent;
}
entrydata;

static void get_entry_data( const tl_hashmap* this, entrydata* ent,
                            const void* key )
{
    ent->binsize  = sizeof(tl_hashmap_entry) + this->keysize + this->objsize;
    ent->binsize += 2 * PADDING;

    ent->index = this->hash( key ) % this->bincount;

    ent->ent = (tl_hashmap_entry*)((char*)this->bins+ent->index*ent->binsize);
    ent->used = this->bitmap[ ent->index / (sizeof(int)*CHAR_BIT) ];
    ent->used = (ent->used >> (ent->index % (sizeof(int)*CHAR_BIT))) & 0x01;
}

/****************************************************************************/

int tl_hashmap_init( tl_hashmap* this, size_t keysize, size_t objsize,
                     size_t bincount, tl_hash keyhash, tl_compare keycompare )
{
    size_t i, binsize, mapcount;
    char* ptr;

    /* sanity check */
    if( !this || !keysize || !objsize || !bincount )
        return 0;

    if( !keyhash || !keycompare )
        return 0;

    /* allocate bins */
    binsize = sizeof(tl_hashmap_entry) + keysize + objsize + 2*PADDING;
    this->bins = malloc( binsize * bincount );

    if( !this->bins )
        return 0;

    /* allocate usage bitmap */
    mapcount = 1 + ((sizeof(int)*CHAR_BIT) / bincount);
    this->bitmap = malloc( mapcount * sizeof(int) );

    if( !this->bitmap )
    {
        free( this->bins );<commit_msg>[fix] hash map bitmap size<commit_after>#include "tl_hashmap.h"

#include <stdlib.h>
#include <string.h>
#include <limits.h>



#ifdef TL_ALLIGN_MEMORY
    #define PADDING sizeof(void*)
    #define ALLIGN( ptr )\
            if( ((size_t)(ptr)) % PADDING )\
                (ptr) += PADDING - (((size_t)(ptr)) % PADDING)
#else
    #define PADDING 0
    #define ALLIGN( ptr )
#endif



typedef struct
{
    size_t binsize;
    size_t index;
    int used;
    tl_hashmap_entry* ent;
}
entrydata;

static void get_entry_data( const tl_hashmap* this, entrydata* ent,
                            const void* key )
{
    ent->binsize  = sizeof(tl_hashmap_entry) + this->keysize + this->objsize;
    ent->binsize += 2 * PADDING;

    ent->index = this->hash( key ) % this->bincount;

    ent->ent = (tl_hashmap_entry*)((char*)this->bins+ent->index*ent->binsize);
    ent->used = this->bitmap[ ent->index / (sizeof(int)*CHAR_BIT) ];
    ent->used = (ent->used >> (ent->index % (sizeof(int)*CHAR_BIT))) & 0x01;
}

/****************************************************************************/

int tl_hashmap_init( tl_hashmap* this, size_t keysize, size_t objsize,
                     size_t bincount, tl_hash keyhash, tl_compare keycompare )
{
    size_t i, binsize, mapcount;
    char* ptr;

    /* sanity check */
    if( !this || !keysize || !objsize || !bincount )
        return 0;

    if( !keyhash || !keycompare )
        return 0;

    /* allocate bins */
    binsize = sizeof(tl_hashmap_entry) + keysize + objsize + 2*PADDING;
    this->bins = malloc( binsize * bincount );

    if( !this->bins )
        return 0;

    /* allocate usage bitmap */
    mapcount = 1 + (bincount / (sizeof(int)*CHAR_BIT));
    this->bitmap = malloc( mapcount * sizeof(int) );

    if( !this->bitmap )
    {
        free( this->bins );
{'fraction_non_alphanumeric': 0.18658570245652775, 'fraction_numerical': 0.005520287054926856, 'mean_word_length': 2.778936392075078, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 57, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4282074', 'n_tokens_mistral': 1327, 'n_tokens_neox': 1237, 'n_words': 282}