<commit_before>/*
 * This file is part of hat-trie.
 *
 * Copyright (c) 2011 by <NAME> <<EMAIL>>
 *
 * See ahtable.h for description of the Array Hash Table.
 *
 */

#include "ahtable.h"
#include "misc.h"
#include "murmurhash3.h"
#include "portable_endian.h"
#include <assert.h>
#include <string.h>

const double ahtable_max_load_factor = 100000.0; /* arbitrary large number => don't resize */
const size_t ahtable_initial_size = 4096;

static size_t keylen(slot_t s) {
    if (0x1 & *s) {
        return (size_t) (*((uint16_t*) s) >> 1);
    }
    else {
        return (size_t) (*s >> 1);
    }
}


ahtable_t* ahtable_create()
{
    return ahtable_create_n(ahtable_initial_size);
}


ahtable_t* ahtable_create_n(size_t n)
{
    ahtable_t* table = malloc_or_die(sizeof(ahtable_t));
    table->flag = 0;
    table->c0 = table->c1 = '\0';

    table->n = n;
    table->m = 0;
    table->max_m = (size_t) (ahtable_max_load_factor * (double) table->n);
    table->slots = malloc_or_die(n * sizeof(slot_t));
    memset(table->slots, 0, n * sizeof(slot_t));

    table->slot_sizes = malloc_or_die(n * sizeof(size_t));
    memset(table->slot_sizes, 0, n * sizeof(size_t));

    return table;
}

void ahtable_save(const ahtable_t* table, FILE* fd)
{
    if (table == NULL) return;

    /* Store table metadata as 64-bit network-ordered (big-endian) values so
     * that architectures with larger capacity can take advantage of size.
     */
    uint64_t n = htobe64(table->n);
    fwrite(&n, sizeof(uint64_t), 1, fd);

    uint64_t m = htobe64(table->m);
    fwrite(&m, sizeof(uint64_t), 1, fd);

    uint64_t max_m = htobe64(table->max_m);
    fwrite(&max_m, sizeof(uint64_t), 1, fd);

    fwrite(&table->flag, sizeof(uint8_t), 1, fd);

    fwrite(&table->c0, sizeof(unsigned char), 1, fd);
    fwrite(&table->c1, sizeof(unsigned char), 1, fd);

    size_t i;
    uint32_t slot_size;
    for (i = 0; i < table->n; ++i) {
        slot_size = htobe32(table->slot_sizes[i]);
        fwrite(&slot_size, sizeof(uint32_t), 1, fd);
        if(table->slot_sizes[i] > 0) {
            fwrite(table->slots[i], sizeof(unsigned char), table->slot_sizes[i], fd);
        }
    }
}

/* Loads a 64-bit value from disk and casts it into a size_t, which may or may
 * not be 64-bit. As long as the loaded value fits inside size_t, we're good.
 * Returns 0 if the value didn't fit, 1 otherwise.
 */
static uint8_t read_u64bit_to_size_t(size_t* dest, FILE* fd)
{
    uint64_t value;
    fread(&value, sizeof(uint64_t), 1, fd);
    if (value > (size_t)-1) {
        printf("Unable to load 64-bit data from file\n");
        return 0;
    } else {
        *dest = (size_t)be64toh(value);
        return 1;
    }
}

ahtable_t* ahtable_load(FILE* fd)
{
    size_t n;
    if (!read_u64bit_to_size_t(&n, fd)) return NULL;
    ahtable_t* table = ahtable_create_n(n);

    if (!read_u64bit_to_size_t(&table->m, fd)) return NULL;

    if (!read_u64bit_to_size_t(&table->max_m, fd)) return NULL;

    fread(&table->flag, sizeof(uint8_t), 1, fd);

    fread(&table->c0, sizeof(unsigned char), 1, fd);
    fread(&table->c1, sizeof(unsigned char), 1, fd);

    size_t i;
    uint32_t slot_size;<commit_msg>Fix ahtable_load failure on 32bit systems. Fixes #31<commit_after>/*
 * This file is part of hat-trie.
 *
 * Copyright (c) 2011 by <NAME> <<EMAIL>>
 *
 * See ahtable.h for description of the Array Hash Table.
 *
 */

#include "ahtable.h"
#include "misc.h"
#include "murmurhash3.h"
#include "portable_endian.h"
#include <assert.h>
#include <string.h>

const double ahtable_max_load_factor = 100000.0; /* arbitrary large number => don't resize */
const size_t ahtable_initial_size = 4096;

static size_t keylen(slot_t s) {
    if (0x1 & *s) {
        return (size_t) (*((uint16_t*) s) >> 1);
    }
    else {
        return (size_t) (*s >> 1);
    }
}


ahtable_t* ahtable_create()
{
    return ahtable_create_n(ahtable_initial_size);
}


ahtable_t* ahtable_create_n(size_t n)
{
    ahtable_t* table = malloc_or_die(sizeof(ahtable_t));
    table->flag = 0;
    table->c0 = table->c1 = '\0';

    table->n = n;
    table->m = 0;
    table->max_m = (size_t) (ahtable_max_load_factor * (double) table->n);
    table->slots = malloc_or_die(n * sizeof(slot_t));
    memset(table->slots, 0, n * sizeof(slot_t));

    table->slot_sizes = malloc_or_die(n * sizeof(size_t));
    memset(table->slot_sizes, 0, n * sizeof(size_t));

    return table;
}

void ahtable_save(const ahtable_t* table, FILE* fd)
{
    if (table == NULL) return;

    /* Store table metadata as 64-bit network-ordered (big-endian) values so
     * that architectures with larger capacity can take advantage of size.
     */
    uint64_t n = htobe64(table->n);
    fwrite(&n, sizeof(uint64_t), 1, fd);

    uint64_t m = htobe64(table->m);
    fwrite(&m, sizeof(uint64_t), 1, fd);

    uint64_t max_m = htobe64(table->max_m);
    fwrite(&max_m, sizeof(uint64_t), 1, fd);

    fwrite(&table->flag, sizeof(uint8_t), 1, fd);

    fwrite(&table->c0, sizeof(unsigned char), 1, fd);
    fwrite(&table->c1, sizeof(unsigned char), 1, fd);

    size_t i;
    uint32_t slot_size;
    for (i = 0; i < table->n; ++i) {
        slot_size = htobe32(table->slot_sizes[i]);
        fwrite(&slot_size, sizeof(uint32_t), 1, fd);
        if(table->slot_sizes[i] > 0) {
            fwrite(table->slots[i], sizeof(unsigned char), table->slot_sizes[i], fd);
        }
    }
}

/* Loads a 64-bit value from disk and casts it into a size_t, which may or may
 * not be 64-bit. As long as the loaded value fits inside size_t, we're good.
 * Returns 0 if the value didn't fit, 1 otherwise.
 */
static uint8_t read_u64bit_to_size_t(size_t* dest, FILE* fd)
{
    uint64_t value;
    fread(&value, sizeof(uint64_t), 1, fd);
    value = be64toh(value);
    if (value > (size_t)-1) {
        printf("Unable to load 64-bit data from file\n");
        return 0;
    } else {
        *dest = (size_t)value;
        return 1;
    }
}

ahtable_t* ahtable_load(FILE* fd)
{
    size_t n;
    if (!read_u64bit_to_size_t(&n, fd)) return NULL;
    ahtable_t* table = ahtable_create_n(n);

    if (!read_u64bit_to_size_t(&table->m, fd)) return NULL;

    if (!read_u64bit_to_size_t(&table->max_m, fd)) return NULL;

    fread(&table->flag, sizeof(uint8_t), 1, fd);

    fread(&table->c0, sizeof(unsigned char), 1, fd);
    fread(&table->c1, sizeof(unsigned char), 1, fd);

    size_t i;
    uint32_t slot_size;
{'fraction_non_alphanumeric': 0.150093808630394, 'fraction_numerical': 0.032520325203252036, 'mean_word_length': 3.230820105820106, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 81, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7006226', 'n_tokens_mistral': 2677, 'n_tokens_neox': 2490, 'n_words': 608}