<commit_before>#include <assert.h>

#include "fnv_1a.h"

uint128_t fnv_1a(const void * restrict const buf,
                 const size_t                len,
                 const size_t                skip_pos,
                 const size_t                skip_len)
{
    assert((skip_pos <= len) && (skip_pos + skip_len <= len));

    static const uint128_t prime =
        (((uint128_t)0x0000000001000000) << 64) | 0x000000000000013B;
    uint128_t hash =
        (((uint128_t)0x6C62272E07BB0142) << 64) | 0x62B821756295C58D;

    // two consecutive loops should be faster than one loop with an "if"
    const uint8_t * restrict const bytes = buf;
    for (size_t i = 0; i < skip_pos; i++) {
        hash ^= bytes[i];
        hash *= prime;
    }
    for (size_t i = skip_pos + skip_len; i < len; i++) {
        hash ^= bytes[i];
        hash *= prime;
    }
    return hash;
}
<commit_msg>Use the new assert() macro<commit_after>#include "fnv_1a.h"
#include "debug.h"

uint128_t fnv_1a(const void * restrict const buf,
                 const size_t                len,
                 const size_t                skip_pos,
                 const size_t                skip_len)
{
    assert((skip_pos <= len) && (skip_pos + skip_len <= len),
           "len %zu, skip_pos %zu, skip_len %zu", len, skip_pos, skip_len);

    static const uint128_t prime =
        (((uint128_t)0x0000000001000000) << 64) | 0x000000000000013B;
    uint128_t hash =
        (((uint128_t)0x6C62272E07BB0142) << 64) | 0x62B821756295C58D;

    // two consecutive loops should be faster than one loop with an "if"
    const uint8_t * restrict const bytes = buf;
    for (size_t i = 0; i < skip_pos; i++) {
        hash ^= bytes[i];
        hash *= prime;
    }
    for (size_t i = skip_pos + skip_len; i < len; i++) {
        hash ^= bytes[i];
        hash *= prime;
    }
    return hash;
}

{'fraction_non_alphanumeric': 0.11378205128205128, 'fraction_numerical': 0.08867521367521368, 'mean_word_length': 2.0016025641025643, 'pattern_counts': {'":': 0, '<': 20, '<?xml version=': 0, '>': 4, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 4, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6044840', 'n_tokens_mistral': 754, 'n_tokens_neox': 633, 'n_words': 171}