<commit_before>#ifndef __MATH_H__
#define __MATH_H__

#include <debug.h>

static inline uint lg(uint x)  __attribute__ ((pure));
static inline uint short_lg(uint x)  __attribute__ ((pure));

static inline uint lg(uint x) {

    uint r = 0;

    if (x & 0xFFFF0000) { r += 16; x >>= 16; }
    if (x & 0x0000FF00) { r +=  8; x >>=  8; }
    if (x & 0x000000F0) { r +=  4; x >>=  4; }

    switch (x) {
    case 0:
	return r;
    case 1:
	return r + 1;
    case 2:
    case 3:
	return r + 2;
    case 4:
    case 5:
    case 6:
    case 7:
	return r + 3;
    case 8:
    case 9:
    case 10:
    case 11:
    case 12:
    case 13:
    case 14:
    case 15:
	return r + 4;
    };

    assert(false, "lg bug!");
    return 0;
}

static inline uint short_lg(uint x) {

    uint r = 0;

    if (x & 0x0000FF00) { r +=  8; x >>=  8; }
    if (x & 0x000000F0) { r +=  4; x >>=  4; }

    switch (x) {
    case 0:
	return r;
    case 1:
	return r + 1;
    case 2:
    case 3:
	return r + 2;
    case 4:
    case 5:
    case 6:
    case 7:
	return r + 3;
    case 8:
    case 9:
    case 10:
    case 11:
    case 12:
    case 13:
    case 14:
    case 15:
	return r + 4;
    };

    debug_log("%u", x);
    assert(false, "short_lg bug!");
    return 0;
}

#endif
<commit_msg>Add a MOD macro to math.h to handle the base 2 modulo operations<commit_after>
#ifndef __MATH_H__
#define __MATH_H__

#include <debug.h>

#define MOD(x, y) (x & (y - 1))


static inline uint lg(uint x)  __attribute__ ((pure));
static inline uint lg(uint x) {

    uint r = 0;

    if (x & 0xFFFF0000) { r += 16; x >>= 16; }
    if (x & 0x0000FF00) { r +=  8; x >>=  8; }
    if (x & 0x000000F0) { r +=  4; x >>=  4; }

    switch (x) {
    case 0:
	return r;
    case 1:
	return r + 1;
    case 2:
    case 3:
	return r + 2;
    case 4:
    case 5:
    case 6:
    case 7:
	return r + 3;
    case 8:
    case 9:
    case 10:
    case 11:
    case 12:
    case 13:
    case 14:
    case 15:
	return r + 4;
    };

    assert(false, "lg bug!");
    return 0;
}

static inline uint short_lg(uint x)  __attribute__ ((pure));
static inline uint short_lg(uint x) {

    uint r = 0;

    if (x & 0x0000FF00) { r +=  8; x >>=  8; }
    if (x & 0x000000F0) { r +=  4; x >>=  4; }

    switch (x) {
    case 0:
	return r;
    case 1:
	return r + 1;
    case 2:
    case 3:
	return r + 2;
    case 4:
    case 5:
    case 6:
    case 7:
	return r + 3;
    case 8:
    case 9:
    case 10:
    case 11:
    case 12:
    case 13:
    case 14:
    case 15:
	return r + 4;
    };

    debug_log("%u", x);
    assert(false, "short_lg bug!");
    return 0;
}

#endif

{'fraction_non_alphanumeric': 0.14110898661567878, 'fraction_numerical': 0.07954110898661568, 'mean_word_length': 1.8342361863488623, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 25, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7269457', 'n_tokens_mistral': 1285, 'n_tokens_neox': 1088, 'n_words': 379}