<commit_before>
#pragma once

#include <stdint.h>

// -----------------------------------------------------------------------------

// murmur 32 bit hashes for small inputs

uint32_t murmur4 (uint32_t data); // 4 bytes
uint32_t murmur8 (uint64_t data); // 8 bytes
uint32_t murmur16(uint64_t data1, uint64_t data2); // 16 bytes

// -----------------------------------------------------------------------------

#ifdef MURMUR_IMPLEMENTATION

// based on https://github.com/aappleby/smhasher/blob/master/src/MurmurHash3.cpp
// MurmurHash3 was written by <NAME>, and is placed in the public
// domain. The author hereby disclaims copyright to this source code.

#if defined(_MSC_VER)
#include <stdlib.h>
#define _ROTL32(x,y) _rotl(x, y)
#else
inline uint32_t _rotl32(uint32_t x, int8_t r) { return (x << r) | (x >> (32 - r)); }
#define _ROTL32(x,y) _rotl32(x, y)
#endif

#ifndef _MURMUR_SEED
#define _MURMUR_SEED 0x5f3759df
#endif

uint32_t murmur4(uint32_t data)
{
	uint32_t h1 = _MURMUR_SEED;
	uint32_t k1 = data;
	k1 *= 0xcc9e2d51; // round
	k1  = _ROTL32(k1, 15);
	k1 *= 0x1b873593;
	h1 ^= k1;
	h1  = _ROTL32(h1, 13);
	h1  = h1 * 5 + 0xe6546b64;
	h1 ^= 4;        // len
	h1 ^= h1 >> 16; // fmix32
	h1 *= 0x85ebca6b;
	h1 ^= h1 >> 13;
	h1 *= 0xc2b2ae35;
	h1 ^= h1 >> 16;
	return h1;
}

uint32_t murmur8(uint64_t data)
{
	uint32_t h1 = _MURMUR_SEED;
	uint32_t k1 = ((uint32_t*)&data)[0];
	k1 *= 0xcc9e2d51; // round
	k1  = _ROTL32(k1, 15);
	k1 *= 0x1b873593;
	h1 ^= k1;
	h1  = _ROTL32(h1, 13);
	h1  = h1 * 5 + 0xe6546b64;
	k1 = ((uint32_t*)&data)[1];
	k1 *= 0xcc9e2d51; // round
	k1  = _ROTL32(k1, 15);
	k1 *= 0x1b873593;
	h1 ^= k1;
	h1  = _ROTL32(h1, 13);
	h1  = h1 * 5 + 0xe6546b64;
	h1 ^= 8; // len
	h1 ^= h1 >> 16; // fmix32
	h1 *= 0x85ebca6b;
	h1 ^= h1 >> 13;
	h1 *= 0xc2b2ae35;
	h1 ^= h1 >> 16;
	return h1;
}

uint32_t murmur16(uint64_t data1, uint64_t data2)
{
	uint32_t h1 = _MURMUR_SEED;
	uint32_t k1 = ((uint32_t*)&data1)[0];
	k1 *= 0xcc9e2d51; // round
	k1  = _ROTL32(k1, 15);
	k1 *= 0x1b873593;
	h1 ^= k1;
	h1  = _ROTL32(h1, 13);
	h1  = h1 * 5 + 0xe6546b64;
	k1 = ((uint32_t*)&data1)[1];
	k1 *= 0xcc9e2d51; // round
	k1  = _ROTL32(k1, 15);
	k1 *= 0x1b873593;
	h1 ^= k1;
	h1  = _ROTL32(h1, 13);
	h1  = h1 * 5 + 0xe6546b64;
	k1 = ((uint32_t*)&data2)[0];
	k1 *= 0xcc9e2d51; // round
	k1  = _ROTL32(k1, 15);
	k1 *= 0x1b873593;
	h1 ^= k1;
	h1  = _ROTL32(h1, 13);
	h1  = h1 * 5 + 0xe6546b64;
	k1 = ((uint32_t*)&data2)[1];
	k1 *= 0xcc9e2d51; // round
	k1  = _ROTL32(k1, 15);
	k1 *= 0x1b873593;
	h1 ^= k1;
	h1  = _ROTL32(h1, 13);
	h1  = h1 * 5 + 0xe6546b64;
	h1 ^= 16; // len
	h1 ^= h1 >> 16; // fmix32
	h1 *= 0x85ebca6b;
	h1 ^= h1 >> 13;
	h1 *= 0xc2b2ae35;
	h1 ^= h1 >> 16;
	return h1;
}

#endif<commit_msg>Added 16-byte array overload. Small optimizations.<commit_after>
#pragma once

#include <stdint.h>

// -----------------------------------------------------------------------------

// murmur 32 bit hashes for small inputs

uint32_t murmur4 (uint32_t data); // 4 bytes
uint32_t murmur8 (uint64_t data); // 8 bytes
uint32_t murmur16(uint64_t data1, uint64_t data2); // 16 bytes
uint32_t murmur16_array(const uint32_t* data); // 16 bytes

// -----------------------------------------------------------------------------

#ifdef MURMUR_IMPLEMENTATION

// based on https://github.com/aappleby/smhasher/blob/master/src/MurmurHash3.cpp
// MurmurHash3 was written by <NAME>, and is placed in the public
// domain. The author hereby disclaims copyright to this source code.

#if defined(_MSC_VER)
#include <stdlib.h>
#define _ROTL32(x,y) _rotl(x, y)
#else
inline uint32_t _rotl32(uint32_t x, int8_t r) { return (x << r) | (x >> (32 - r)); }
#define _ROTL32(x,y) _rotl32(x, y)
#endif

#ifndef _MURMUR_SEED
#define _MURMUR_SEED 0x5f3759df
#endif

uint32_t murmur4(uint32_t data)
{
	uint32_t h1 = _MURMUR_SEED;
	uint32_t k1 = data;
	k1 *= 0xcc9e2d51; // round
	k1  = _ROTL32(k1, 15);
	k1 *= 0x1b873593;
	h1 ^= k1;
	h1  = _ROTL32(h1, 13);
	h1  = h1 * 5 + 0xe6546b64;
	h1 ^= 4;        // len
	h1 ^= h1 >> 16; // fmix32
	h1 *= 0x85ebca6b;
	h1 ^= h1 >> 13;
	h1 *= 0xc2b2ae35;
	h1 ^= h1 >> 16;
	return h1;
}

uint32_t murmur8(uint64_t data)
{
	uint32_t data_high = -1, data_low = -1;
	data_high &= data >> 32;
	data_low &= data;

	uint32_t h1 = _MURMUR_SEED;
	uint32_t k1 = data_high;
	k1 *= 0xcc9e2d51; // round
	k1  = _ROTL32(k1, 15);
	k1 *= 0x1b873593;
	h1 ^= k1;
	h1  = _ROTL32(h1, 13);
	h1  = h1 * 5 + 0xe6546b64;
	k1 = data_low;
	k1 *= 0xcc9e2d51; // round
	k1  = _ROTL32(k1, 15);
	k1 *= 0x1b873593;
	h1 ^= k1;
	h1  = _ROTL32(h1, 13);
	h1  = h1 * 5 + 0xe6546b64;
	h1 ^= 8; // len
	h1 ^= h1 >> 16; // fmix32
	h1 *= 0x85ebca6b;
	h1 ^= h1 >> 13;
	h1 *= 0xc2b2ae35;
	h1 ^= h1 >> 16;
	return h1;
}

uint32_t murmur16(uint64_t data1, uint64_t data2)
{
	uint32_t data1_high = -1, data1_low = -1, data2_high = -1, data2_low = -1;
	data1_high &= data1 >> 32;
	data1_low &= data1;

	data2_high &= data2 >> 32;
	data2_low &= data2;

	uint32_t h1 = _MURMUR_SEED;
	uint32_t k1 = data1_high;
	k1 *= 0xcc9e2d51; // round
	k1  = _ROTL32(k1, 15);
	k1 *= 0x1b873593;
	h1 ^= k1;
	h1  = _ROTL32(h1, 13);
	h1  = h1 * 5 + 0xe6546b64;
	k1 = data1_low;
	k1 *= 0xcc9e2d51; // round
	k1  = _ROTL32(k1, 15);
	k1 *= 0x1b873593;
	h1 ^= k1;
	h1  = _ROTL32(h1, 13);
	h1  = h1 * 5 + 0xe6546b64;
	k1 = data2_high;
	k1 *= 0xcc9e2d51; // round
	k1  = _ROTL32(k1, 15);
	k1 *= 0x1b873593;
	h1 ^= k1;
	h1  = _ROTL32(h1, 13);
	h1  = h1 * 5 + 0xe6546b64;
	k1 = data2_low;
	k1 *= 0xcc9e2d51; // round
	k1  = _ROTL32(k1, 15);
	k1 *= 0x1b873593;
	h1 ^= k1;
	h1  = _ROTL32(h1, 13);
	h1  = h1 * 5 + 0xe6546b64;
	h1 ^= 16; // len
	h1 ^= h1 >> 16; // fmix32
	h1 *= 0x85ebca6b;
	h1 ^= h1 >> 13;
	h1 *= 0xc2b2ae35;
	h1 ^= h1 >> 16;
	return h1;
}

uint32_t murmur16_array(const uint32_t* data)
{
	uint32_t h1 = _MURMUR_SEED;
	uint32_t k1 = data[0];
	k1 *= 0xcc9e2d51; // round
	k1  = _ROTL32(k1, 15);
	k1 *= 0x1b873593;
	h1 ^= k1;
	h1  = _ROTL32(h1, 13);
	h1  = h1 * 5 + 0xe6546b64;
	k1 = data[1];
	k1 *= 0xcc9e2d51; // round
	k1  = _ROTL32(k1, 15);
	k1 *= 0x1b873593;
	h1 ^= k1;
	h1  = _ROTL32(h1, 13);
	h1  = h1 * 5 + 0xe6546b64;
	k1 = data[2];
	k1 *= 0xcc9e2d51; // round
	k1  = _ROTL32(k1, 15);
	k1 *= 0x1b873593;
	h1 ^= k1;
	h1  = _ROTL32(h1, 13);
	h1  = h1 * 5 + 0xe6546b64;
	k1 = data[3];
	k1 *= 0xcc9e2d51; // round
	k1  = _ROTL32(k1, 15);
	k1 *= 0x1b873593;
	h1 ^= k1;
	h1  = _ROTL32(h1, 13);
	h1  = h1 * 5 + 0xe6546b64;
	h1 ^= 16; // len
	h1 ^= h1 >> 16; // fmix32
	h1 *= 0x85ebca6b;
	h1 ^= h1 >> 13;
	h1 *= 0xc2b2ae35;
	h1 ^= h1 >> 16;
	return h1;
}

#endif
{'fraction_non_alphanumeric': 0.20635164160785516, 'fraction_numerical': 0.17888922982509972, 'mean_word_length': 2.5956977385548816, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 61, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7339594', 'n_tokens_mistral': 3970, 'n_tokens_neox': 3500, 'n_words': 748}