<commit_before>#include "hash.hpp"
#include "profiling.hpp"

namespace clover {
namespace util {

uint32 byteHash32(const uint8* buf, SizeType size){
	// Modified FNV-1a
	uint32 hash= 2166136261;
	for (SizeType i= 0; i < size; ++i){
		hash= ((hash ^ buf[i]) + 379721)*16777619;
	}
	return hash;
}

uint32 Hash32<real32>::operator()(real32 value) const {
	union { real32 f; uint32 i; } u;
	u.f= value;
	return u.i;
}

uint32 Hash32<real64>::operator()(real64 value) const {
	union { real64 f; uint32 i[2]; } u;
	u.f= value;
	return u.i[0]^u.i[1];
}

} // util
} // clover
<commit_msg>Duff's device seems to be helpful for hash function<commit_after>#include "hash.hpp"
#include "profiling.hpp"

namespace clover {
namespace util {

uint32 byteHash32(const uint8* buf, SizeType size){
#define USE_DUFF_OPTIMZ 1
	PROFILE();

	// Modified FNV-1a
	uint32 hash= 2166136261;

#if USE_DUFF_OPTIMZ
#define HASH \
	do { hash= ((hash ^ *buf++) + 379721)*16777619; } while (0)
	if (size == 0)
		return hash;
	SizeType n= (size + 7)/8;
	switch (size % 8) {
		case 0: do { HASH;
		case 7: HASH;
		case 6: HASH;
		case 5: HASH;
		case 4: HASH;
		case 3: HASH;
		case 2: HASH;
		case 1: HASH; } while (--n > 0);
	}	
#undef HASH
#else
	for (SizeType i= 0; i < size; ++i){
		hash= ((hash ^ buf[i]) + 379721)*16777619;
	}
#endif

	return hash;
}

uint32 Hash32<real32>::operator()(real32 value) const {
	union { real32 f; uint32 i; } u;
	u.f= value;
	return u.i;
}

uint32 Hash32<real64>::operator()(real64 value) const {
	union { real64 f; uint32 i[2]; } u;
	u.f= value;
	return u.i[0]^u.i[1];
}

} // util
} // clover

{'fraction_non_alphanumeric': 0.16677076826983137, 'fraction_numerical': 0.09306683322923173, 'mean_word_length': 3.697947214076246, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 8, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1846293', 'n_tokens_mistral': 786, 'n_tokens_neox': 682, 'n_words': 191}