<commit_before>
#include <stdio.h>
#include <string.h>
#include <windows.h>
#include <intrin.h>
#include <time.h>


unsigned __int64 djb(const char *str, size_t len);

static __forceinline unsigned __int64 djb_c(const char *str, size_t len)
{
        register unsigned __int64 hash = 5381;

        /* variant with the hash unrolled eight times */
        for (; len >= 8; len -= 8) {
                hash = ((hash << 5) + hash) + *str++;
                hash = ((hash << 5) + hash) + *str++;
                hash = ((hash << 5) + hash) + *str++;
                hash = ((hash << 5) + hash) + *str++;
                hash = ((hash << 5) + hash) + *str++;
                hash = ((hash << 5) + hash) + *str++;
                hash = ((hash << 5) + hash) + *str++;
                hash = ((hash << 5) + hash) + *str++;
        }
        switch (len) {
                case 7: hash = ((hash << 5) + hash) + *str++; /* fallthrough... */
                case 6: hash = ((hash << 5) + hash) + *str++; /* fallthrough... */
                case 5: hash = ((hash << 5) + hash) + *str++; /* fallthrough... */
                case 4: hash = ((hash << 5) + hash) + *str++; /* fallthrough... */
                case 3: hash = ((hash << 5) + hash) + *str++; /* fallthrough... */
                case 2: hash = ((hash << 5) + hash) + *str++; /* fallthrough... */
                case 1: hash = ((hash << 5) + hash) + *str++; break;
                case 0: break;
		default: __assume(0);
        }
        return hash;
}

/*unsigned __int64 djb_c(const char *str, size_t len)
{
	size_t cnt = 0;
	unsigned __int64 hash = 5381;

	for (; cnt < len; cnt++) {
		hash = (hash << 5) + hash + str[cnt];
	}

	return hash;
}*/

void
init_rand(void)
{
	static __int64 start = 0;
	__int64 freq, end;

	if (!QueryPerformanceFrequency((LARGE_INTEGER *)&freq)) {
		return;
	}

	if (0 == start) {
		QueryPerformanceCounter((LARGE_INTEGER *)&start);
		srand((unsigned)time(NULL));
	} else {
		QueryPerformanceCounter((LARGE_INTEGER *)&end);
		srand((unsigned)((end - start)+rand()));
		start = end;
	}
}

#define LEN 64

static void
testit(double *diff1)
{
	char *hello;
	unsigned __int64 h0 = 0, h1 = 0;

	__int64 freq, start, end;
	double elapsed0, elapsed1;
	int i = 0;
	size_t len;
	FILETIME tod;

	init_rand();

	len = LEN;

	//len = rand()*rand();
	//len = rand();

	hello = (char *)malloc(sizeof(char) * len);
	memset(hello, 'X', len);

	if (!QueryPerformanceFrequency((LARGE_INTEGER *)&freq)) {
		return;
	}

	QueryPerformanceCounter((LARGE_INTEGER *)&start);
	h0 = djb(hello, len);
	QueryPerformanceCounter((LARGE_INTEGER *)&end);
	elapsed0 = ((end - start)*1000000)/freq;

	QueryPerformanceCounter((LARGE_INTEGER *)&start);
	h1 = djb_c(hello, len);
	QueryPerformanceCounter((LARGE_INTEGER *)&end);
	elapsed1 = ((end - start)*1000000)/freq;

	*diff1 = elapsed0-elapsed1;<commit_msg>add random string generator<commit_after>
#include <stdio.h>
#include <string.h>
#include <windows.h>
#include <intrin.h>
#include <time.h>


/* asm implementation */
unsigned __int64 djb(const char *str, size_t len);

/* c implementation with unroll */
static __forceinline unsigned __int64 djb_c(const char *str, size_t len)
{
        register unsigned __int64 hash = 5381;

        /* variant with the hash unrolled eight times */
        for (; len >= 8; len -= 8) {
                hash = ((hash << 5) + hash) + *str++;
                hash = ((hash << 5) + hash) + *str++;
                hash = ((hash << 5) + hash) + *str++;
                hash = ((hash << 5) + hash) + *str++;
                hash = ((hash << 5) + hash) + *str++;
                hash = ((hash << 5) + hash) + *str++;
                hash = ((hash << 5) + hash) + *str++;
                hash = ((hash << 5) + hash) + *str++;
        }
        switch (len) {
                case 7: hash = ((hash << 5) + hash) + *str++; /* fallthrough... */
                case 6: hash = ((hash << 5) + hash) + *str++; /* fallthrough... */
                case 5: hash = ((hash << 5) + hash) + *str++; /* fallthrough... */
                case 4: hash = ((hash << 5) + hash) + *str++; /* fallthrough... */
                case 3: hash = ((hash << 5) + hash) + *str++; /* fallthrough... */
                case 2: hash = ((hash << 5) + hash) + *str++; /* fallthrough... */
                case 1: hash = ((hash << 5) + hash) + *str++; break;
                case 0: break;
		default: __assume(0);
        }
        return hash;
}

/* simple c implementation */
/*unsigned __int64 djb_c(const char *str, size_t len)
{
	size_t cnt = 0;
	unsigned __int64 hash = 5381;

	for (; cnt < len; cnt++) {
		hash = (hash << 5) + hash + str[cnt];
	}

	return hash;
}*/

/* helpers */
void
init_rand(void)
{
	static __int64 start = 0;
	__int64 freq, end;

	if (!QueryPerformanceFrequency((LARGE_INTEGER *)&freq)) {
		return;
	}

	if (0 == start) {
		QueryPerformanceCounter((LARGE_INTEGER *)&start);
		srand((unsigned)time(NULL));
	} else {
		QueryPerformanceCounter((LARGE_INTEGER *)&end);
		srand((unsigned)((end - start)+rand()));
		start = end;
	}
}

#define LEN 64
char *
get_random_str(size_t *len)
{
	char *hello;
	size_t i;

	*len = LEN;
	init_rand();
	//*len = rand();

	hello = (char *)malloc(sizeof(char) * *len);

	for (i = 0; i < *len; i++) {
		hello[i] = 'A' + (rand() % 26);
	}

	return hello;
}


/* test */

static void
testit(double *diff1)
{
	char *hello;
	size_t len;
	unsigned __int64 h0 = 0, h1 = 0;
	__int64 freq, start, end;
	double elapsed0, elapsed1;
	int i = 0;
	FILETIME tod;

	hello = get_random_str(&len);

	if (!QueryPerformanceFrequency((LARGE_INTEGER *)&freq)) {
		return;
	}

	QueryPerformanceCounter((LARGE_INTEGER *)&start);
	h0 = djb(hello, len);
	QueryPerformanceCounter((LARGE_INTEGER *)&end);
	elapsed0 = ((end - start)*1000000)/freq;

	QueryPerformanceCounter((LARGE_INTEGER *)&start);
	h1 = djb_c(hello, len);
	QueryPerformanceCounter((LARGE_INTEGER *)&end);
	elapsed1 = ((end - start)*1000000)/freq;

	*diff1 = elapsed0-elapsed1;
{'fraction_non_alphanumeric': 0.1858792036138531, 'fraction_numerical': 0.029613518487535552, 'mean_word_length': 2.45748987854251, 'pattern_counts': {'":': 0, '<': 80, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7080708', 'n_tokens_mistral': 2302, 'n_tokens_neox': 2081, 'n_words': 567}