<commit_before>#define _LARGEFILE64_SOURCE

#include <stdio.h>
#include <stdlib.h>
#include <unistd.h>
#include <sys/types.h>
#include <sys/stat.h>
#include <sys/time.h>
#include <fcntl.h>
#include <errno.h>
#include <string.h>
#include <libgen.h>

typedef unsigned char u8;
typedef unsigned int u32;
typedef unsigned long long u64;

u64 global_error_count = 0;

/* 8MB should be future proof for at least a couple of years...  */
#define DEFAULT_BLOCK_SIZE 8*1024*1024
unsigned int BLOCK_SIZE = 0;
u8 *buf_g; /* Generated */
u8 *buf_f; /* File */

void genranddata(u64 pos, unsigned int seed)
{
	unsigned int seed_short = (pos >> 32) ^ (pos & 0xffffffff) ^ seed;
	unsigned int i;

	srand(seed_short); rand(); rand();

	for (i=0; i<BLOCK_SIZE; i++)
		buf_g[i] = rand();
}

static void verifyblock(u64 pos, unsigned int seed, unsigned int size)
{
	unsigned int i;

	genranddata(pos, seed);

	for (i=0;i<size;i++) {
		if (buf_f[i] != buf_g[i]) {
			printf("\x1b[1;31m%#.8llx: MISMATCH: %#.2x != %#.2x\x1b[0m\n", pos+i, buf_f[i], buf_g[i]);
			global_error_count++;
		}
	}
}

static void showstatus(u64 pos)
{
	printf("%#.8llx...\r", pos);
}

struct showstatus_state
{<commit_msg>sdtest: Make random pattern block size agnostic<commit_after>#define _LARGEFILE64_SOURCE

#include <stdio.h>
#include <stdlib.h>
#include <unistd.h>
#include <sys/types.h>
#include <sys/stat.h>
#include <sys/time.h>
#include <fcntl.h>
#include <errno.h>
#include <string.h>
#include <libgen.h>
#include <assert.h>

typedef unsigned char u8;
typedef unsigned int u32;
typedef unsigned long long u64;

u64 global_error_count = 0;

/* 8MB should be future proof for at least a couple of years...  */
#define DEFAULT_BLOCK_SIZE 8*1024*1024
unsigned int BLOCK_SIZE = 0;
u8 *buf_g; /* Generated */
u8 *buf_f; /* File */
#define RAND_BLOCK_SIZE 4*1024 /* Data is seeded every 4K to be block size agnostic */

void _genranddata(u64 pos, u64 off, unsigned int seed)
{
	unsigned int seed_short = (pos >> 32) ^ (pos & 0xffffffff) ^ seed;
	unsigned int i;
	int tmp;

	assert(RAND_MAX > 1<<16); /* Assert that rand() gives us at least 2
				     bytes of random data (typically 31 bits,
				     but just using 16 simplifies the below
				     loop). The compiler should optimise this
				     out. */

	srand(seed_short); rand(); rand();

	for (i=0; i < RAND_BLOCK_SIZE; i+=2) {
		tmp = rand();
		buf_g[off+i  ] = (tmp & 0x000000ff)      ;
		buf_g[off+i+1] = (tmp & 0x0000ff00) >>  8;
	}
}

void genranddata(u64 pos, unsigned int seed)
{
	u64 off;

	assert(pos % RAND_BLOCK_SIZE == 0);
	assert(BLOCK_SIZE % RAND_BLOCK_SIZE == 0);

	for (off=0; off < BLOCK_SIZE; off += RAND_BLOCK_SIZE)
		_genranddata(pos+off, off, seed);
}

static void verifyblock(u64 pos, unsigned int seed, unsigned int size)
{
	unsigned int i;

	genranddata(pos, seed);

	for (i=0;i<size;i++) {
		if (buf_f[i] != buf_g[i]) {
			printf("\x1b[1;31m%#.8llx: MISMATCH: %#.2x != %#.2x\x1b[0m\n", pos+i, buf_f[i], buf_g[i]);
			global_error_count++;
		}
	}
}

static void showstatus(u64 pos)
{
	printf("%#.8llx...\r", pos);
}

struct showstatus_state
{
{'fraction_non_alphanumeric': 0.15606748864373785, 'fraction_numerical': 0.04153147306943543, 'mean_word_length': 4.021172638436482, 'pattern_counts': {'":': 0, '<': 31, '<?xml version=': 0, '>': 31, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 2, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3759813', 'n_tokens_mistral': 1367, 'n_tokens_neox': 1213, 'n_words': 337}