<commit_before>#include <bert/buffer.h>

#include "test.h"
#include <string.h>

#define OUTPUT_SIZE	(BERT_CHUNK_SIZE * 4)

int main()
{
	unsigned char data[BERT_CHUNK_SIZE];
	bert_buffer_t buffer;

	bert_buffer_init(&buffer);

	memset(data,'A',BERT_CHUNK_SIZE);
	bert_buffer_write(&buffer,data,BERT_CHUNK_SIZE);

	memset(data,'B',BERT_CHUNK_SIZE);
	bert_buffer_write(&buffer,data,BERT_CHUNK_SIZE);

	unsigned char output[OUTPUT_SIZE];
	size_t result;

	if ((result = bert_buffer_read(output,&buffer,OUTPUT_SIZE)) != (BERT_CHUNK_SIZE * 2))
	{
		test_fail("bert_buffer_read only read %u bytes, expected %u",result,OUTPUT_SIZE);
	}

	int i;

	for (i=0;i<BERT_CHUNK_SIZE;i++)
	{
		if (output[i] != 'A')
		{
			test_fail("bert_buffer_read return unexpected data");
		}
	}

	for (i=BERT_CHUNK_SIZE;i<(BERT_CHUNK_SIZE * 2);i++)
	{
		if (output[i] != 'B')
		{
			test_fail("bert_buffer_read return unexpected data");
		}
	}

	return 0;<commit_msg>Adjusted the buffer big read test to check where the buffer index lands.<commit_after>#include <bert/buffer.h>

#include "test.h"
#include <string.h>

#define OUTPUT_SIZE	(BERT_CHUNK_SIZE * 4)

int main()
{
	unsigned char data1[BERT_CHUNK_SIZE];
	unsigned char data2[BERT_CHUNK_SIZE / 2];
	bert_buffer_t buffer;

	bert_buffer_init(&buffer);

	memset(data1,'A',BERT_CHUNK_SIZE);
	bert_buffer_write(&buffer,data1,BERT_CHUNK_SIZE);

	memset(data2,'B',BERT_CHUNK_SIZE / 2);
	bert_buffer_write(&buffer,data2,BERT_CHUNK_SIZE / 2);

	unsigned char output[OUTPUT_SIZE];
	unsigned int output_length = (BERT_CHUNK_SIZE + (BERT_CHUNK_SIZE / 2));
	size_t result;

	if ((result = bert_buffer_read(output,&buffer,OUTPUT_SIZE)) != output_length)
	{
		test_fail("bert_buffer_read only read %u bytes, expected %u",result,output_length);
	}

	int i;

	for (i=0;i<BERT_CHUNK_SIZE;i++)
	{
		if (output[i] != 'A')
		{
			test_fail("bert_buffer_read return unexpected data");
		}
	}

	for (i=BERT_CHUNK_SIZE;i<(BERT_CHUNK_SIZE + (BERT_CHUNK_SIZE / 2));i++)
	{
		if (output[i] != 'B')
		{
			test_fail("bert_buffer_read return unexpected data");
		}
	}

	unsigned int expected_index = (BERT_CHUNK_SIZE / 2);

	if (buffer.index != expected_index)
	{
		test_fail("buffer index was set to %u, should be %u",buffer.index,expected_index);
	}

	return 0;
{'fraction_non_alphanumeric': 0.1431095406360424, 'fraction_numerical': 0.0088339222614841, 'mean_word_length': 5.62280701754386, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2460366', 'n_tokens_mistral': 961, 'n_tokens_neox': 932, 'n_words': 164}