<commit_before>#include "base/stream_roller.h"

#include "core/aver.h"

#include "cr_constants.h"

using namespace roller;
using std::vector;
using std::pair;
using std::unique_ptr;

namespace cr {

// listStreamContents
vector<pair<i32, i64>> listStreamContents( void* pointer ) {
	ui8* buffer = (ui8*)pointer;

	// do some validation
	if ( buffer[0] != 0x73 || buffer[1] != 0x72 ) {
		throw RollerException( "Not a valid StreamRoller stream (invalid begin transmission designation)" );
	}

	if ( (buffer[3] != (ui8)CR_MAJOR_VERSION)
			|| (buffer[4] != (ui8)CR_MINOR_VERSION)
			|| (buffer[5] != (ui8)CR_PATCH_VERSION) ) {

		throw RollerException( "Stream is of a different version" );
	}

	ui32* numberAddress = (ui32*)(buffer + 14);
	ui32 numberOfObjects = *numberAddress;

	i64 objectOffset = 18;
	
	vector<pair<i32, i64>> objects;
	objects.reserve( numberOfObjects );
	for ( ui32 i=0; i<numberOfObjects; i++ ) {

		i64* objectSizeAddress = (i64*)(buffer + objectOffset);
		i64 objectSize = *objectSizeAddress;

		i32* objectDefinitionHashAddress = (i32*)(buffer + objectOffset + sizeof(i64));
		i32 objectDefinitionHash = *objectDefinitionHashAddress;

		objects.push_back( pair<i32, i64>( objectDefinitionHash, objectOffset + sizeof(i64) + sizeof(i32) ));
	}<commit_msg>Adjust buffer when reading stream contents<commit_after>#include "base/stream_roller.h"

#include "core/aver.h"

#include "cr_constants.h"

using namespace roller;
using std::vector;
using std::pair;
using std::unique_ptr;

namespace cr {

// listStreamContents
vector<pair<i32, i64>> listStreamContents( void* pointer ) {
	ui8* buffer = (ui8*)pointer;

	// do some validation
	if ( buffer[0] != 0x73 || buffer[1] != 0x72 ) {
		throw RollerException( "Not a valid StreamRoller stream (invalid begin transmission designation)" );
	}

	if ( (buffer[3] != (ui8)CR_MAJOR_VERSION)
			|| (buffer[4] != (ui8)CR_MINOR_VERSION)
			|| (buffer[5] != (ui8)CR_PATCH_VERSION) ) {

		throw RollerException( "Stream is of a different version" );
	}

	ui32* numberAddress = (ui32*)(buffer + 14);
	ui32 numberOfObjects = *numberAddress;

	i64 objectOffset = 18;
	
	vector<pair<i32, i64>> objects;
	objects.reserve( numberOfObjects );
	for ( ui32 i=0; i<numberOfObjects; i++ ) {

		i64* objectSizeAddress = (i64*)(buffer + objectOffset);
		i64 objectSize = *objectSizeAddress;

		i32* objectDefinitionHashAddress = (i32*)(buffer + objectOffset + sizeof(i64));
		i32 objectDefinitionHash = *objectDefinitionHashAddress;

		objects.push_back( pair<i32, i64>( objectDefinitionHash, objectOffset + sizeof(i64) + sizeof(i32) ));

		buffer += objectSize;
	}
{'fraction_non_alphanumeric': 0.14061302681992338, 'fraction_numerical': 0.04674329501915709, 'mean_word_length': 5.246411483253588, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7456759', 'n_tokens_mistral': 997, 'n_tokens_neox': 893, 'n_words': 211}