<commit_before>#include <bitset>
#include "streams.h"

namespace augs {
	void stream::read(char* data, size_t bytes) {
		memcpy(data, buf.data() + pos, bytes);
		pos += bytes;
	}

	bool stream::operator==(const stream& b) const {
		return std::equal(data(), data() + size(), b.data(), b.data() + size());
	}

	char* stream::data() {
		return buf.data();
	}

	const char* stream::data() const {
		return buf.data();
	}

	void stream::write(const char* data, size_t bytes) {
		memcpy(buf.data() + pos, data, bytes);
		pos += bytes;
	}

	void output_stream_reserver::write(const char*, size_t bytes) {
		pos += bytes;
	}

	stream output_stream_reserver::make_stream() {
		stream reserved;
		reserved.reserve(pos);
		return std::move(reserved);
	}

	void stream::reserve(size_t bytes) {
		reset_pos();
		buf.resize(bytes);
	};
	
	void stream::reserve(const output_stream_reserver& r) {
		reserve(r.size());
	};

	void bits_stream::read_variable_length_index(unsigned short& idx) {
		std::bitset<4> index_bitcount;
		read(index_bitcount);

		auto bit_count = index_bitcount.to_ulong();

		std::bitset<16> decoded_index;
		read(decoded_index, bit_count);

		idx = decoded_index.to_ulong();
	}

	void bits_stream::write_variable_length_index(unsigned short idx) {
		if (idx == 0) {
			write(std::bitset<4>());
			return;
		}

		unsigned long mask = idx;
		unsigned long index;

		bool non_zero = _BitScanReverse(&index, mask);

		if (non_zero) {
			const unsigned bits_needed = index + 1;
			
			write(std::bitset<4>(bits_needed));
			write(std::bitset<16>(idx), bits_needed);
		}
	}
}<commit_msg>unnecessary if<commit_after>#include <bitset>
#include "streams.h"

namespace augs {
	void stream::read(char* data, size_t bytes) {
		memcpy(data, buf.data() + pos, bytes);
		pos += bytes;
	}

	bool stream::operator==(const stream& b) const {
		return std::equal(data(), data() + size(), b.data(), b.data() + size());
	}

	char* stream::data() {
		return buf.data();
	}

	const char* stream::data() const {
		return buf.data();
	}

	void stream::write(const char* data, size_t bytes) {
		memcpy(buf.data() + pos, data, bytes);
		pos += bytes;
	}

	void output_stream_reserver::write(const char*, size_t bytes) {
		pos += bytes;
	}

	stream output_stream_reserver::make_stream() {
		stream reserved;
		reserved.reserve(pos);
		return std::move(reserved);
	}

	void stream::reserve(size_t bytes) {
		reset_pos();
		buf.resize(bytes);
	};
	
	void stream::reserve(const output_stream_reserver& r) {
		reserve(r.size());
	};

	void bits_stream::read_variable_length_index(unsigned short& idx) {
		std::bitset<4> index_bitcount;
		read(index_bitcount);

		auto bit_count = index_bitcount.to_ulong();

		std::bitset<16> decoded_index;
		read(decoded_index, bit_count);

		idx = decoded_index.to_ulong();
	}

	void bits_stream::write_variable_length_index(unsigned short idx) {
		if (idx == 0) {
			write(std::bitset<4>());
			return;
		}

		unsigned long mask = idx;
		unsigned long index;

		bool non_zero = _BitScanReverse(&index, mask);

		const unsigned bits_needed = index + 1;
		
		write(std::bitset<4>(bits_needed));
		write(std::bitset<16>(idx), bits_needed);
	}
}
{'fraction_non_alphanumeric': 0.1591125198098257, 'fraction_numerical': 0.005705229793977813, 'mean_word_length': 4.595744680851064, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1317758', 'n_tokens_mistral': 1273, 'n_tokens_neox': 1132, 'n_words': 241}