<commit_before>#include "line_accumulator.h"

#include <cassert>
#include <iostream>
#include <algorithm>

namespace {

static std::mt19937 generator;

std::uint8_t getRandomIndex(
	const int          seed,
	const std::uint8_t n
) {
	generator.seed(seed);

	return std::uniform_int_distribution<std::uint8_t>{0, n}(generator);
}

std::vector<std::uint8_t> getRandomIndizes(
	const int          seed,
	const std::uint8_t n
) {
	generator.seed(seed);

	std::vector<std::uint8_t> indizes(n);
	std::iota(indizes.begin(), indizes.end(), 0);
	std::shuffle(
		indizes.begin(),
		indizes.end(),
		generator
	);

	return indizes;
}

}

namespace justify {

LineAccumulator::LineAccumulator(const std::uint8_t max_length):
	max_length_{max_length},
	length_{0},
	tokens_{} { }

LineAccumulator::~LineAccumulator() {
	this->discharge(false);
}

std::uint8_t LineAccumulator::getMissing() const {
	return this->max_length_ - this->length_;
}

void LineAccumulator::operator()(const std::string& token) {
	if ( ( this->length_ + token.length() ) > this->max_length_ ) {
		this->discharge(true);
	}

	this->tokens_.emplace_back(token, 0);
	this->length_ += token.length();

	if ( this->length_ < this->max_length_ ) {
		this->tokens_.back().second += 1;
		this->length_               += 1;
	}
}

void LineAccumulator::justify() {
	const int seed = this->tokens_.size()
	               + this->getMissing();

	switch ( this->tokens_.size() ) {
		// There is no sensible block justification of null or any single token
		case 0:
		case 1: {
			this->length_ = this->max_length_;

			break;
		}
		// i.e. left align the first token and right align the second token
		case 2: {
			this->tokens_[0].second += this->getMissing();
			this->length_            = this->max_length_;

			break;
		}
		// most common branch for normal texts, add the maximum equal amount of
		// spaces to all but the last token
		default: {
			const std::uint8_t base = this->getMissing()<commit_msg>Implement support for UTF8 multi-byte code points<commit_after>#include "line_accumulator.h"

#include <cassert>
#include <iostream>
#include <algorithm>

namespace {

static std::mt19937 generator;

std::uint8_t getRandomIndex(
	const int          seed,
	const std::uint8_t n
) {
	generator.seed(seed);

	return std::uniform_int_distribution<std::uint8_t>{0, n}(generator);
}

std::vector<std::uint8_t> getRandomIndizes(
	const int          seed,
	const std::uint8_t n
) {
	generator.seed(seed);

	std::vector<std::uint8_t> indizes(n);
	std::iota(indizes.begin(), indizes.end(), 0);
	std::shuffle(
		indizes.begin(),
		indizes.end(),
		generator
	);

	return indizes;
}

std::size_t getCharacterLength(const std::string& token) {
	std::size_t codeUnitIndex  = 0;
	std::size_t codePointIndex = 0;

	while ( token.data()[codeUnitIndex] ) {
		// advance `codePointIndex` if current unit is not a continuation byte
		// see RFC3629 for further information
		if ( (token.data()[codeUnitIndex] & 0b11000000 ) != 0b10000000 ) {
			++codePointIndex;
		}
		++codeUnitIndex;
	}

	return codePointIndex;
}

}

namespace justify {

LineAccumulator::LineAccumulator(const std::uint8_t max_length):
	max_length_{max_length},
	length_{0},
	tokens_{} { }

LineAccumulator::~LineAccumulator() {
	this->discharge(false);
}

std::uint8_t LineAccumulator::getMissing() const {
	return this->max_length_ - this->length_;
}

void LineAccumulator::operator()(const std::string& token) {
	const std::size_t tokenLength = getCharacterLength(token);

	if ( ( this->length_ + tokenLength ) > this->max_length_ ) {
		this->discharge(true);
	}

	this->tokens_.emplace_back(token, 0);
	this->length_ += tokenLength;

	if ( this->length_ < this->max_length_ ) {
		this->tokens_.back().second += 1;
		this->length_               += 1;
	}
}

void LineAccumulator::justify() {
	const int seed = this->tokens_.size()
	               + this->getMissing();

	switch ( this->tokens_.size() ) {
		// There is no sensible block justification of null or any single token
		case 0:
		case 1: {
			this->length_ = this->max_length_;

			break;
		}
		// i.e. left align the first token and right align the second token
		case 2: {
			this->tokens_[0].second += this->getMissing();
			this->length_            = this->max_length_;

			break;
		}
		// most common branch for normal texts, add the maximum equal amount of
		// spaces to all but the last token
		default: {
			const std::uint8_t base = this->getMissing()
{'fraction_non_alphanumeric': 0.1430505415162455, 'fraction_numerical': 0.01647111913357401, 'mean_word_length': 3.898342541436464, 'pattern_counts': {'":': 0, '<': 17, '<?xml version=': 0, '>': 61, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 1, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5489542', 'n_tokens_mistral': 1639, 'n_tokens_neox': 1508, 'n_words': 346}