<commit_before>#include <textformatter.h>
#include <utils.h>
#include <strprintf.h>
#include <htmlrenderer.h>
#include <stflpp.h>
#include <assert.h>
#include <limits.h>
#include <algorithm>

namespace newsboat {

textformatter::textformatter() { }

textformatter::~textformatter() { }

void textformatter::add_line(LineType type, std::string line) {
	LOG(level::DEBUG,
		"textformatter::add_line: `%s' (line type %i)",
		line,
		type);

	auto clean_line =
		utils::wstr2str(
			utils::clean_nonprintable_characters(utils::str2wstr(line)));
	lines.push_back(std::make_pair(type, clean_line));
}

void textformatter::add_lines(
		const std::vector<std::pair<LineType, std::string>>& lines)
{
	for (auto line : lines) {
		add_line(line.first,
			utils::replace_all(line.second, "\t", "        "));
	}
}

std::vector<std::string> wrap_line(
		const std::string& line,
		const size_t width)
{
	if (line.empty()) {
		return { "" };
	}

	std::vector<std::string> result;
	std::vector<std::string> words = utils::tokenize_spaced(line);

	std::string prefix;
	auto iswhitespace =
		[](const std::string& input)
		{
			return std::all_of(
					input.cbegin(),
					input.cend(),
					[](std::string::value_type c) { return std::isspace(c); });
		};
	if (iswhitespace(words[0])) {
		prefix = utils::substr_with_width(words[0], width);
		words[0] = prefix;
	}

	std::string curline = "";

	for (auto word : words) {
		size_t word_width = utils::strwidth_stfl(word);
		size_t curline_width = utils::strwidth_stfl(curline);

		// for languages (e.g., CJK) don't use a space as a word boundary
		while (word_width > width) {
			size_t space_left = width - curline_width;
			std::string part = utils::substr_with_width(word, space_left);
			curline.append(part);
			word.erase(0, part.length());
			result.push_back(curline);
			curline = prefix;
			if (part.empty()) {
				// discard the current word
				word.clear();
				word_width = 0;
			}

			word_width = utils::strwidth_stfl(word);
			curline_width = utils::strwidth_stfl(curline);
		}

		if ((curline_width + word_width) > width) {
			result.push_back(curline);
			if (iswhitespace(word)) {
				curline = prefix;
			} else {
				curline = prefix + word;
			}
		} else {
			curline.append(word);
		}<commit_msg>Correct hard-wrapping condition (#82)<commit_after>#include <textformatter.h>
#include <utils.h>
#include <strprintf.h>
#include <htmlrenderer.h>
#include <stflpp.h>
#include <assert.h>
#include <limits.h>
#include <algorithm>

namespace newsboat {

textformatter::textformatter() { }

textformatter::~textformatter() { }

void textformatter::add_line(LineType type, std::string line) {
	LOG(level::DEBUG,
		"textformatter::add_line: `%s' (line type %i)",
		line,
		type);

	auto clean_line =
		utils::wstr2str(
			utils::clean_nonprintable_characters(utils::str2wstr(line)));
	lines.push_back(std::make_pair(type, clean_line));
}

void textformatter::add_lines(
		const std::vector<std::pair<LineType, std::string>>& lines)
{
	for (auto line : lines) {
		add_line(line.first,
			utils::replace_all(line.second, "\t", "        "));
	}
}

std::vector<std::string> wrap_line(
		const std::string& line,
		const size_t width)
{
	if (line.empty()) {
		return { "" };
	}

	std::vector<std::string> result;
	std::vector<std::string> words = utils::tokenize_spaced(line);

	std::string prefix;
	size_t prefix_width = 0;
	auto iswhitespace =
		[](const std::string& input)
		{
			return std::all_of(
					input.cbegin(),
					input.cend(),
					[](std::string::value_type c) { return std::isspace(c); });
		};
	if (iswhitespace(words[0])) {
		prefix = utils::substr_with_width(words[0], width);
		prefix_width = utils::strwidth_stfl(prefix);
		words.erase(words.cbegin());
	}

	std::string curline = prefix;

	for (auto word : words) {
		size_t word_width = utils::strwidth_stfl(word);
		size_t curline_width = utils::strwidth_stfl(curline);

		// for languages (e.g., CJK) don't use a space as a word boundary
		while (word_width > (width - prefix_width)) {
			size_t space_left = width - curline_width;
			std::string part = utils::substr_with_width(word, space_left);
			curline.append(part);
			word.erase(0, part.length());
			result.push_back(curline);
			curline = prefix;
			if (part.empty()) {
				// discard the current word
				word.clear();
				word_width = 0;
			}

			word_width = utils::strwidth_stfl(word);
			curline_width = utils::strwidth_stfl(curline);
		}

		if ((curline_width + word_width) > width) {
			result.push_back(curline);
			if (iswhitespace(word)) {
				curline = prefix;
			} else {
				curline = prefix + word;
			}
		} else {
			curline.append(word);
		}
{'fraction_non_alphanumeric': 0.1626402070750647, 'fraction_numerical': 0.003451251078515962, 'mean_word_length': 4.4044289044289044, 'pattern_counts': {'":': 0, '<': 29, '<?xml version=': 0, '>': 33, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '315941', 'n_tokens_mistral': 1897, 'n_tokens_neox': 1700, 'n_words': 327}