<commit_before>/* Distributed under the OSI-approved BSD 3-Clause License.  See accompanying
   file Copyright.txt or https://opensource.org/licenses/BSD-3-Clause for
   details.  */

#pragma once

#include <algorithm>
#include <cctype>
#include <string>

static inline void replace_all_in_string(std::string &main_string, const std::string &from,
                                         const std::string &to) {
	size_t pos = 0;
	while ((pos = main_string.find(from, pos)) != std::string::npos) {
		main_string.replace(pos, from.size(), to);
	}
}

static inline std::string lowerstring(const std::string &val) {
	std::string newval = val;
	std::transform(newval.begin(), newval.end(), newval.begin(),
	               [](char c) { return std::tolower(c); });
	return newval;
}

static inline std::string repeat_string(const std::string &val, size_t n) {
	std::string newval;
	for (size_t i = 0; i < n; i++) {
		newval += val;
	}
	return newval;
}<commit_msg>helpers.h: add replace_invisibles_with_visibles<commit_after>/* Distributed under the OSI-approved BSD 3-Clause License.  See accompanying
   file Copyright.txt or https://opensource.org/licenses/BSD-3-Clause for
   details.  */

#pragma once

#include <algorithm>
#include <cctype>
#include <string>

static inline void replace_all_in_string(std::string &main_string, const std::string &from,
                                         const std::string &to) {
	size_t pos = 0;
	while ((pos = main_string.find(from, pos)) != std::string::npos) {
		main_string.replace(pos, from.size(), to);
	}
}

static inline void replace_invisibles_with_visibles(std::string &val) {
	replace_all_in_string(val, " ", "·");
	replace_all_in_string(val, "\t", "»   ");
}

static inline std::string lowerstring(const std::string &val) {
	std::string newval = val;
	std::transform(newval.begin(), newval.end(), newval.begin(),
	               [](char c) { return std::tolower(c); });
	return newval;
}

static inline std::string repeat_string(const std::string &val, size_t n) {
	std::string newval;
	for (size_t i = 0; i < n; i++) {
		newval += val;
	}
	return newval;
}
{'fraction_non_alphanumeric': 0.14888572783309625, 'fraction_numerical': 0.003793266951161688, 'mean_word_length': 3.7845804988662133, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 9, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2385931', 'n_tokens_mistral': 720, 'n_tokens_neox': 703, 'n_words': 182}