<commit_before>#include "tokenize.hpp"
#include "../utilities/positional_exception.hpp"
#include "../vendor/json.hpp"
#include "../vendor/fmt/format.hpp"
#include <regex>
#include <cstddef>
#include <utility>
#include <optional>

using namespace thewizardplusplus::wizard_parser::lexer;
using namespace thewizardplusplus::wizard_parser::utilities;
using namespace nlohmann;
using namespace fmt;

namespace {

std::match_results<std::string_view::const_iterator> match_lexeme(
	const lexeme& some_lexeme,
	const std::string_view& code
) {
	auto match = std::match_results<std::string_view::const_iterator>{};
	std::regex_search(
		std::cbegin(code),
		std::cend(code),
		match,
		some_lexeme.pattern,
		std::regex_constants::match_continuous
	);

	return match;
}

std::optional<token> find_matched_token(
	const lexeme_group& lexemes,
	const std::string_view& code,
	const std::size_t offset
) {
	for (const auto& some_lexeme: lexemes) {
		const auto match = match_lexeme(some_lexeme, code);
		if (!match.empty()) {
			return token{some_lexeme.type, match.str(), offset};
		}
	}

	return {};
}

}

namespace thewizardplusplus::wizard_parser::lexer {

token_group tokenize(
	const lexeme_group& lexemes,
	const std::string_view& code
) {
	auto tokens = token_group{};
	auto rest_code = code;
	auto offset = std::size_t{};
	while (!rest_code.empty()) {
		auto matched_token = find_matched_token(lexemes, rest_code, offset);
		if (!matched_token) {
			throw positional_exception{
				format(
					"invalid symbol {:s}",
					json(std::string{rest_code[0]}).dump()
				),
				offset
			};
		}

		rest_code = rest_code.substr(matched_token->value.size());
		offset += matched_token->value.size();
		tokens.push_back(std::move(*matched_token));<commit_msg>Refactoring of the lexer module: refactoring of the find_matched_token() function<commit_after>#include "tokenize.hpp"
#include "../utilities/positional_exception.hpp"
#include "../vendor/json.hpp"
#include "../vendor/fmt/format.hpp"
#include <regex>
#include <iterator>
#include <optional>
#include <vector>
#include <tuple>
#include <algorithm>
#include <cstddef>
#include <string>

using namespace thewizardplusplus::wizard_parser::utilities;
using namespace nlohmann;
using namespace fmt;

namespace thewizardplusplus::wizard_parser::lexer {

using match_t = std::match_results<std::string_view::const_iterator>;

match_t match_lexeme(const lexeme& some_lexeme, const std::string_view& code) {
	auto match = match_t{};
	std::regex_search(
		std::cbegin(code),
		std::cend(code),
		match,
		some_lexeme.pattern,
		std::regex_constants::match_continuous
	);

	return match;
}

std::optional<token> find_matched_token(
	const lexeme_group& lexemes,
	const std::string_view& code
) {
	auto matches = std::vector<std::tuple<lexeme, match_t>>{};
	std::transform(
		std::cbegin(lexemes),
		std::cend(lexemes),
		std::back_inserter(matches),
		[&] (const auto& lexeme) {
			const auto match = match_lexeme(lexeme, code);
			return std::make_tuple(lexeme, match);
		}
	);

	const auto match = std::find_if(
		std::cbegin(matches),
		std::cend(matches),
		[] (const auto& match) { return !std::get<match_t>(match).empty(); }
	);
	if (match == std::cend(matches)) {
		return std::nullopt;
	}

	return token{std::get<lexeme>(*match).type, std::get<match_t>(*match).str()};
}

token_group tokenize(
	const lexeme_group& lexemes,
	const std::string_view& code
) {
	auto tokens = token_group{};
	auto rest_code = code;
	auto offset = std::size_t{};
	while (!rest_code.empty()) {
		const auto matched_token = find_matched_token(lexemes, rest_code);
		if (!matched_token) {
			throw positional_exception{
				format(
					"invalid symbol {:s}",
					json(std::string{rest_code[0]}).dump()
				),
				offset
			};
		}

		tokens.push_back({matched_token->type, matched_token->value, offset});
		rest_code = rest_code.substr(matched_token->value.size());
		offset += matched_token->value.size();
{'fraction_non_alphanumeric': 0.15041995418681597, 'fraction_numerical': 0.0005090353779587681, 'mean_word_length': 5.717948717948718, 'pattern_counts': {'":': 0, '<': 25, '<?xml version=': 0, '>': 31, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4503632', 'n_tokens_mistral': 1481, 'n_tokens_neox': 1368, 'n_words': 268}