<commit_before>#include "get_code.h"
#include <fstream>
#include <boost/regex.hpp>
#include <boost/format.hpp>
#include <boost/filesystem.hpp>

using namespace thewizardplusplus::wizard_basic_3;
using namespace boost;
using namespace boost::filesystem;

static const auto INCLUDE_OPERATOR_PATTERN = regex(
	R"pattern(\binclude\s*"((?:\\.|[^"])*)")pattern"
);

struct CodeGetter {
	std::string base_path;

	std::string operator()(const smatch& match) const {
		return GetCode(base_path + match[1]);
	}
};

namespace thewizardplusplus {
namespace wizard_basic_3 {

auto GetCode(const std::string& filename) -> std::string {
	std::ifstream file(filename.c_str());
	if ((file.rdstate() & std::ifstream::failbit) != 0) {
		throw std::runtime_error(
			(format(R"(unable to open file "%s")") % filename).str()
		);
	}

	file.exceptions(std::ifstream::failbit | std::ifstream::badbit);
	auto code = std::string();
	try {
		code = std::string(
			std::istreambuf_iterator<char>(file),
			std::istreambuf_iterator<char>()
		);
	} catch(const std::ifstream::failure& exception) {
		throw std::runtime_error(
			(format(R"(unable to read file "%s" (%s))")
				% filename
				% exception.what()).str()
		);
	}

	auto base_path = path(filename).parent_path().string();<commit_msg>Add test of unique of include files.<commit_after>#include "get_code.h"
#include <set>
#include <fstream>
#include <boost/regex.hpp>
#include <boost/filesystem.hpp>
#include <boost/format.hpp>

using namespace thewizardplusplus::wizard_basic_3;
using namespace boost;
using namespace boost::filesystem;

struct CodeGetter {
	std::string base_path;

	std::string operator()(const smatch& match) const {
		return GetCode(base_path + match[1]);
	}
};

static const auto INCLUDE_OPERATOR_PATTERN = regex(
	R"pattern(\binclude\s*"((?:\\.|[^"])*)")pattern"
);

static auto included_files = std::set<std::string>();

namespace thewizardplusplus {
namespace wizard_basic_3 {

auto GetCode(const std::string& filename) -> std::string {
	const auto absolute_path = canonical(path(filename)).string();
	if (included_files.count(absolute_path)) {
		return "";
	}
	included_files.insert(absolute_path);

	std::ifstream file(filename.c_str());
	if ((file.rdstate() & std::ifstream::failbit) != 0) {
		throw std::runtime_error(
			(format(R"(unable to open file "%s")") % filename).str()
		);
	}

	file.exceptions(std::ifstream::failbit | std::ifstream::badbit);
	auto code = std::string();
	try {
		code = std::string(
			std::istreambuf_iterator<char>(file),
			std::istreambuf_iterator<char>()
		);
	} catch(const std::ifstream::failure& exception) {
		throw std::runtime_error(
			(format(R"(unable to read file "%s" (%s))")
				% filename
				% exception.what()).str()
		);
	}

	auto base_path = path(filename).parent_path().string();
{'fraction_non_alphanumeric': 0.17305625223934074, 'fraction_numerical': 0.002866356144750985, 'mean_word_length': 5.679425837320574, 'pattern_counts': {'":': 0, '<': 17, '<?xml version=': 0, '>': 19, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6535916', 'n_tokens_mistral': 1015, 'n_tokens_neox': 954, 'n_words': 191}