<commit_before>#include <boost/regex.hpp>
#include <iostream>
#include <sstream>

const char* illegalCharacters =
	"(\\[)|(\\])|"
	"(<)|(>)|"
	"(\\*)|"
	"(&)|"
	"(\\t)|"
	"(\\^)|"
	"(%)|"
	"(#)|"
	"(£)|"
	"(@)|"
	"(!)|"
	"(:)|"
	"(;)|"
	"(\\\")|"
	"(')|"
	"(\\$)|"
	"(/)|"
	"(\\))|"
	"(\\()|"
	"(\\\\)|"
	"( )";

int main(int argc, char* argv[])
{
	std::string badFilename = "[]<>*&^%$£@!#()\"':;\\/ \t";
	std::cout << "FINAL   __" << badFilename << "__"<< std::endl;

	boost::regex exp(illegalCharacters);

	std::stringstream ss;

	std::ostream_iterator<char> oi(ss);

	boost::regex_replace(
			oi, badFilename.begin(), badFilename.end(), 
			exp, "X",
			boost::match_default | boost::format_all );

	std::string corrected(ss.str());
	
	std::cout << "INITIAL __" << corrected << "__"<< std::endl;
}
<commit_msg>fixed typo<commit_after>#include <boost/regex.hpp>
#include <iostream>
#include <sstream>

const char* illegalCharacters =
	"(\\[)|(\\])|"
	"(<)|(>)|"
	"(\\*)|"
	"(&)|"
	"(\\t)|"
	"(\\^)|"
	"(%)|"
	"(#)|"
	"(£)|"
	"(@)|"
	"(!)|"
	"(:)|"
	"(;)|"
	"(\\\")|"
	"(')|"
	"(\\$)|"
	"(/)|"
	"(\\))|"
	"(\\()|"
	"(\\\\)|"
	"( )";

int main(int argc, char* argv[])
{
	std::string badFilename = "[]<>*&^%$£@!#()\"':;\\/ \t";
	std::cout << "INITIAL   __" << badFilename << "__"<< std::endl;

	boost::regex exp(illegalCharacters);

	std::stringstream ss;

	std::ostream_iterator<char> oi(ss);

	boost::regex_replace(
			oi, badFilename.begin(), badFilename.end(), 
			exp, "X",
			boost::match_default | boost::format_all );

	std::string corrected(ss.str());
	
	std::cout << "FINAL __" << corrected << "__"<< std::endl;
}

{'fraction_non_alphanumeric': 0.35489833641404805, 'fraction_numerical': 0.0, 'mean_word_length': 3.9212121212121214, 'pattern_counts': {'":': 0, '<': 47, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7376912', 'n_tokens_mistral': 779, 'n_tokens_neox': 736, 'n_words': 89}