<commit_before>///
// A WML parser using boost spirit2.
// Based heavily on http://www.boost.org/doc/libs/1_47_0/libs/spirit/example/qi/mini_xml2.cpp
///

#define BOOST_SPIRIT_USE_PHOENIX_V3
#define BOOST_SPIRIT_UNICODE

#include <boost/config/warning_disable.hpp>
#include <boost/spirit/include/qi.hpp>
#include <boost/spirit/include/phoenix_core.hpp>
#include <boost/spirit/include/phoenix_object.hpp>
#include <boost/spirit/include/phoenix_operator.hpp>
#include <boost/spirit/include/phoenix_fusion.hpp>
#include <boost/spirit/include/phoenix_stl.hpp>
#include <boost/fusion/include/adapt_struct.hpp>
#include <boost/fusion/include/std_pair.hpp>
#include <boost/variant/recursive_variant.hpp>
#include <boost/foreach.hpp>

#include <iostream>
#include <fstream>
#include <string>
#include <vector>

namespace wml
{
	namespace fusion = boost::fusion;
	namespace phoenix = boost::phoenix;
	namespace qi = boost::spirit::qi;

	typedef std::string Str;
	typedef std::pair<Str, Str> Pair;

	///////////////////////////////////////////////////////////////////////////
	//  Our WML tree representation
	///////////////////////////////////////////////////////////////////////////
	struct body;

	typedef boost::variant<boost::recursive_wrapper<body>, Pair>
		node;

	struct body
	{
		Str name;		    // tag name
		std::vector<node> children; // children
	};
}

// We need to tell fusion about our wml struct
// to make it a first-class fusion citizen
BOOST_FUSION_ADAPT_STRUCT(
	wml::body,
	(wml::Str, name)(std::vector<wml::node>, children))

namespace wml
{
	///////////////////////////////////////////////////////////////////////////
	//  Print out the wml tree
	///////////////////////////////////////////////////////////////////////////
	int const tabsize = 4;

	void tab(int indent)
	{
		for(int i = 0; i < indent; ++i)
			std::cout << ' ';
	}

	struct body_printer
	{
		body_printer(int indent = 0)
		    : indent(indent)
		{
		}

		void operator()(body const&) const;

		int indent;
	};

	struct node_printer : boost::static_visitor<>
	{
		node_printer(int indent = 0)
		    : indent(indent)
		{
		}

		void operator()(body const& w) const
		{
			body_printer(indent + tabsize)(w);
		}

		void operator()(Str const& text) const
		{
			tab(indent + tabsize);
			std::cout << "text: \"" << text << '"' << std::endl;
		}

		void operator()(Pair const& p) const
		{
			tab(indent + tabsize);
			std::cout << p.first << ": \"" << p.second << "\"" << std::endl;
		}

		int indent;
	};

	void body_printer::operator()(body const& w) const
	{
		tab(indent);
		std::cout << "tag: \"" << w.name << "\"" << std::endl;
		tab(indent);
		std::cout << '{' << std::endl;

		BOOST_FOREACH(node const& n, w.children) {
			boost::apply_visitor(node_printer(indent), n);
		}

		tab(indent);
		std::cout << '}' << std::endl;
	}

	///////////////////////////////////////////////////////////////////////////
	//  Our WML grammar definition
	///////////////////////////////////////////////////////////////////////////
	template <typename Iterator>
	struct wml_grammar
		: qi::grammar<Iterator, body(), qi::locals<Str>, qi::space_type>
	{
		wml_grammar()
		    : wml_grammar::base_type(wml, "wml")
		{
			using qi::lit;
			using qi::lexeme;
			using qi::on_error;
			using qi::fail;
			using boost::spirit::unicode::char_;
			using boost::spirit::unicode::string;
			using namespace qi::labels;

			using phoenix::construct;
			using phoenix::val;

			pair = key >> lit('=') >> value;
			key = char_("a-zA-Z_") >> *char_("a-zA-Z_0-9");
			value = -lit('_') >> (angle_quoted_string | double_quoted_string | endl_terminated_string);

			angle_quoted_string = lexeme[qi::string("<<") >> +(char_ - '>') >> qi::string(">>")];
			double_quoted_string = lexeme['"' >> +(char_ - '"') >> '"'];
			endl_terminated_string = lexeme[*(char_ - '\n')];

			node = wml | pair;

			start_tag %= lit('[')
				     >> !lit('/')
				     >> lexeme[+(char_ - ']')]
				     >> lit(']');

			end_tag = lit("[/")
				  >> string(_r1)
				  >> lit(']');

			wml %= start_tag[_a = _1]
			       >> *node
			       >> end_tag(_a);

			wml.name("wml");
			node.name("node");
			start_tag.name("start_tag");
			end_tag.name("end_tag");
			key.name("attribute_key");
			value.name("attribute_value");
			pair.name("attribute");

/*			on_error<fail>(
				key, std::cerr << val("Error! Expecting ") << qi::_4			     // what failed?
					       << val(" here: \"") << construct<std::string>(qi::_3, qi::_2) // iterators to error-pos, end

					       << val("\"") << std::endl);

			on_error<fail>(
				value, std::cerr << val("Error! Expecting ") << qi::_4			     // what failed?
					       << val(" here: \"") << construct<std::string>(qi::_3, qi::_2) // iterators to error-pos, end
					       << val("\"") << std::endl);


			on_error<fail>(
				pair, std::cerr << val("Error! Expecting ") << qi::_4			     // what failed?
					       << val(" here: \"") << construct<std::string>(qi::_3, qi::_2) // iterators to error-pos, end
					       << val("\"") << std::endl);

			on_error<fail>(
				node, std::cerr << val("Error! Expecting ") << qi::_4			     // what failed?
					       << val(" here: \"") << construct<std::string>(qi::_3, qi::_2) // iterators to error-pos, end
					       << val("\"") << std::endl);
			on_error<fail>(
				wml, std::cerr << val("Error! Expecting ") << qi::_4			     // what failed?
					       << val(" here: \"") << construct<std::string>(qi::_3, qi::_2) // iterators to error-pos, end
					       << val("\"") << std::endl);*/

      BOOST_SPIRIT_DEBUG_NODE(wml);
      BOOST_SPIRIT_DEBUG_NODE(node);
      BOOST_SPIRIT_DEBUG_NODE(start_tag);
      BOOST_SPIRIT_DEBUG_NODE(end_tag);
      BOOST_SPIRIT_DEBUG_NODE(pair);
      BOOST_SPIRIT_DEBUG_NODE(key);
      BOOST_SPIRIT_DEBUG_NODE(value);
      BOOST_SPIRIT_DEBUG_NODE(double_quoted_string);
      BOOST_SPIRIT_DEBUG_NODE(angle_quoted_string);
      BOOST_SPIRIT_DEBUG_NODE(endl_terminated_string);

		}

		qi::rule<Iterator, wml::body(), qi::locals<Str>, qi::space_type> wml;
		qi::rule<Iterator, wml::node(), qi::space_type> node;
		qi::rule<Iterator, Str(), qi::space_type> start_tag;
		qi::rule<Iterator, void(Str), qi::space_type> end_tag;
		qi::rule<Iterator, Pair(), qi::space_type> pair;
		qi::rule<Iterator, Str(), qi::space_type> key;
		qi::rule<Iterator, Str(), qi::space_type> value;
		qi::rule<Iterator, Str(), qi::space_type> double_quoted_string;
		qi::rule<Iterator, Str(), qi::space_type> angle_quoted_string;
		qi::rule<Iterator, Str(), qi::space_type> endl_terminated_string;
<commit_msg>more error reporting and parsing fixups (expectation ops, fix >>)<commit_after>///
// A WML parser using boost spirit2.
// Based heavily on http://www.boost.org/doc/libs/1_47_0/libs/spirit/example/qi/mini_xml2.cpp
///

#define BOOST_SPIRIT_USE_PHOENIX_V3
#define BOOST_SPIRIT_UNICODE

#include <boost/config/warning_disable.hpp>
#include <boost/spirit/include/qi.hpp>
#include <boost/spirit/include/phoenix_core.hpp>
#include <boost/spirit/include/phoenix_object.hpp>
#include <boost/spirit/include/phoenix_operator.hpp>
#include <boost/spirit/include/phoenix_fusion.hpp>
#include <boost/spirit/include/phoenix_stl.hpp>
#include <boost/fusion/include/adapt_struct.hpp>
#include <boost/fusion/include/std_pair.hpp>
#include <boost/variant/recursive_variant.hpp>
#include <boost/foreach.hpp>

#include <iostream>
#include <fstream>
#include <string>
#include <vector>

namespace wml
{
	namespace fusion = boost::fusion;
	namespace phoenix = boost::phoenix;
	namespace qi = boost::spirit::qi;

	typedef std::string Str;
	typedef std::pair<Str, Str> Pair;

	///////////////////////////////////////////////////////////////////////////
	//  Our WML tree representation
	///////////////////////////////////////////////////////////////////////////
	struct body;

	typedef boost::variant<boost::recursive_wrapper<body>, Pair>
		node;

	struct body
	{
		Str name;		    // tag name
		std::vector<node> children; // children
	};
}

// We need to tell fusion about our wml struct
// to make it a first-class fusion citizen
BOOST_FUSION_ADAPT_STRUCT(
	wml::body,
	(wml::Str, name)(std::vector<wml::node>, children))

namespace wml
{
	///////////////////////////////////////////////////////////////////////////
	//  Print out the wml tree
	///////////////////////////////////////////////////////////////////////////
	int const tabsize = 4;

	void tab(int indent)
	{
		for(int i = 0; i < indent; ++i)
			std::cout << ' ';
	}

	struct body_printer
	{
		body_printer(int indent = 0)
		    : indent(indent)
		{
		}

		void operator()(body const&) const;

		int indent;
	};

	struct node_printer : boost::static_visitor<>
	{
		node_printer(int indent = 0)
		    : indent(indent)
		{
		}

		void operator()(body const& w) const
		{
			body_printer(indent + tabsize)(w);
		}

		void operator()(Str const& text) const
		{
			tab(indent + tabsize);
			std::cout << "text: \"" << text << '"' << std::endl;
		}

		void operator()(Pair const& p) const
		{
			tab(indent + tabsize);
			std::cout << p.first << ": \"" << p.second << "\"" << std::endl;
		}

		int indent;
	};

	void body_printer::operator()(body const& w) const
	{
		tab(indent);
		std::cout << "tag: \"" << w.name << "\"" << std::endl;
		tab(indent);
		std::cout << '{' << std::endl;

		BOOST_FOREACH(node const& n, w.children) {
			boost::apply_visitor(node_printer(indent), n);
		}

		tab(indent);
		std::cout << '}' << std::endl;
	}

	///////////////////////////////////////////////////////////////////////////
	//  Our WML grammar definition
	///////////////////////////////////////////////////////////////////////////
	template <typename Iterator>
	struct wml_grammar
		: qi::grammar<Iterator, body(), qi::locals<Str>, qi::space_type>
	{
		wml_grammar()
		    : wml_grammar::base_type(wml, "wml")
		{
			using qi::lit;
			using qi::lexeme;
			using qi::on_error;
			using qi::fail;
			using boost::spirit::unicode::char_;
			using boost::spirit::unicode::string;
			using namespace qi::labels;

			using phoenix::construct;
			using phoenix::val;

			pair = key > lit('=') > value;
			key = char_("a-zA-Z_") >> *char_("a-zA-Z_0-9");
			value = -lit('_') >> (angle_quoted_string | double_quoted_string | endl_terminated_string);

			angle_quoted_string = lexeme[qi::string("<<") >> +(char_ - ">>") >> qi::string(">>")];
			double_quoted_string = lexeme['"' >> +(char_ - '"') >> '"'];
			endl_terminated_string = lexeme[*(char_ - '\n')];

			node = wml | pair;

			start_tag %= lit('[')
				     >> !lit('/')
				     >> lexeme[+(char_ - ']')]
				     >> lit(']');

			end_tag = lit("[/")
				  > string(_r1)
				  > lit(']');

			wml %= start_tag[_a = _1]
			       >> *node
			       > end_tag(_a);

			wml.name("wml");
			node.name("node");
			start_tag.name("start_tag");
			end_tag.name("end_tag");
			key.name("attribute_key");
			value.name("attribute_value");
			pair.name("attribute");

			on_error<fail>(
				key, std::cerr << val("Error! Expecting ") << qi::_4			     // what failed?
					       << val(" here: \"") << construct<std::string>(qi::_3, qi::_2) // iterators to error-pos, end

					       << val("\"") << std::endl);

			on_error<fail>(
				value, std::cerr << val("Error! Expecting ") << qi::_4			     // what failed?
					       << val(" here: \"") << construct<std::string>(qi::_3, qi::_2) // iterators to error-pos, end
					       << val("\"") << std::endl);


			on_error<fail>(
				pair, std::cerr << val("Error! Expecting ") << qi::_4			     // what failed?
					       << val(" here: \"") << construct<std::string>(qi::_3, qi::_2) // iterators to error-pos, end
					       << val("\"") << std::endl);

			on_error<fail>(
				start_tag, std::cerr << val("Error! Expecting ") << qi::_4			     // what failed?
					       << val(" here: \"") << construct<std::string>(qi::_3, qi::_2) // iterators to error-pos, end
					       << val("\"") << std::endl);

			on_error<fail>(
				end_tag, std::cerr << val("Error! Expecting ") << qi::_4			     // what failed?
					       << val(" here: \"") << construct<std::string>(qi::_3, qi::_2) // iterators to error-pos, end
					       << val("\"") << std::endl);

			on_error<fail>(
				wml, std::cerr << val("Error! Expecting ") << qi::_4			     // what failed?
					       << val(" here: \"") << construct<std::string>(qi::_3, qi::_2) // iterators to error-pos, end
					       << val("\"") << std::endl);

      BOOST_SPIRIT_DEBUG_NODE(wml);
      BOOST_SPIRIT_DEBUG_NODE(node);
      BOOST_SPIRIT_DEBUG_NODE(start_tag);
      BOOST_SPIRIT_DEBUG_NODE(end_tag);
      BOOST_SPIRIT_DEBUG_NODE(pair);
      BOOST_SPIRIT_DEBUG_NODE(key);
      BOOST_SPIRIT_DEBUG_NODE(value);
      BOOST_SPIRIT_DEBUG_NODE(double_quoted_string);
      BOOST_SPIRIT_DEBUG_NODE(angle_quoted_string);
      BOOST_SPIRIT_DEBUG_NODE(endl_terminated_string);

		}

		qi::rule<Iterator, wml::body(), qi::locals<Str>, qi::space_type> wml;
		qi::rule<Iterator, wml::node(), qi::space_type> node;
		qi::rule<Iterator, Str(), qi::space_type> start_tag;
		qi::rule<Iterator, void(Str), qi::space_type> end_tag;
		qi::rule<Iterator, Pair(), qi::space_type> pair;
		qi::rule<Iterator, Str(), qi::space_type> key;
		qi::rule<Iterator, Str(), qi::space_type> value;
		qi::rule<Iterator, Str(), qi::space_type> double_quoted_string;
		qi::rule<Iterator, Str(), qi::space_type> angle_quoted_string;
		qi::rule<Iterator, Str(), qi::space_type> endl_terminated_string;

{'fraction_non_alphanumeric': 0.2429243521768352, 'fraction_numerical': 0.004704652378463147, 'mean_word_length': 2.66001639792293, 'pattern_counts': {'":': 2, '<': 305, '<?xml version=': 0, '>': 159, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1503689', 'n_tokens_mistral': 5251, 'n_tokens_neox': 4609, 'n_words': 863}