<commit_before>/**
 * @file
 *
 * @brief deserialization implementation for xerces plugin
 *
 * @copyright BSD License (see LICENSE.md or https://www.libelektra.org)
 */

#include "deserializer.hpp"
#include "util.hpp"

#include <xercesc/dom/DOM.hpp>
#include <xercesc/dom/DOMAttr.hpp>
#include <xercesc/dom/DOMDocument.hpp>
#include <xercesc/dom/DOMImplementation.hpp>
#include <xercesc/dom/DOMNode.hpp>
#include <xercesc/parsers/XercesDOMParser.hpp>

#include <algorithm>
#include <iostream>
#include <locale>
#include <map>

#include <kdbease.h>
#include <kdblogger.h>
#include <key.hpp>

XERCES_CPP_NAMESPACE_USE
using namespace std;
using namespace kdb;
using namespace xerces;

namespace
{

XercesPtr<DOMDocument> doc2dom (std::string const & src)
{
	XercesDOMParser parser;
	parser.setValidationScheme (XercesDOMParser::Val_Auto);

	parser.parse (asXMLCh (src));
	return XercesPtr<DOMDocument> (parser.adoptDocument ());
}

string trim (string const & str)
{
	stringstream ss (str);
	string to;
	string trimmed;

	while (getline (ss, to, '\n'))
	{
		// Remove whitespace lines, most likely caused by pretty printing
		if (!all_of (to.begin (), to.end (), [](char c) { return isspace (c, locale ()); })) trimmed += to;
	}

	return trimmed;
}

string getElementText (DOMNode const * parent)
{
	string str;

	for (auto child = parent->getFirstChild (); child != NULL; child = child->getNextSibling ())
	{
		if (DOMNode::NodeType::TEXT_NODE == child->getNodeType () || DOMNode::NodeType::CDATA_SECTION_NODE == child->getNodeType ())
		{
			DOMText * data = dynamic_cast<DOMText *> (child);
			if (!data->getIsElementContentWhitespace ()) str += toStr (data->getData ());
		}
	}

	// Trim whitespace that is most likely due to pretty printing
	return trim (str);
}

Key newNodeKey (Key const & parent, DOMNode const * node)
{
	Key childKey (parent.getFullName (), KEY_END);
	const string keyName = toStr (node->getNodeName ());
	childKey.addBaseName (keyName);
	return childKey;
}

void node2key (DOMNode const * n, Key const & parent, KeySet const & ks, Key & current)
{
	const string keyName = toStr (n->getNodeName ());
	ELEKTRA_LOG_DEBUG ("Encountered Element: %s with parent %s", keyName.c_str (), current.getFullName ().c_str ());

	if (!ks.size ())
	{ // we map the parent key to the xml root element
		// preserve the original name if its different
		auto parentName = parent.rbegin ();
		if (parentName != parent.rend () && (*parentName) != keyName)
		{
			ELEKTRA_LOG_DEBUG ("parent name %s differs from root element name %s", (*parentName).c_str (), keyName.c_str ());
			current.setMeta (ELEKTRA_XERCES_ORIGINAL_ROOT_NAME, keyName);
		}
	}
	else
		current.addBaseName (keyName);

	const string text = getElementText (n);
	current.set<string> (text);

	if (!current.isValid ()) throw XercesPluginException ("Given keyset contains invalid keys to serialize");

	ELEKTRA_LOG_DEBUG ("new parent is %s with value %s", current.getFullName ().c_str (), current.get<string> ().c_str ());<commit_msg>Xerces: Correct minor spelling mistake<commit_after>/**
 * @file
 *
 * @brief deserialization implementation for xerces plugin
 *
 * @copyright BSD License (see LICENSE.md or https://www.libelektra.org)
 */

#include "deserializer.hpp"
#include "util.hpp"

#include <xercesc/dom/DOM.hpp>
#include <xercesc/dom/DOMAttr.hpp>
#include <xercesc/dom/DOMDocument.hpp>
#include <xercesc/dom/DOMImplementation.hpp>
#include <xercesc/dom/DOMNode.hpp>
#include <xercesc/parsers/XercesDOMParser.hpp>

#include <algorithm>
#include <iostream>
#include <locale>
#include <map>

#include <kdbease.h>
#include <kdblogger.h>
#include <key.hpp>

XERCES_CPP_NAMESPACE_USE
using namespace std;
using namespace kdb;
using namespace xerces;

namespace
{

XercesPtr<DOMDocument> doc2dom (std::string const & src)
{
	XercesDOMParser parser;
	parser.setValidationScheme (XercesDOMParser::Val_Auto);

	parser.parse (asXMLCh (src));
	return XercesPtr<DOMDocument> (parser.adoptDocument ());
}

string trim (string const & str)
{
	stringstream ss (str);
	string to;
	string trimmed;

	while (getline (ss, to, '\n'))
	{
		// Remove whitespace lines, most likely caused by pretty printing
		if (!all_of (to.begin (), to.end (), [](char c) { return isspace (c, locale ()); })) trimmed += to;
	}

	return trimmed;
}

string getElementText (DOMNode const * parent)
{
	string str;

	for (auto child = parent->getFirstChild (); child != NULL; child = child->getNextSibling ())
	{
		if (DOMNode::NodeType::TEXT_NODE == child->getNodeType () || DOMNode::NodeType::CDATA_SECTION_NODE == child->getNodeType ())
		{
			DOMText * data = dynamic_cast<DOMText *> (child);
			if (!data->getIsElementContentWhitespace ()) str += toStr (data->getData ());
		}
	}

	// Trim whitespace that is most likely due to pretty printing
	return trim (str);
}

Key newNodeKey (Key const & parent, DOMNode const * node)
{
	Key childKey (parent.getFullName (), KEY_END);
	const string keyName = toStr (node->getNodeName ());
	childKey.addBaseName (keyName);
	return childKey;
}

void node2key (DOMNode const * n, Key const & parent, KeySet const & ks, Key & current)
{
	const string keyName = toStr (n->getNodeName ());
	ELEKTRA_LOG_DEBUG ("Encountered Element: %s with parent %s", keyName.c_str (), current.getFullName ().c_str ());

	if (!ks.size ())
	{ // we map the parent key to the xml root element
		// preserve the original name if it is different
		auto parentName = parent.rbegin ();
		if (parentName != parent.rend () && (*parentName) != keyName)
		{
			ELEKTRA_LOG_DEBUG ("parent name %s differs from root element name %s", (*parentName).c_str (), keyName.c_str ());
			current.setMeta (ELEKTRA_XERCES_ORIGINAL_ROOT_NAME, keyName);
		}
	}
	else
		current.addBaseName (keyName);

	const string text = getElementText (n);
	current.set<string> (text);

	if (!current.isValid ()) throw XercesPluginException ("Given keyset contains invalid keys to serialize");

	ELEKTRA_LOG_DEBUG ("new parent is %s with value %s", current.getFullName ().c_str (), current.get<string> ().c_str ());
{'fraction_non_alphanumeric': 0.14129534537021698, 'fraction_numerical': 0.0006625807520291535, 'mean_word_length': 4.783524904214559, 'pattern_counts': {'":': 0, '<': 39, '<?xml version=': 0, '>': 55, 'https://': 2, 'lorem ipsum': 0, 'www.': 2, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4993766', 'n_tokens_mistral': 2175, 'n_tokens_neox': 2072, 'n_words': 559}