<commit_before>/* rsspp - Copyright (C) 2008-2009 <NAME> <<EMAIL>>
 * Licensed under the MIT/X Consortium License. See file LICENSE
 * for more information.
 */

#include <config.h>
#include <rsspp_internal.h>
#include <utils.h>
#include <cstring>

namespace rsspp {


void atom_parser::parse_feed(feed& f, xmlNode * rootNode) {
	if (!rootNode)
		throw exception(_("XML root node is NULL"));

	f.language = get_prop(rootNode, "lang");
	globalbase = get_prop(rootNode, "base", XML_URI);

	for (xmlNode * node = rootNode->children; node != NULL; node = node->next) {
		if (node_is(node, "title")) {
			f.title = get_content(node);
			f.title_type = get_prop(node, "type");
			if (f.title_type == "")
				f.title_type = "text";
		} else if (node_is(node, "subtitle")) {
			f.description = get_content(node);
		} else if (node_is(node, "link")) {
			std::string rel = get_prop(node, "rel");
			if (rel == "alternate") {
				f.link = newsbeuter::utils::absolute_url(globalbase, get_prop(node, "href"));
			}
		} else if (node_is(node, "updated")) {
			f.pubDate = w3cdtf_to_rfc822(get_content(node));
		} else if (node_is(node, "entry")) {
			f.items.push_back(parse_entry(node));
		}
	}

}

item atom_parser::parse_entry(xmlNode * entryNode) {
	item it;
	std::string summary;
	std::string summary_type;
	std::string updated;

	std::string base = get_prop(entryNode, "base", XML_URI);
	if (base == "")
		base = globalbase;

	for (xmlNode * node = entryNode->children; node != NULL; node = node->next) {
		if (node_is(node, "author")) {
			for (xmlNode * authornode = node->children; authornode != NULL; authornode = authornode->next) {
				if (node_is(authornode, "name")) {
					it.author = get_content(authornode);
				} // TODO: is there more?
			}
		} else if (node_is(node, "title")) {
			it.title = get_content(node);
			it.title_type = get_prop(node, "type");
			if (it.title_type == "")
				it.title_type = "text";
		} else if (node_is(node, "content")) {
			std::string mode = get_prop(node, "mode");
			std::string type = get_prop(node, "type");
			if (mode == "xml" || mode == "") {
				if (type == "html" || type == "text") {
					it.description = get_content(node);
				} else {
					it.description = get_xml_content(node);
				}
			} else if (mode == "escaped") {
				it.description = get_content(node);
			}
			it.description_type = type;
			if (it.description_type == "")
				it.description_type = "text";
		} else if (node_is(node, "id")) {
			it.guid = get_content(node);
			it.guid_isPermaLink = false;
		} else if (node_is(node, "published")) {
			it.pubDate = w3cdtf_to_rfc822(get_content(node));
		} else if (node_is(node, "updated")) {
			updated = w3cdtf_to_rfc822(get_content(node));
		} else if (node_is(node, "link")) {
			std::string rel = get_prop(node, "rel");
			if (rel == "" || rel == "alternate") {
				it.link = newsbeuter::utils::absolute_url(base, get_prop(node, "href"));
			} else if (rel == "enclosure") {
				it.enclosure_url = get_prop(node, "href");
				it.enclosure_type = get_prop(node, "type");
			}
		} else if (node_is(node, "summary")) {
			std::string mode = get_prop(node, "mode");
			if (mode == "xml" || mode == "") {
				summary = get_xml_content(node);
			} else if (mode == "escaped") {
				summary = get_content(node);
			}
			summary_type = get_prop(node, "type");
			if (summary_type == "")
				summary_type = "text";
		}
	} // for

	if (it.description == "") {
		it.description = summary;
		it.description_type = summary_type;
	}

	if (it.pubDate == "") {
		it.pubDate = updated;
	}

	return it;
}<commit_msg>another Atom improvement.<commit_after>/* rsspp - Copyright (C) 2008-2009 <NAME> <<EMAIL>>
 * Licensed under the MIT/X Consortium License. See file LICENSE
 * for more information.
 */

#include <config.h>
#include <rsspp_internal.h>
#include <utils.h>
#include <cstring>

namespace rsspp {


void atom_parser::parse_feed(feed& f, xmlNode * rootNode) {
	if (!rootNode)
		throw exception(_("XML root node is NULL"));

	f.language = get_prop(rootNode, "lang");
	globalbase = get_prop(rootNode, "base", XML_URI);

	for (xmlNode * node = rootNode->children; node != NULL; node = node->next) {
		if (node_is(node, "title")) {
			f.title = get_content(node);
			f.title_type = get_prop(node, "type");
			if (f.title_type == "")
				f.title_type = "text";
		} else if (node_is(node, "subtitle")) {
			f.description = get_content(node);
		} else if (node_is(node, "link")) {
			std::string rel = get_prop(node, "rel");
			if (rel == "alternate") {
				f.link = newsbeuter::utils::absolute_url(globalbase, get_prop(node, "href"));
			}
		} else if (node_is(node, "updated")) {
			f.pubDate = w3cdtf_to_rfc822(get_content(node));
		} else if (node_is(node, "entry")) {
			f.items.push_back(parse_entry(node));
		}
	}

}

item atom_parser::parse_entry(xmlNode * entryNode) {
	item it;
	std::string summary;
	std::string summary_type;
	std::string updated;

	std::string base = get_prop(entryNode, "base", XML_URI);
	if (base == "")
		base = globalbase;

	for (xmlNode * node = entryNode->children; node != NULL; node = node->next) {
		if (node_is(node, "author")) {
			for (xmlNode * authornode = node->children; authornode != NULL; authornode = authornode->next) {
				if (node_is(authornode, "name")) {
					it.author = get_content(authornode);
				} // TODO: is there more?
			}
		} else if (node_is(node, "title")) {
			it.title = get_content(node);
			it.title_type = get_prop(node, "type");
			if (it.title_type == "")
				it.title_type = "text";
		} else if (node_is(node, "content")) {
			std::string mode = get_prop(node, "mode");
			std::string type = get_prop(node, "type");
			if (mode == "xml" || mode == "") {
				if (type == "html" || type == "text") {
					it.description = get_content(node);
				} else {
					it.description = get_xml_content(node);
				}
			} else if (mode == "escaped") {
				it.description = get_content(node);
			}
			it.description_type = type;
			if (it.description_type == "")
				it.description_type = "text";
		} else if (node_is(node, "id")) {
			it.guid = get_content(node);
			it.guid_isPermaLink = false;
		} else if (node_is(node, "published")) {
			it.pubDate = w3cdtf_to_rfc822(get_content(node));
		} else if (node_is(node, "updated")) {
			updated = w3cdtf_to_rfc822(get_content(node));
		} else if (node_is(node, "link")) {
			std::string rel = get_prop(node, "rel");
			if (rel == "" || rel == "alternate") {
				it.link = newsbeuter::utils::absolute_url(base, get_prop(node, "href"));
			} else if (rel == "enclosure") {
				it.enclosure_url = get_prop(node, "href");
				it.enclosure_type = get_prop(node, "type");
			}
		} else if (node_is(node, "summary")) {
			std::string mode = get_prop(node, "mode");
			summary_type = get_prop(node, "type");
			if (mode == "xml" || mode == "") {
				if (summary_type == "html" || summary_type == "text") {
					summary = get_content(node);
				} else {
					summary = get_xml_content(node);
				}
			} else if (mode == "escaped") {
				summary = get_content(node);
			}
			if (summary_type == "")
				summary_type = "text";
		}
	} // for

	if (it.description == "") {
		it.description = summary;
		it.description_type = summary_type;
	}

	if (it.pubDate == "") {
		it.pubDate = updated;
	}

	return it;
}
{'fraction_non_alphanumeric': 0.18045216432313207, 'fraction_numerical': 0.005514199062586159, 'mean_word_length': 3.735639686684073, 'pattern_counts': {'":': 0, '<': 17, '<?xml version=': 0, '>': 29, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 18}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4892918', 'n_tokens_mistral': 3021, 'n_tokens_neox': 2660, 'n_words': 582}