<commit_before>/* rsspp - Copyright (C) 2008-2009 <NAME> <<EMAIL>>
 * Licensed under the MIT/X Consortium License. See file LICENSE
 * for more information.
 */

#include <rsspp.h>
#include <rsspp_internal.h>
#include <libxml/parser.h>
#include <libxml/tree.h>
#include <curl/curl.h>
#include <logger.h>
#include <cstring>

using namespace newsbeuter;

static size_t my_write_data(void *buffer, size_t size, size_t nmemb, void *userp) {
	std::string * pbuf = static_cast<std::string *>(userp);
	pbuf->append(static_cast<const char *>(buffer), size * nmemb);
	return size * nmemb;
}

namespace rsspp {

parser::parser(unsigned int timeout, const char * user_agent, const char * proxy, const char * proxy_auth) 
	: to(timeout), ua(user_agent), prx(proxy), prxauth(proxy_auth), doc(0) {
}

parser::~parser() {
	if (doc)
		xmlFreeDoc(doc);
	xmlCleanupParser();
}

feed parser::parse_url(const std::string& url) {
	std::string buf;
	CURLcode ret;

	CURL * easyhandle = curl_easy_init();
	if (!easyhandle) {
		throw exception(0, "curl_easy_init() failed");
	}

	if (ua) {
		curl_easy_setopt(easyhandle, CURLOPT_USERAGENT, ua);
	}
	curl_easy_setopt(easyhandle, CURLOPT_URL, url.c_str());
	curl_easy_setopt(easyhandle, CURLOPT_WRITEFUNCTION, my_write_data);
	curl_easy_setopt(easyhandle, CURLOPT_WRITEDATA, &buf);
	curl_easy_setopt(easyhandle, CURLOPT_NOSIGNAL, 1);
	curl_easy_setopt(easyhandle, CURLOPT_FOLLOWLOCATION, 1);
	curl_easy_setopt(easyhandle, CURLOPT_FAILONERROR, 1);<commit_msg>remove early cleanup.<commit_after>/* rsspp - Copyright (C) 2008-2009 <NAME> <<EMAIL>>
 * Licensed under the MIT/X Consortium License. See file LICENSE
 * for more information.
 */

#include <rsspp.h>
#include <rsspp_internal.h>
#include <libxml/parser.h>
#include <libxml/tree.h>
#include <curl/curl.h>
#include <logger.h>
#include <cstring>

using namespace newsbeuter;

static size_t my_write_data(void *buffer, size_t size, size_t nmemb, void *userp) {
	std::string * pbuf = static_cast<std::string *>(userp);
	pbuf->append(static_cast<const char *>(buffer), size * nmemb);
	return size * nmemb;
}

namespace rsspp {

parser::parser(unsigned int timeout, const char * user_agent, const char * proxy, const char * proxy_auth) 
	: to(timeout), ua(user_agent), prx(proxy), prxauth(proxy_auth), doc(0) {
}

parser::~parser() {
	if (doc)
		xmlFreeDoc(doc);
}

feed parser::parse_url(const std::string& url) {
	std::string buf;
	CURLcode ret;

	CURL * easyhandle = curl_easy_init();
	if (!easyhandle) {
		throw exception(0, "curl_easy_init() failed");
	}

	if (ua) {
		curl_easy_setopt(easyhandle, CURLOPT_USERAGENT, ua);
	}
	curl_easy_setopt(easyhandle, CURLOPT_URL, url.c_str());
	curl_easy_setopt(easyhandle, CURLOPT_WRITEFUNCTION, my_write_data);
	curl_easy_setopt(easyhandle, CURLOPT_WRITEDATA, &buf);
	curl_easy_setopt(easyhandle, CURLOPT_NOSIGNAL, 1);
	curl_easy_setopt(easyhandle, CURLOPT_FOLLOWLOCATION, 1);
	curl_easy_setopt(easyhandle, CURLOPT_FAILONERROR, 1);
{'fraction_non_alphanumeric': 0.137580481192816, 'fraction_numerical': 0.00881057268722467, 'mean_word_length': 6.235294117647059, 'pattern_counts': {'":': 0, '<': 27, '<?xml version=': 0, '>': 29, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 7}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4575313', 'n_tokens_mistral': 1222, 'n_tokens_neox': 1099, 'n_words': 244}