<commit_before>/*
    Flexisip, a flexible SIP proxy server with media capabilities.
    Copyright (C) 2010-2015  Belledonne Communications SARL, All rights reserved.

    This program is free software: you can redistribute it and/or modify
    it under the terms of the GNU Affero General Public License as
    published by the Free Software Foundation, either version 3 of the
    License, or (at your option) any later version.

    This program is distributed in the hope that it will be useful,
    but WITHOUT ANY WARRANTY; without even the implied warranty of
    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
    GNU Affero General Public License for more details.

    You should have received a copy of the GNU Affero General Public License
    along with this program.  If not, see <http://www.gnu.org/licenses/>.
*/

#include "module.hh"
#include "agent.hh"
#include "transaction.hh"
#include "etchosts.hh"
#include <sstream>

#include <sofia-sip/su_md5.h>
#include <sofia-sip/sip_status.h>
#include <sofia-sip/tport.h>

using namespace ::std;

class StatisticsCollector: public Module, ModuleToolbox {
public:
	StatisticsCollector(Agent *ag);
	~StatisticsCollector();
	virtual void onDeclare(GenericStruct * module_config);
	virtual void onLoad(const GenericStruct *root);
	virtual void onRequest(shared_ptr<RequestSipEvent> &ev);
	virtual void onResponse(shared_ptr<ResponseSipEvent> &ev);
private:
	int managePublishContent(const shared_ptr<RequestSipEvent> ev);
	bool containsMandatoryFields(char *data, usize_t len);

	static ModuleInfo<StatisticsCollector> sInfo;
	url_t *mCollectorAddress;
	su_home_t mHome;
};

StatisticsCollector::StatisticsCollector(Agent *ag) : Module(ag), mCollectorAddress(NULL) {
	su_home_init(&mHome);
}

StatisticsCollector::~StatisticsCollector(){
	su_home_deinit(&mHome);
}

void StatisticsCollector::onDeclare(GenericStruct * module_config) {
	ConfigItemDescriptor items[] = {
			{ String, "collector-address", "SIP URI of the statistics collector. "
			"Note that application/vq-rtcpxr messages for this address will be deleted by this module and thus not be delivered.", "" },

			config_item_end
	};
	module_config->addChildrenValues(items);

	/* modify the default value for "enabled" */
	module_config->get<ConfigBoolean>("enabled")->setDefault("false");
}

void StatisticsCollector::onLoad(const GenericStruct *mc) {
	string value = mc->get<ConfigString>("collector-address")->read();
	if (value.size()>0){
		mCollectorAddress=url_make(&mHome,value.c_str());
		if (mCollectorAddress==NULL){
			LOGF("StatisticsCollector: Invalid collector address '%s'",value.c_str());
		}
	}else mCollectorAddress=NULL;
}

void StatisticsCollector::onRequest(shared_ptr<RequestSipEvent> &ev) {
	const shared_ptr<MsgSip> &ms = ev->getMsgSip();
	sip_t *sip = ms->getSip();
	url_t *url = sip->sip_request->rq_url;
	// verify collector address AND content type
	if (mCollectorAddress
		&& (url_cmp(mCollectorAddress,url)==0)
		&& (strcmp("application/vq-rtcpxr", sip->sip_content_type->c_type) == 0)
		&& (strcmp("vq-rtcpxr", sip->sip_content_type->c_subtype) != 0)) {
		// some treatment
		int err = managePublishContent(ev);
		ev->reply(err, sip_status_phrase(err), SIPTAG_SERVER_STR(getAgent()->getServerString()), TAG_END());
		return;
	}
}

void StatisticsCollector::onResponse(shared_ptr<ResponseSipEvent> &ev) {
}

/*avoid crash if x is NULL on libc versions <4.5.26 */
#define __strstr(x, y) ((x==NULL)?NULL:strstr(x,y))

bool StatisticsCollector::containsMandatoryFields(char *body, usize_t len) {
	char * remote_metrics_start = __strstr(body, "RemoteMetrics:");

	if (!(
		__strstr(body, "VQIntervalReport\r\n")			== body ||
		__strstr(body, "VQSessionReport\r\n")			== body ||
		__strstr(body, "VQSessionReport: CallTerm\r\n") == body))
		return false;

	if (!(body=__strstr(body, "CallID:"))) return false;
	if (!(body=__strstr(body, "LocalID:"))) return false;
	if (!(body=__strstr(body, "RemoteID:"))) return false;
	if (!(body=__strstr(body, "OrigID:"))) return false;
	if (!(body=__strstr(body, "LocalGroup:"))) return false;
	if (!(body=__strstr(body, "RemoteGroup:"))) return false;
	if (!(body=__strstr(body, "LocalAddr:"))) return false;
		if (!(body=__strstr(body, "IP="))) return false;
		if (!(body=__strstr(body, "PORT="))) return false;
		if (!(body=__strstr(body, "SSRC="))) return false;
	if (!(body=__strstr(body, "RemoteAddr:"))) return false;
		if (!(body=__strstr(body, "IP="))) return false;
		if (!(body=__strstr(body, "PORT="))) return false;
		if (!(body=__strstr(body, "SSRC="))) return false;
	if (!(body=__strstr(body, "LocalMetrics:"))) return false;
	if (!(body=__strstr(body, "Timestamps:"))) return false;
		if (!(body=__strstr(body, "START="))) return false;
		if (!(body=__strstr(body, "STOP="))) return false;

	/* We should have not reached RemoteMetrics section yet */
	if (remote_metrics_start && body >= remote_metrics_start) return false;

	return true;
}

int StatisticsCollector::managePublishContent(const shared_ptr<RequestSipEvent> ev) {
	const shared_ptr<MsgSip> &ms = ev->getMsgSip();
	sip_t *sip = ms->getSip();
	int err = 200;
	std::string statusPhrase = "OK";

	if (! sip) {
		err = 400;
		statusPhrase = "Invalid SIP";
	}

	// verify that packet contains data
	if (! sip->sip_payload || sip->sip_payload->pl_len == 0 || ! sip->sip_payload->pl_data ) {
		err = 606;
		statusPhrase = "No data in packet payload";
	// verify that packet contains mandatory fields
	} else if (! containsMandatoryFields(sip->sip_payload->pl_data, sip->sip_payload->pl_len)) {
		err = 606;
		statusPhrase = "One or several mandatory fields missing";
	}

	auto log=make_shared<CallQualityStatisticsLog>(sip->sip_from, sip->sip_to,sip->sip_payload?sip->sip_payload->pl_data:NULL);
	log->setStatusCode(err,statusPhrase.c_str());
	if (sip->sip_user_agent) log->setUserAgent(sip->sip_user_agent);
	log->setCompleted();
	ev->setEventLog(log);

	return err;
}

ModuleInfo<StatisticsCollector> StatisticsCollector::sInfo("StatisticsCollector", "The purpose of the StatisticsCollector module is to "
		"collect call statistics (RFC 6035) and store them on the server.",
		ModuleInfoBase::ModuleOid::StatisticsCollector);
<commit_msg>module-statisics-collector.cc: add some logging on setup and on receive<commit_after>/*
    Flexisip, a flexible SIP proxy server with media capabilities.
    Copyright (C) 2010-2015  Belledonne Communications SARL, All rights reserved.

    This program is free software: you can redistribute it and/or modify
    it under the terms of the GNU Affero General Public License as
    published by the Free Software Foundation, either version 3 of the
    License, or (at your option) any later version.

    This program is distributed in the hope that it will be useful,
    but WITHOUT ANY WARRANTY; without even the implied warranty of
    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
    GNU Affero General Public License for more details.

    You should have received a copy of the GNU Affero General Public License
    along with this program.  If not, see <http://www.gnu.org/licenses/>.
*/

#include "module.hh"
#include "agent.hh"
#include "transaction.hh"
#include "etchosts.hh"
#include <sstream>

#include <sofia-sip/su_md5.h>
#include <sofia-sip/sip_status.h>
#include <sofia-sip/tport.h>

using namespace ::std;

class StatisticsCollector: public Module, ModuleToolbox {
public:
	StatisticsCollector(Agent *ag);
	~StatisticsCollector();
	virtual void onDeclare(GenericStruct * module_config);
	virtual void onLoad(const GenericStruct *root);
	virtual void onRequest(shared_ptr<RequestSipEvent> &ev);
	virtual void onResponse(shared_ptr<ResponseSipEvent> &ev);
private:
	int managePublishContent(const shared_ptr<RequestSipEvent> ev);
	bool containsMandatoryFields(char *data, usize_t len);

	static ModuleInfo<StatisticsCollector> sInfo;
	url_t *mCollectorAddress;
	su_home_t mHome;
};

StatisticsCollector::StatisticsCollector(Agent *ag) : Module(ag), mCollectorAddress(NULL) {
	su_home_init(&mHome);
}

StatisticsCollector::~StatisticsCollector(){
	su_home_deinit(&mHome);
}

void StatisticsCollector::onDeclare(GenericStruct * module_config) {
	ConfigItemDescriptor items[] = {
			{ String, "collector-address", "SIP URI of the statistics collector. "
			"Note that application/vq-rtcpxr messages for this address will be deleted by this module and thus not be delivered.", "" },

			config_item_end
	};
	module_config->addChildrenValues(items);

	/* modify the default value for "enabled" */
	module_config->get<ConfigBoolean>("enabled")->setDefault("false");
}

void StatisticsCollector::onLoad(const GenericStruct *mc) {
	string value = mc->get<ConfigString>("collector-address")->read();
	if (value.size()>0){
		mCollectorAddress=url_make(&mHome,value.c_str());
		if (mCollectorAddress==NULL){
			LOGF("StatisticsCollector: Invalid collector address '%s'",value.c_str());
		}
	}else{
		mCollectorAddress=NULL;
	}
	LOGI("StatisticsCollector: setup with collector address '%s'",value.c_str());
}

void StatisticsCollector::onRequest(shared_ptr<RequestSipEvent> &ev) {
	const shared_ptr<MsgSip> &ms = ev->getMsgSip();
	sip_t *sip = ms->getSip();
	url_t *url = sip->sip_request->rq_url;
	// verify collector address AND content type
	if (mCollectorAddress && (url_cmp(mCollectorAddress,url)==0)) {
		if ((strcmp("application/vq-rtcpxr", sip->sip_content_type->c_type) == 0)
			&& (strcmp("vq-rtcpxr", sip->sip_content_type->c_subtype) != 0)) {
			// some treatment
			int err = managePublishContent(ev);
			ev->reply(err, sip_status_phrase(err), SIPTAG_SERVER_STR(getAgent()->getServerString()), TAG_END());
		} else {
			LOGI("StatisticsCollector: received PUBLISH for collector %s with invalid type, ignoring", mCollectorAddress->url_user);
		}
	}
}

void StatisticsCollector::onResponse(shared_ptr<ResponseSipEvent> &ev) {
}

/*avoid crash if x is NULL on libc versions <4.5.26 */
#define __strstr(x, y) ((x==NULL)?NULL:strstr(x,y))

bool StatisticsCollector::containsMandatoryFields(char *body, usize_t len) {
	char * remote_metrics_start = __strstr(body, "RemoteMetrics:");

	if (!(
		__strstr(body, "VQIntervalReport\r\n")			== body ||
		__strstr(body, "VQSessionReport\r\n")			== body ||
		__strstr(body, "VQSessionReport: CallTerm\r\n") == body))
		return false;

	if (!(body=__strstr(body, "CallID:"))) return false;
	if (!(body=__strstr(body, "LocalID:"))) return false;
	if (!(body=__strstr(body, "RemoteID:"))) return false;
	if (!(body=__strstr(body, "OrigID:"))) return false;
	if (!(body=__strstr(body, "LocalGroup:"))) return false;
	if (!(body=__strstr(body, "RemoteGroup:"))) return false;
	if (!(body=__strstr(body, "LocalAddr:"))) return false;
		if (!(body=__strstr(body, "IP="))) return false;
		if (!(body=__strstr(body, "PORT="))) return false;
		if (!(body=__strstr(body, "SSRC="))) return false;
	if (!(body=__strstr(body, "RemoteAddr:"))) return false;
		if (!(body=__strstr(body, "IP="))) return false;
		if (!(body=__strstr(body, "PORT="))) return false;
		if (!(body=__strstr(body, "SSRC="))) return false;
	if (!(body=__strstr(body, "LocalMetrics:"))) return false;
	if (!(body=__strstr(body, "Timestamps:"))) return false;
		if (!(body=__strstr(body, "START="))) return false;
		if (!(body=__strstr(body, "STOP="))) return false;

	/* We should have not reached RemoteMetrics section yet */
	if (remote_metrics_start && body >= remote_metrics_start) return false;

	return true;
}

int StatisticsCollector::managePublishContent(const shared_ptr<RequestSipEvent> ev) {
	const shared_ptr<MsgSip> &ms = ev->getMsgSip();
	sip_t *sip = ms->getSip();
	int err = 200;
	std::string statusPhrase = "OK";

	if (! sip) {
		err = 400;
		statusPhrase = "Invalid SIP";
	}

	// verify that packet contains data
	if (! sip->sip_payload || sip->sip_payload->pl_len == 0 || ! sip->sip_payload->pl_data ) {
		err = 606;
		statusPhrase = "No data in packet payload";
	// verify that packet contains mandatory fields
	} else if (! containsMandatoryFields(sip->sip_payload->pl_data, sip->sip_payload->pl_len)) {
		err = 606;
		statusPhrase = "One or several mandatory fields missing";
	}

	auto log=make_shared<CallQualityStatisticsLog>(sip->sip_from, sip->sip_to,sip->sip_payload?sip->sip_payload->pl_data:NULL);
	log->setStatusCode(err,statusPhrase.c_str());
	if (sip->sip_user_agent) log->setUserAgent(sip->sip_user_agent);
	log->setCompleted();
	ev->setEventLog(log);

	return err;
}

ModuleInfo<StatisticsCollector> StatisticsCollector::sInfo("StatisticsCollector", "The purpose of the StatisticsCollector module is to "
		"collect call statistics (RFC 6035) and store them on the server.",
		ModuleInfoBase::ModuleOid::StatisticsCollector);

{'fraction_non_alphanumeric': 0.1416312391749331, 'fraction_numerical': 0.005510943158557708, 'mean_word_length': 5.675249605885444, 'pattern_counts': {'":': 0, '<': 41, '<?xml version=': 0, '>': 118, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2118337', 'n_tokens_mistral': 4382, 'n_tokens_neox': 4197, 'n_words': 1078}