<commit_before>/*	opendatacon
 *
 *	Copyright (c) 2014:
 *
 *		DCrip3fJguWgVCLrZFfA7sIGgvx1Ou3fHfCxnrz4svAi
 *		yxeOtDhDCXf1Z4ApgXvX5ahqQmzRfJ2DoX8S05SqHA==
 *	
 *	Licensed under the Apache License, Version 2.0 (the "License");
 *	you may not use this file except in compliance with the License.
 *	You may obtain a copy of the License at
 *	
 *		http://www.apache.org/licenses/LICENSE-2.0
 *
 *	Unless required by applicable law or agreed to in writing, software
 *	distributed under the License is distributed on an "AS IS" BASIS,
 *	WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 *	See the License for the specific language governing permissions and
 *	limitations under the License.
 */ 
/*
 * DataConcentrator.cpp
 *
 *  Created on: 15/07/2014
 *      Author: <NAME> <<EMAIL>>
 */

#include <dlfcn.h>
#include <thread>
#include <asio.hpp>
#include <opendnp3/LogLevels.h>

#include "DataConcentrator.h"
#include "Console.h"

#include <asiodnp3/ConsoleLogger.h>
#include "logging_cmds.h"
#include "DNP3OutstationPort.h"
#include "DNP3MasterPort.h"
#include "../JSONPort/JSONClientPort.h"
#include "NullPort.h"

DataConcentrator::DataConcentrator(std::string FileName):
	DNP3Mgr(std::thread::hardware_concurrency()),
	LOG_LEVEL(opendnp3::levels::NORMAL),
	AdvConsoleLog(asiodnp3::ConsoleLogger::Instance(),LOG_LEVEL),
	FileLog("datacon_log"),
	AdvFileLog(FileLog,LOG_LEVEL),
	IOS(std::thread::hardware_concurrency()),
	ios_working(new asio::io_service::work(IOS))
{
	//fire up some worker threads
	for(size_t i=0; i < std::thread::hardware_concurrency(); ++i)
		std::thread([&](){IOS.run();}).detach();

	AdvConsoleLog.AddIngoreAlways(".*"); //silence all console messages by default
	DNP3Mgr.AddLogSubscriber(&AdvConsoleLog);
	DNP3Mgr.AddLogSubscriber(&AdvFileLog);

	//Parse the configs and create all the ports and connections
	ProcessFile(FileName);

	for(auto& port : DataPorts)
	{
		port.second->AddLogSubscriber(&AdvConsoleLog);
		port.second->AddLogSubscriber(&AdvFileLog);
		port.second->SetIOS(&IOS);
		port.second->SetLogLevel(LOG_LEVEL);
	}
	for(auto& conn : DataConnectors)
	{
		conn.second->AddLogSubscriber(&AdvConsoleLog);
		conn.second->AddLogSubscriber(&AdvFileLog);
		conn.second->SetIOS(&IOS);
		conn.second->SetLogLevel(LOG_LEVEL);
	}
}
DataConcentrator::~DataConcentrator()
{
	//turn everything off
	this->Shutdown();
	DNP3Mgr.Shutdown();
	//tell the io service to let it's run functions return once there's no handlers left (letting our threads end)
	ios_working.reset();
	//help finish any work
	IOS.run();
}

void DataConcentrator::ProcessElements(const Json::Value& JSONRoot)
{
	if(!JSONRoot["LogFileSizekB"].isNull())
		FileLog.SetLogFileSizekB(JSONRoot["LogFileSizekB"].asUInt());

	if(!JSONRoot["NumLogFiles"].isNull())
		FileLog.SetNumLogFiles(JSONRoot["NumLogFiles"].asUInt());

	if(!JSONRoot["LogName"].isNull())
		FileLog.SetLogName(JSONRoot["LogName"].asString());

	if(!JSONRoot["LOG_LEVEL"].isNull())
	{
		std::string value = JSONRoot["LOG_LEVEL"].asString();
		if(value == "ALL")
			LOG_LEVEL = opendnp3::levels::ALL;
		else if(value == "ALL_COMMS")
			LOG_LEVEL = opendnp3::levels::ALL_COMMS;
		else if(value == "NORMAL")
			LOG_LEVEL = opendnp3::levels::NORMAL;
		else if(value == "NOTHING")
			LOG_LEVEL = opendnp3::levels::NOTHING;
		else
			std::cout << "Warning: invalid LOG_LEVEL setting: '" << value << "' : ignoring and using 'NORMAL' log level." << std::endl;
		AdvFileLog.SetLogLevel(LOG_LEVEL);
		AdvConsoleLog.SetLogLevel(LOG_LEVEL);
	}

	if(!JSONRoot["Ports"].isNull())
	{
		const Json::Value Ports = JSONRoot["Ports"];

		for(Json::Value::ArrayIndex n = 0; n < Ports.size(); ++n)
		{
			if(Ports[n]["Type"].isNull() || Ports[n]["Name"].isNull() || Ports[n]["ConfFilename"].isNull())
			{
				std::cout<<"Warning: invalid port config: need at least Type, Name, ConfFilename: \n'"<<Ports[n].toStyledString()<<"\n' : ignoring"<<std::endl;
			}
			if(Ports[n]["Type"].asString() == "DNP3Outstation")
			{
				DataPorts[Ports[n]["Name"].asString()] = std::unique_ptr<DataPort>(new DNP3OutstationPort(Ports[n]["Name"].asString(), Ports[n]["ConfFilename"].asString(), Ports[n]["ConfOverrides"].asString()));
			}
			else if(Ports[n]["Type"].asString() == "DNP3Master")
			{
				DataPorts[Ports[n]["Name"].asString()] = std::unique_ptr<DataPort>(new DNP3MasterPort(Ports[n]["Name"].asString(), Ports[n]["ConfFilename"].asString(), Ports[n]["ConfOverrides"].asString()));
			}
			else if(Ports[n]["Type"].asString() == "Null")
			{
				DataPorts[Ports[n]["Name"].asString()] = std::unique_ptr<DataPort>(new NullPort(Ports[n]["Name"].asString(), Ports[n]["ConfFilename"].asString(), Ports[n]["ConfOverrides"].asString()));
			}
			else
			{
				std::string libname;
				if(Ports[n]["Library"].isNull())
				{
					libname = "lib"+Ports[n]["Type"].asString()+"Port.so";
				}
				else
				{
					libname = "lib"+Ports[n]["Library"].asString()+".so";
				}

				void* portlib = dlopen(libname.c_str(),RTLD_LAZY);
				if(portlib == nullptr)
				{
					std::cout << "Warning: failed to load library '"<<libname<<"' skipping port..."<<std::endl;
					continue;
				}
				std::string new_funcname = "new_"+Ports[n]["Type"].asString()+"Port";
				auto new_port_func = (DataPort*(*)(std::string,std::string,std::string))dlsym(portlib, new_funcname.c_str());
				if(new_port_func == nullptr)
				{
					std::cout << "Warning: failed to load symbol '"<<new_funcname<<"' for port type '"<<Ports[n]["Type"].asString()<<"' skipping port..."<<std::endl;
					continue;
				}

				DataPorts[Ports[n]["Name"].asString()] = std::unique_ptr<DataPort>(new_port_func(Ports[n]["Name"].asString(), Ports[n]["ConfFilename"].asString(), Ports[n]["ConfOverrides"].asString()));
			}
		}
	}

	if(!JSONRoot["Connectors"].isNull())
	{
		const Json::Value Connectors = JSONRoot["Connectors"];

		for(Json::Value::ArrayIndex n = 0; n < Connectors.size(); ++n)
		{
			if(Connectors[n]["Name"].isNull() || Connectors[n]["ConfFilename"].isNull())
			{
				std::cout<<"Warning: invalid Connector config: need at least Name, ConfFilename: \n'"<<Connectors[n].toStyledString()<<"\n' : ignoring"<<std::endl;
			}
			DataConnectors[Connectors[n]["Name"].asString()] = std::unique_ptr<DataConnector>(new DataConnector(Connectors[n]["Name"].asString(), Connectors[n]["ConfFilename"].asString(), Connectors[n]["ConfOverrides"].asString()));
		}
	}
}
void DataConcentrator::BuildOrRebuild()
{
	for(auto& Name_n_Port : DataPorts)
	{
		Name_n_Port.second->BuildOrRebuild(DNP3Mgr,LOG_LEVEL);
	}
	for(auto& Name_n_Conn : DataConnectors)
	{
		Name_n_Conn.second->BuildOrRebuild(DNP3Mgr,LOG_LEVEL);<commit_msg>Added some comments to the dynamic loading of DataPort libs<commit_after>/*	opendatacon
 *
 *	Copyright (c) 2014:
 *
 *		DCrip3fJguWgVCLrZFfA7sIGgvx1Ou3fHfCxnrz4svAi
 *		yxeOtDhDCXf1Z4ApgXvX5ahqQmzRfJ2DoX8S05SqHA==
 *	
 *	Licensed under the Apache License, Version 2.0 (the "License");
 *	you may not use this file except in compliance with the License.
 *	You may obtain a copy of the License at
 *	
 *		http://www.apache.org/licenses/LICENSE-2.0
 *
 *	Unless required by applicable law or agreed to in writing, software
 *	distributed under the License is distributed on an "AS IS" BASIS,
 *	WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 *	See the License for the specific language governing permissions and
 *	limitations under the License.
 */ 
/*
 * DataConcentrator.cpp
 *
 *  Created on: 15/07/2014
 *      Author: <NAME> <<EMAIL>>
 */

#include <dlfcn.h>
#include <thread>
#include <asio.hpp>
#include <opendnp3/LogLevels.h>

#include "DataConcentrator.h"
#include "Console.h"

#include <asiodnp3/ConsoleLogger.h>
#include "logging_cmds.h"
#include "DNP3OutstationPort.h"
#include "DNP3MasterPort.h"
#include "../JSONPort/JSONClientPort.h"
#include "NullPort.h"

DataConcentrator::DataConcentrator(std::string FileName):
	DNP3Mgr(std::thread::hardware_concurrency()),
	LOG_LEVEL(opendnp3::levels::NORMAL),
	AdvConsoleLog(asiodnp3::ConsoleLogger::Instance(),LOG_LEVEL),
	FileLog("datacon_log"),
	AdvFileLog(FileLog,LOG_LEVEL),
	IOS(std::thread::hardware_concurrency()),
	ios_working(new asio::io_service::work(IOS))
{
	//fire up some worker threads
	for(size_t i=0; i < std::thread::hardware_concurrency(); ++i)
		std::thread([&](){IOS.run();}).detach();

	AdvConsoleLog.AddIngoreAlways(".*"); //silence all console messages by default
	DNP3Mgr.AddLogSubscriber(&AdvConsoleLog);
	DNP3Mgr.AddLogSubscriber(&AdvFileLog);

	//Parse the configs and create all the ports and connections
	ProcessFile(FileName);

	for(auto& port : DataPorts)
	{
		port.second->AddLogSubscriber(&AdvConsoleLog);
		port.second->AddLogSubscriber(&AdvFileLog);
		port.second->SetIOS(&IOS);
		port.second->SetLogLevel(LOG_LEVEL);
	}
	for(auto& conn : DataConnectors)
	{
		conn.second->AddLogSubscriber(&AdvConsoleLog);
		conn.second->AddLogSubscriber(&AdvFileLog);
		conn.second->SetIOS(&IOS);
		conn.second->SetLogLevel(LOG_LEVEL);
	}
}
DataConcentrator::~DataConcentrator()
{
	//turn everything off
	this->Shutdown();
	DNP3Mgr.Shutdown();
	//tell the io service to let it's run functions return once there's no handlers left (letting our threads end)
	ios_working.reset();
	//help finish any work
	IOS.run();
}

void DataConcentrator::ProcessElements(const Json::Value& JSONRoot)
{
	if(!JSONRoot["LogFileSizekB"].isNull())
		FileLog.SetLogFileSizekB(JSONRoot["LogFileSizekB"].asUInt());

	if(!JSONRoot["NumLogFiles"].isNull())
		FileLog.SetNumLogFiles(JSONRoot["NumLogFiles"].asUInt());

	if(!JSONRoot["LogName"].isNull())
		FileLog.SetLogName(JSONRoot["LogName"].asString());

	if(!JSONRoot["LOG_LEVEL"].isNull())
	{
		std::string value = JSONRoot["LOG_LEVEL"].asString();
		if(value == "ALL")
			LOG_LEVEL = opendnp3::levels::ALL;
		else if(value == "ALL_COMMS")
			LOG_LEVEL = opendnp3::levels::ALL_COMMS;
		else if(value == "NORMAL")
			LOG_LEVEL = opendnp3::levels::NORMAL;
		else if(value == "NOTHING")
			LOG_LEVEL = opendnp3::levels::NOTHING;
		else
			std::cout << "Warning: invalid LOG_LEVEL setting: '" << value << "' : ignoring and using 'NORMAL' log level." << std::endl;
		AdvFileLog.SetLogLevel(LOG_LEVEL);
		AdvConsoleLog.SetLogLevel(LOG_LEVEL);
	}

	if(!JSONRoot["Ports"].isNull())
	{
		const Json::Value Ports = JSONRoot["Ports"];

		for(Json::Value::ArrayIndex n = 0; n < Ports.size(); ++n)
		{
			if(Ports[n]["Type"].isNull() || Ports[n]["Name"].isNull() || Ports[n]["ConfFilename"].isNull())
			{
				std::cout<<"Warning: invalid port config: need at least Type, Name, ConfFilename: \n'"<<Ports[n].toStyledString()<<"\n' : ignoring"<<std::endl;
			}
			if(Ports[n]["Type"].asString() == "DNP3Outstation")
			{
				DataPorts[Ports[n]["Name"].asString()] = std::unique_ptr<DataPort>(new DNP3OutstationPort(Ports[n]["Name"].asString(), Ports[n]["ConfFilename"].asString(), Ports[n]["ConfOverrides"].asString()));
			}
			else if(Ports[n]["Type"].asString() == "DNP3Master")
			{
				DataPorts[Ports[n]["Name"].asString()] = std::unique_ptr<DataPort>(new DNP3MasterPort(Ports[n]["Name"].asString(), Ports[n]["ConfFilename"].asString(), Ports[n]["ConfOverrides"].asString()));
			}
			else if(Ports[n]["Type"].asString() == "Null")
			{
				DataPorts[Ports[n]["Name"].asString()] = std::unique_ptr<DataPort>(new NullPort(Ports[n]["Name"].asString(), Ports[n]["ConfFilename"].asString(), Ports[n]["ConfOverrides"].asString()));
			}
			else
			{
				//Looks for a specific library (for libs that implement more than one class)
				std::string libname;
				if(!Ports[n]["Library"].isNull())
				{
					libname = "lib"+Ports[n]["Library"].asString()+".so";
				}
				//Otherwise use the naming convention lib<Type>Port.so to find the default lib that implements a type of port
				else
				{
					libname = "lib"+Ports[n]["Type"].asString()+"Port.so";
				}

				//try to load the lib
				void* portlib = dlopen(libname.c_str(),RTLD_LAZY);
				if(portlib == nullptr)
				{
					std::cout << "Warning: failed to load library '"<<libname<<"' skipping port..."<<std::endl;
					continue;
				}

				//Our API says the library should export a creation function: DataPort* new_<Type>Port(Name, Filename, Overrides)
				//it should return a pointer to a heap allocated instance of a descendant of DataPort
				std::string new_funcname = "new_"+Ports[n]["Type"].asString()+"Port";
				auto new_port_func = (DataPort*(*)(std::string,std::string,std::string))dlsym(portlib, new_funcname.c_str());
				if(new_port_func == nullptr)
				{
					std::cout << "Warning: failed to load symbol '"<<new_funcname<<"' for port type '"<<Ports[n]["Type"].asString()<<"' skipping port..."<<std::endl;
					continue;
				}

				//call the creation function and wrap the returned pointer to a new port
				DataPorts[Ports[n]["Name"].asString()] = std::unique_ptr<DataPort>(new_port_func(Ports[n]["Name"].asString(), Ports[n]["ConfFilename"].asString(), Ports[n]["ConfOverrides"].asString()));
			}
		}
	}

	if(!JSONRoot["Connectors"].isNull())
	{
		const Json::Value Connectors = JSONRoot["Connectors"];

		for(Json::Value::ArrayIndex n = 0; n < Connectors.size(); ++n)
		{
			if(Connectors[n]["Name"].isNull() || Connectors[n]["ConfFilename"].isNull())
			{
				std::cout<<"Warning: invalid Connector config: need at least Name, ConfFilename: \n'"<<Connectors[n].toStyledString()<<"\n' : ignoring"<<std::endl;
			}
			DataConnectors[Connectors[n]["Name"].asString()] = std::unique_ptr<DataConnector>(new DataConnector(Connectors[n]["Name"].asString(), Connectors[n]["ConfFilename"].asString(), Connectors[n]["ConfOverrides"].asString()));
		}
	}
}
void DataConcentrator::BuildOrRebuild()
{
	for(auto& Name_n_Port : DataPorts)
	{
		Name_n_Port.second->BuildOrRebuild(DNP3Mgr,LOG_LEVEL);
	}
	for(auto& Name_n_Conn : DataConnectors)
	{
		Name_n_Conn.second->BuildOrRebuild(DNP3Mgr,LOG_LEVEL);
{'fraction_non_alphanumeric': 0.1804630969609262, 'fraction_numerical': 0.007380607814761216, 'mean_word_length': 5.522416234072676, 'pattern_counts': {'":': 0, '<': 125, '<?xml version=': 0, '>': 53, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5415509', 'n_tokens_mistral': 5405, 'n_tokens_neox': 4965, 'n_words': 865}