<commit_before>// ------------------------------------------------------------------------
// Pion is a development platform for building Reactors that process Events
// ------------------------------------------------------------------------
// Copyright (C) 2007-2010 Atomic Labs, Inc.  (http://www.atomiclabs.com)
//
// Pion is free software: you can redistribute it and/or modify it under the
// terms of the GNU Affero General Public License as published by the Free
// Software Foundation, either version 3 of the License, or (at your option)
// any later version.
//
// Pion is distributed in the hope that it will be useful, but WITHOUT ANY
// WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
// FOR A PARTICULAR PURPOSE.  See the GNU Affero General Public License for
// more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with Pion.  If not, see <http://www.gnu.org/licenses/>.
//

#include <string>
#include <set>
#include <list>
#include <pion/PionConfig.hpp>
#include <boost/test/unit_test.hpp>
#include <boost/filesystem/operations.hpp>
#include <pion/PionId.hpp>
#include <pion/platform/PionPlatformUnitTest.hpp>
#include <pion/platform/Event.hpp>
#include <pion/platform/VocabularyManager.hpp>
#include <pion/platform/ReactionEngine.hpp>
#include <pion/platform/CodecFactory.hpp>
#include <pion/platform/ProtocolFactory.hpp>
#include <pion/platform/DatabaseManager.hpp>

using namespace pion::platform;

// static strings used by these unit tests
static const std::string LOG_OUTPUT_FILE(LOG_FILE_DIR + "tr2.out");
static const std::string CLF_LOG_FILE(LOG_FILE_DIR + "tr2.log");

/// external functions defined in PionPlatformUnitTests.cpp
extern void cleanup_platform_config_files(void);


// The names ReactionEngineReadyToAddReactors2_F and ReactionEngineReadyToAddReactors2_S are used so that the unit test framework 
// won't confuse them with ReactionEngineReadyToAddReactors_F and ReactionEngineReadyToAddReactors_S in TransformReactorTests.
// Regrettably, it WILL compile without the 2's, but some tests will end up running with the wrong fixture.  
class ReactionEngineReadyToAddReactors2_F {
public:
	ReactionEngineReadyToAddReactors2_F() :
		m_vocab_mgr(), m_codec_factory(m_vocab_mgr), m_protocol_factory(m_vocab_mgr), m_database_mgr(m_vocab_mgr),
		m_reaction_engine(NULL)
	{
		cleanup_platform_config_files();
		
		m_vocab_mgr.setConfigFile(VOCABS_CONFIG_FILE);
		m_vocab_mgr.openConfigFile();
		m_codec_factory.setConfigFile(CODECS_CONFIG_FILE);
		m_codec_factory.openConfigFile();

		boost::filesystem::remove(LOG_OUTPUT_FILE);
		BOOST_REQUIRE(! boost::filesystem::exists(LOG_OUTPUT_FILE)); // Confirm, to avoid accidentally passing tests due to previous output.

		// Create a new (empty) reactor configuration file.
		m_reactor_config_file = CONFIG_FILE_DIR + "reactors1.xml";
		if (boost::filesystem::exists(m_reactor_config_file))
			boost::filesystem::remove(m_reactor_config_file);

		// Create and initialize a ReactionEngine.
		m_reaction_engine = new ReactionEngine(m_vocab_mgr, m_codec_factory, m_protocol_factory, m_database_mgr);
		m_reaction_engine->setConfigFile(m_reactor_config_file);
		m_reaction_engine->createConfigFile();
	}
	virtual ~ReactionEngineReadyToAddReactors2_F() {
		delete m_reaction_engine;
	}

	VocabularyManager	m_vocab_mgr;
	CodecFactory		m_codec_factory;
	ProtocolFactory		m_protocol_factory;
	DatabaseManager		m_database_mgr;
	ReactionEngine*		m_reaction_engine;
	std::string			m_reactor_config_file;
};

BOOST_FIXTURE_TEST_SUITE(ReactionEngineReadyToAddReactors2_S, ReactionEngineReadyToAddReactors2_F)

BOOST_AUTO_TEST_CASE(checkFilterCLFLogFile) {
	xmlNodePtr config_ptr = PionPlatformUnitTest::makeReactorConfigFromString(
		"<Plugin>FilterReactor</Plugin>"
		"<Comparison>"
			"<Term>urn:vocab:clickstream#c-ip</Term>"
			"<Type>starts-with</Type>"
			"<Value>66.249</Value>"
		"</Comparison>");
	std::string reactor_id = m_reaction_engine->addReactor(config_ptr);

	// Add a LogOutputReactor and connect it to the FilterReactor so it has something to send output to.
	config_ptr = PionPlatformUnitTest::makeReactorConfigFromString(
		"<Name>CLF Log Writer</Name>"
		"<Plugin>LogOutputReactor</Plugin>"
		"<Codec>3f49f2da-bfe3-11dc-8875-0016cb926e68</Codec>"
		"<Filename>../" + LOG_OUTPUT_FILE + "</Filename>");
	std::string log_writer_id = m_reaction_engine->addReactor(config_ptr);
	m_reaction_engine->addReactorConnection(reactor_id, log_writer_id);

	// Start the ReactionEngine.
	m_reaction_engine->start();

	// Read in events from a CLF log file.
	CodecPtr clf_codec_ptr = m_codec_factory.getCodec("3f49f2da-bfe3-11dc-8875-0016cb926e68");
	boost::uint64_t events_read = PionPlatformUnitTest::feedFileToReactor(
		*m_reaction_engine, reactor_id, *clf_codec_ptr, CLF_LOG_FILE);

	// Check that exactly 7 events passed the filter.
	PionPlatformUnitTest::checkReactorEventsOut(*m_reaction_engine, log_writer_id, static_cast<boost::uint64_t>(7));
}

BOOST_AUTO_TEST_SUITE_END()


class FilterReactorEventValidator_F : public ReactionEngineReadyToAddReactors2_F {
public:
	FilterReactorEventValidator_F() {
		m_num_events_validated = 0;
		m_page_event_ref = m_vocab_mgr.getVocabulary().findTerm("urn:vocab:clickstream#page-event");
		m_page_number_ref = m_vocab_mgr.getVocabulary().findTerm("urn:vocab:clickstream#page-number");
		m_sc_content_term_ref = m_vocab_mgr.getVocabulary().findTerm("urn:vocab:clickstream#sc-content");
	}

	typedef std::set<unsigned> ExpectedEventIds;

	void addEventWithContentTerm(const std::string& content, bool should_pass_filter) {
		EventPtr e(m_event_factory.create(m_page_event_ref));
		e->setUInt(m_page_number_ref, ++m_curr_event_id);
		e->setString(m_sc_content_term_ref, content);
		m_events.push_back(e);
		if (should_pass_filter)
			m_expected_event_ids.insert(m_curr_event_id);
	}

	void MockEventHandler(EventPtr& e, const ExpectedEventIds& expected_event_ids) {
		if (! e.get())
			return;

		// Check that the ID of the Event is in the set of expected event IDs.
		BOOST_CHECK(expected_event_ids.count(e->getUInt(m_page_number_ref)));

		++m_num_events_validated;
	}

	void sendEventsAndValidateOutput(std::list<EventPtr>& events, const std::string& reactor_id) {
		// Add an Event handler to check that the output of the FilterReactor is as expected.
		Reactor::EventHandler eventValidator = boost::bind(&FilterReactorEventValidator_F::MockEventHandler, this, _1, m_expected_event_ids);
		m_reaction_engine->addTempConnectionOut(reactor_id, pion::PionId().to_string(), "blah", eventValidator);

		// Start the ReactionEngine and send the Events to the FilterReactor.
		m_reaction_engine->start();
		for (std::list<EventPtr>::iterator i = events.begin(); i != events.end(); ++i)
			m_reaction_engine->send(reactor_id, *i);

		// Check that the expected number of Events were output by the FilterReactor and validated.
		int num_events_expected = m_expected_event_ids.size();
		PionPlatformUnitTest::checkReactorEventsOut(*m_reaction_engine, reactor_id, num_events_expected);
		BOOST_CHECK_EQUAL(m_num_events_validated, num_events_expected);
	}

	EventFactory		m_event_factory;<commit_msg>Patch to get new FilterReactorTests to work more reliably, especially on Linux and Mac where they seemed to be failing 100%<commit_after>// ------------------------------------------------------------------------
// Pion is a development platform for building Reactors that process Events
// ------------------------------------------------------------------------
// Copyright (C) 2007-2010 Atomic Labs, Inc.  (http://www.atomiclabs.com)
//
// Pion is free software: you can redistribute it and/or modify it under the
// terms of the GNU Affero General Public License as published by the Free
// Software Foundation, either version 3 of the License, or (at your option)
// any later version.
//
// Pion is distributed in the hope that it will be useful, but WITHOUT ANY
// WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
// FOR A PARTICULAR PURPOSE.  See the GNU Affero General Public License for
// more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with Pion.  If not, see <http://www.gnu.org/licenses/>.
//

#include <string>
#include <set>
#include <list>
#include <pion/PionConfig.hpp>
#include <boost/test/unit_test.hpp>
#include <boost/filesystem/operations.hpp>
#include <pion/PionId.hpp>
#include <pion/platform/PionPlatformUnitTest.hpp>
#include <pion/platform/Event.hpp>
#include <pion/platform/VocabularyManager.hpp>
#include <pion/platform/ReactionEngine.hpp>
#include <pion/platform/CodecFactory.hpp>
#include <pion/platform/ProtocolFactory.hpp>
#include <pion/platform/DatabaseManager.hpp>

using namespace pion::platform;

// static strings used by these unit tests
static const std::string LOG_OUTPUT_FILE(LOG_FILE_DIR + "tr2.out");
static const std::string CLF_LOG_FILE(LOG_FILE_DIR + "tr2.log");

/// external functions defined in PionPlatformUnitTests.cpp
extern void cleanup_platform_config_files(void);


// The names ReactionEngineReadyToAddReactors2_F and ReactionEngineReadyToAddReactors2_S are used so that the unit test framework 
// won't confuse them with ReactionEngineReadyToAddReactors_F and ReactionEngineReadyToAddReactors_S in TransformReactorTests.
// Regrettably, it WILL compile without the 2's, but some tests will end up running with the wrong fixture.  
class ReactionEngineReadyToAddReactors2_F {
public:
	ReactionEngineReadyToAddReactors2_F() :
		m_vocab_mgr(), m_codec_factory(m_vocab_mgr), m_protocol_factory(m_vocab_mgr), m_database_mgr(m_vocab_mgr),
		m_reaction_engine(NULL)
	{
		cleanup_platform_config_files();
		
		m_vocab_mgr.setConfigFile(VOCABS_CONFIG_FILE);
		m_vocab_mgr.openConfigFile();
		m_codec_factory.setConfigFile(CODECS_CONFIG_FILE);
		m_codec_factory.openConfigFile();

		boost::filesystem::remove(LOG_OUTPUT_FILE);
		BOOST_REQUIRE(! boost::filesystem::exists(LOG_OUTPUT_FILE)); // Confirm, to avoid accidentally passing tests due to previous output.

		// Create a new (empty) reactor configuration file.
		m_reactor_config_file = CONFIG_FILE_DIR + "reactors1.xml";
		if (boost::filesystem::exists(m_reactor_config_file))
			boost::filesystem::remove(m_reactor_config_file);

		// Create and initialize a ReactionEngine.
		m_reaction_engine = new ReactionEngine(m_vocab_mgr, m_codec_factory, m_protocol_factory, m_database_mgr);
		m_reaction_engine->setConfigFile(m_reactor_config_file);
		m_reaction_engine->createConfigFile();
	}
	virtual ~ReactionEngineReadyToAddReactors2_F() {
		delete m_reaction_engine;
	}

	VocabularyManager	m_vocab_mgr;
	CodecFactory		m_codec_factory;
	ProtocolFactory		m_protocol_factory;
	DatabaseManager		m_database_mgr;
	ReactionEngine*		m_reaction_engine;
	std::string			m_reactor_config_file;
};

BOOST_FIXTURE_TEST_SUITE(ReactionEngineReadyToAddReactors2_S, ReactionEngineReadyToAddReactors2_F)

BOOST_AUTO_TEST_CASE(checkFilterCLFLogFile) {
	xmlNodePtr config_ptr = PionPlatformUnitTest::makeReactorConfigFromString(
		"<Plugin>FilterReactor</Plugin>"
		"<Comparison>"
			"<Term>urn:vocab:clickstream#c-ip</Term>"
			"<Type>starts-with</Type>"
			"<Value>66.249</Value>"
		"</Comparison>");
	std::string reactor_id = m_reaction_engine->addReactor(config_ptr);

	// Add a LogOutputReactor and connect it to the FilterReactor so it has something to send output to.
	config_ptr = PionPlatformUnitTest::makeReactorConfigFromString(
		"<Name>CLF Log Writer</Name>"
		"<Plugin>LogOutputReactor</Plugin>"
		"<Codec>3f49f2da-bfe3-11dc-8875-0016cb926e68</Codec>"
		"<Filename>../" + LOG_OUTPUT_FILE + "</Filename>");
	std::string log_writer_id = m_reaction_engine->addReactor(config_ptr);
	m_reaction_engine->addReactorConnection(reactor_id, log_writer_id);

	// Start the ReactionEngine.
	m_reaction_engine->start();

	// Read in events from a CLF log file.
	CodecPtr clf_codec_ptr = m_codec_factory.getCodec("3f49f2da-bfe3-11dc-8875-0016cb926e68");
	boost::uint64_t events_read = PionPlatformUnitTest::feedFileToReactor(
		*m_reaction_engine, reactor_id, *clf_codec_ptr, CLF_LOG_FILE);

	// Check that exactly 7 events passed the filter.
	PionPlatformUnitTest::checkReactorEventsOut(*m_reaction_engine, log_writer_id, static_cast<boost::uint64_t>(7));
}

BOOST_AUTO_TEST_SUITE_END()


class FilterReactorEventValidator_F : public ReactionEngineReadyToAddReactors2_F {
public:
	FilterReactorEventValidator_F() {
		m_num_events_validated = 0;
		m_page_event_ref = m_vocab_mgr.getVocabulary().findTerm("urn:vocab:clickstream#page-event");
		m_page_number_ref = m_vocab_mgr.getVocabulary().findTerm("urn:vocab:clickstream#page-number");
		m_sc_content_term_ref = m_vocab_mgr.getVocabulary().findTerm("urn:vocab:clickstream#sc-content");
	}

	typedef std::set<unsigned> ExpectedEventIds;

	void addEventWithContentTerm(const std::string& content, bool should_pass_filter) {
		EventPtr e(m_event_factory.create(m_page_event_ref));
		e->setUInt(m_page_number_ref, ++m_curr_event_id);
		e->setString(m_sc_content_term_ref, content);
		m_events.push_back(e);
		if (should_pass_filter)
			m_expected_event_ids.insert(m_curr_event_id);
	}

	void MockEventHandler(EventPtr& e, const ExpectedEventIds& expected_event_ids) {
		if (! e.get())
			return;

		// Check that the ID of the Event is in the set of expected event IDs.
		BOOST_CHECK(expected_event_ids.count(e->getUInt(m_page_number_ref)));

		++m_num_events_validated;
	}

	void sendEventsAndValidateOutput(std::list<EventPtr>& events, const std::string& reactor_id) {
		// Add an Event handler to check that the output of the FilterReactor is as expected.
		Reactor::EventHandler eventValidator = boost::bind(&FilterReactorEventValidator_F::MockEventHandler, this, _1, m_expected_event_ids);
		m_reaction_engine->addTempConnectionOut(reactor_id, pion::PionId().to_string(), "blah", eventValidator);

		// Start the ReactionEngine and send the Events to the FilterReactor.
		m_reaction_engine->start();
		for (std::list<EventPtr>::iterator i = events.begin(); i != events.end(); ++i)
			m_reaction_engine->send(reactor_id, *i);

		// Check that the expected number of Events were output by the FilterReactor and validated.
		int num_events_expected = m_expected_event_ids.size();

		// wait up to one second for the number to exceed the expected value
		const int num_checks_allowed = 10 * 1;	// one second
		for (int i = 0; i < num_checks_allowed; ++i) {
			pion::PionScheduler::sleep(0, 100000000); // 0.1 seconds
			if (m_num_events_validated >= num_events_expected) break;
		}
		BOOST_CHECK_EQUAL( m_num_events_validated, num_events_expected );
	}

	EventFactory		m_event_factory;
{'fraction_non_alphanumeric': 0.11962960441206509, 'fraction_numerical': 0.011370599850207667, 'mean_word_length': 5.4, 'pattern_counts': {'":': 0, '<': 78, '<?xml version=': 0, '>': 102, 'https://': 0, 'lorem ipsum': 0, 'www.': 4, 'xml': 4}, 'pii_count': 0, 'substrings_counts': 1, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4212719', 'n_tokens_mistral': 5108, 'n_tokens_neox': 4785, 'n_words': 1048}