<commit_before>#include <iostream>
#include <iomanip>
#include <string>
#include <fstream>
#include <vector>
#include <array>
#include <cmath>
#include <numeric>
#include <algorithm>
#include <cassert>
#include <chrono>
#include <thread>
//#include <immintrin.h>
#include <openbabel/obconversion.h>
#include <openbabel/mol.h>
#include <boost/tokenizer.hpp>
#include <boost/filesystem/operations.hpp>
#include <boost/filesystem/fstream.hpp>
#include <boost/iostreams/device/file.hpp>
#include <boost/iostreams/filtering_stream.hpp>
#include <boost/iostreams/filter/gzip.hpp>
#include <boost/date_time/posix_time/posix_time.hpp>
#include <mongo/client/dbclient.h>
#include <Poco/Net/MailMessage.h>
#include <Poco/Net/MailRecipient.h>
#include <Poco/Net/SMTPClientSession.h>
using namespace std;
using namespace std::chrono;
using namespace OpenBabel;
using namespace boost::filesystem;
using namespace boost::iostreams;
using namespace boost::gregorian;
using namespace boost::posix_time;
using namespace mongo;
using namespace bson;
using namespace Poco::Net;

inline static string now()
{
	return to_simple_string(second_clock::local_time()) + " ";
}

int main(int argc, char* argv[])
{
	// Check the required number of command line arguments.
	if (argc != 5)
	{
		cout << "usr host user pwd jobs_path" << endl;
		return 0;
	}

	// Fetch command line arguments.
	const auto host = argv[1];
	const auto user = argv[2];
	const auto pwd = argv[3];
	const path jobs_path = argv[4];

	DBClientConnection conn;
	{
		// Connect to host and authenticate user.
		cout << now() << "Connecting to " << host << " and authenticating " << user << endl;
		string errmsg;
		if ((!conn.connect(host, errmsg)) || (!conn.auth("istar", user, pwd, errmsg)))
		{
			cerr << now() << errmsg << endl;
			return 1;
		}
	}

	// Initialize constants.
	cout << now() << "Initializing" << endl;
	const auto collection = "istar.usr";
//	const auto m256s = _mm256_set1_pd(-0. ); // -0.  = 1 << 63
	const auto epoch = date(1970, 1, 1);
	const size_t num_usrs = 2;
	constexpr array<size_t, num_usrs> qn = { 12, 60 };
	constexpr array<double, num_usrs> qv = { 1.0 / qn[0], 1.0 / qn[1] };
	const size_t num_references = 4;
	const size_t num_subsets = 5;
	const array<string, num_subsets> SmartsPatterns =
	{
		"[!#1]", // heavy
		"[#6+0!$(*~[#7,#8,F]),SH0+0v2,s+0,S^3,Cl+0,Br+0,I+0]", // hydrophobic
		"[a]", // aromatic
		"[$([O,S;H1;v2]-[!$(*=[O,N,P,S])]),$([O,S;H0;v2]),$([O,S;-]),$([N&v3;H1,H2]-[!$(*=[O,N,P,S])]),$([N;v3;H0]),$([n,o,s;+0]),F]", // acceptor
		"[N!H0v3,N!H0+v4,OH+0,SH+0,nH+0]", // donor
	};

	// Read the header bin file.
	vector<size_t> headers;
	{
		std::ifstream hdr_bin("16_hdr.bin", ios::binary | ios::ate);
		const size_t num_bytes = hdr_bin.tellg();
		headers.resize(num_bytes / sizeof(size_t));
		hdr_bin.seekg(0);
		hdr_bin.read(reinterpret_cast<char*>(headers.data()), num_bytes);
	}
	const size_t num_ligands = headers.size();

	// Search the features for records similar to the query.
//	vector<array<double, qn.back()>> features;
	array<array<double, qn.back()>, 2> qlw;
	array<array<double, 4>, 1> aw;
	auto l = qlw[1];
	auto a = aw.front();
	array<vector<double>, 2> scores{ vector<double>(num_ligands), vector<double>(num_ligands) };
	vector<size_t> scase(num_ligands);
	string line;
	std::ifstream usrcat_bin("16_usrcat.bin", ios::binary);
	std::ifstream lig_pdbqt("16_lig.pdbqt");
	cout << now() << "Entering event loop" << endl;
	while (true)
	{
		// Fetch jobs.
		auto cursor = conn.query(collection, QUERY("done" << BSON("$exists" << false)).sort("submitted"), 1); // Each batch processes 1 job.
		while (cursor->more())
		{
			// Obtain job properties.
			const auto job = cursor->next();
			const auto _id = job["_id"].OID();
			cout << now() << "Executing job " << _id.str() << endl;
			const auto job_path = jobs_path / _id.str();
			const auto format = job["format"].String();
			const auto email = job["email"].String();

			// Record job starting time stamp.
			conn.update(collection, BSON("_id" << _id), BSON("$set" << BSON("started" << Date_t(duration_cast<std::chrono::milliseconds>(system_clock::now().time_since_epoch()).count()))));

			// Parse the user-supplied ligand.
			OBMol obMol;
			OBConversion obConversion;
			obConversion.SetInFormat(format.c_str());
			obConversion.ReadFile(&obMol, (job_path / ("ligand." + format)).string());
			const auto num_atoms = obMol.NumAtoms();
//			obMol.AddHydrogens(); // Adding hydrogens does not seem to affect SMARTS matching.
			array<vector<int>, num_subsets> subsets;
			for (size_t k = 0; k < num_subsets; ++k)
			{
				auto& subset = subsets[k];
				subset.reserve(num_atoms);
				OBSmartsPattern smarts;
				smarts.Init(SmartsPatterns[k]);
				smarts.Match(obMol);
				for (const auto& map : smarts.GetMapList())
				{
					subset.push_back(map.front());
				}
			}
			const auto& subset0 = subsets.front();
			if (subset0.empty())
			{
				// Record job completion time stamp.
				const auto millis_since_epoch = duration_cast<std::chrono::milliseconds>(system_clock::now().time_since_epoch()).count();
				conn.update(collection, BSON("_id" << _id), BSON("$set" << BSON("done" << Date_t(millis_since_epoch))));

				// Send error notification email.
				cout << now() << "Sending an error notification email to " << email << endl;
				MailMessage message;
				message.setSender("istar <<EMAIL>>");
				message.setSubject("Your usr job has failed to complete");
				message.setContent("Your usr job submitted on " + to_simple_string(ptime(epoch, boost::posix_time::milliseconds(job["submitted"].Date().millis))) + " UTC with description \"" + job["description"].String() + "\" failed on " + to_simple_string(ptime(epoch, boost::posix_time::milliseconds(millis_since_epoch))) + " UTC because your provided ligand failed to be parsed.");
				message.addRecipient(MailRecipient(MailRecipient::PRIMARY_RECIPIENT, email));
				SMTPClientSession session("172.16.58.3");
				session.login();
				session.sendMessage(message);<commit_msg>Renamed SmartsPatterns to SubsetSMARTS<commit_after>#include <iostream>
#include <iomanip>
#include <string>
#include <fstream>
#include <vector>
#include <array>
#include <cmath>
#include <numeric>
#include <algorithm>
#include <cassert>
#include <chrono>
#include <thread>
//#include <immintrin.h>
#include <openbabel/obconversion.h>
#include <openbabel/mol.h>
#include <boost/tokenizer.hpp>
#include <boost/filesystem/operations.hpp>
#include <boost/filesystem/fstream.hpp>
#include <boost/iostreams/device/file.hpp>
#include <boost/iostreams/filtering_stream.hpp>
#include <boost/iostreams/filter/gzip.hpp>
#include <boost/date_time/posix_time/posix_time.hpp>
#include <mongo/client/dbclient.h>
#include <Poco/Net/MailMessage.h>
#include <Poco/Net/MailRecipient.h>
#include <Poco/Net/SMTPClientSession.h>
using namespace std;
using namespace std::chrono;
using namespace OpenBabel;
using namespace boost::filesystem;
using namespace boost::iostreams;
using namespace boost::gregorian;
using namespace boost::posix_time;
using namespace mongo;
using namespace bson;
using namespace Poco::Net;

inline static string now()
{
	return to_simple_string(second_clock::local_time()) + " ";
}

int main(int argc, char* argv[])
{
	// Check the required number of command line arguments.
	if (argc != 5)
	{
		cout << "usr host user pwd jobs_path" << endl;
		return 0;
	}

	// Fetch command line arguments.
	const auto host = argv[1];
	const auto user = argv[2];
	const auto pwd = argv[3];
	const path jobs_path = argv[4];

	DBClientConnection conn;
	{
		// Connect to host and authenticate user.
		cout << now() << "Connecting to " << host << " and authenticating " << user << endl;
		string errmsg;
		if ((!conn.connect(host, errmsg)) || (!conn.auth("istar", user, pwd, errmsg)))
		{
			cerr << now() << errmsg << endl;
			return 1;
		}
	}

	// Initialize constants.
	cout << now() << "Initializing" << endl;
	const auto collection = "istar.usr";
//	const auto m256s = _mm256_set1_pd(-0. ); // -0.  = 1 << 63
	const auto epoch = date(1970, 1, 1);
	const size_t num_usrs = 2;
	constexpr array<size_t, num_usrs> qn = { 12, 60 };
	constexpr array<double, num_usrs> qv = { 1.0 / qn[0], 1.0 / qn[1] };
	const size_t num_references = 4;
	const size_t num_subsets = 5;
	const array<string, num_subsets> SubsetSMARTS =
	{
		"[!#1]", // heavy
		"[#6+0!$(*~[#7,#8,F]),SH0+0v2,s+0,S^3,Cl+0,Br+0,I+0]", // hydrophobic
		"[a]", // aromatic
		"[$([O,S;H1;v2]-[!$(*=[O,N,P,S])]),$([O,S;H0;v2]),$([O,S;-]),$([N&v3;H1,H2]-[!$(*=[O,N,P,S])]),$([N;v3;H0]),$([n,o,s;+0]),F]", // acceptor
		"[N!H0v3,N!H0+v4,OH+0,SH+0,nH+0]", // donor
	};

	// Read the header bin file.
	vector<size_t> headers;
	{
		std::ifstream hdr_bin("16_hdr.bin", ios::binary | ios::ate);
		const size_t num_bytes = hdr_bin.tellg();
		headers.resize(num_bytes / sizeof(size_t));
		hdr_bin.seekg(0);
		hdr_bin.read(reinterpret_cast<char*>(headers.data()), num_bytes);
	}
	const size_t num_ligands = headers.size();

	// Search the features for records similar to the query.
//	vector<array<double, qn.back()>> features;
	array<array<double, qn.back()>, 2> qlw;
	array<array<double, 4>, 1> aw;
	auto l = qlw[1];
	auto a = aw.front();
	array<vector<double>, 2> scores{ vector<double>(num_ligands), vector<double>(num_ligands) };
	vector<size_t> scase(num_ligands);
	string line;
	std::ifstream usrcat_bin("16_usrcat.bin", ios::binary);
	std::ifstream lig_pdbqt("16_lig.pdbqt");
	cout << now() << "Entering event loop" << endl;
	while (true)
	{
		// Fetch jobs.
		auto cursor = conn.query(collection, QUERY("done" << BSON("$exists" << false)).sort("submitted"), 1); // Each batch processes 1 job.
		while (cursor->more())
		{
			// Obtain job properties.
			const auto job = cursor->next();
			const auto _id = job["_id"].OID();
			cout << now() << "Executing job " << _id.str() << endl;
			const auto job_path = jobs_path / _id.str();
			const auto format = job["format"].String();
			const auto email = job["email"].String();

			// Record job starting time stamp.
			conn.update(collection, BSON("_id" << _id), BSON("$set" << BSON("started" << Date_t(duration_cast<std::chrono::milliseconds>(system_clock::now().time_since_epoch()).count()))));

			// Parse the user-supplied ligand.
			OBMol obMol;
			OBConversion obConversion;
			obConversion.SetInFormat(format.c_str());
			obConversion.ReadFile(&obMol, (job_path / ("ligand." + format)).string());
			const auto num_atoms = obMol.NumAtoms();
//			obMol.AddHydrogens(); // Adding hydrogens does not seem to affect SMARTS matching.
			array<vector<int>, num_subsets> subsets;
			for (size_t k = 0; k < num_subsets; ++k)
			{
				auto& subset = subsets[k];
				subset.reserve(num_atoms);
				OBSmartsPattern smarts;
				smarts.Init(SubsetSMARTS[k]);
				smarts.Match(obMol);
				for (const auto& map : smarts.GetMapList())
				{
					subset.push_back(map.front());
				}
			}
			const auto& subset0 = subsets.front();
			if (subset0.empty())
			{
				// Record job completion time stamp.
				const auto millis_since_epoch = duration_cast<std::chrono::milliseconds>(system_clock::now().time_since_epoch()).count();
				conn.update(collection, BSON("_id" << _id), BSON("$set" << BSON("done" << Date_t(millis_since_epoch))));

				// Send error notification email.
				cout << now() << "Sending an error notification email to " << email << endl;
				MailMessage message;
				message.setSender("istar <<EMAIL>>");
				message.setSubject("Your usr job has failed to complete");
				message.setContent("Your usr job submitted on " + to_simple_string(ptime(epoch, boost::posix_time::milliseconds(job["submitted"].Date().millis))) + " UTC with description \"" + job["description"].String() + "\" failed on " + to_simple_string(ptime(epoch, boost::posix_time::milliseconds(millis_since_epoch))) + " UTC because your provided ligand failed to be parsed.");
				message.addRecipient(MailRecipient(MailRecipient::PRIMARY_RECIPIENT, email));
				SMTPClientSession session("172.16.58.3");
				session.login();
				session.sendMessage(message);
{'fraction_non_alphanumeric': 0.17740460553661982, 'fraction_numerical': 0.015462631972732562, 'mean_word_length': 4.642589118198874, 'pattern_counts': {'":': 0, '<': 237, '<?xml version=': 0, '>': 103, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 2, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1732988', 'n_tokens_mistral': 4793, 'n_tokens_neox': 4357, 'n_words': 970}