<commit_before>#include "main/module.hpp"
#include "main/spine.hpp"

#include "lib/spdlog/spdlog.h"

int main(int argc, char **argv) {
    spdlog::set_async_mode(1048576); //queue size must be power of 2
    spdlog::set_level(spdlog::level::info);

	std::shared_ptr<spdlog::logger> logger = spdlog::stdout_logger_mt("Logger");
	logger->info("Main started, Logger up");

	Spine *spine = new Spine;
	spine->setLogger(logger);
	zmq::context_t spineCtx(1);

	try {
		spine->setSocketContext(&spineCtx);
		spine->openSockets();
		spine->subscribe(spine->name());
	} catch (const zmq::error_t &e) {
		logger->info("Exception: {}", e.what());
 		return 1;
	}

	logger->info("Loading spine modules");
	spine->loadModules("./modules");
	logger->info("{}: {}", "Main", "Running spine");
	bool spineReturn = spine->run();

	delete spine;
	return spineReturn ? 0 : 1;
}<commit_msg>Better logging and comments<commit_after>#include "main/module.hpp"
#include "main/spine.hpp"

#include "lib/spdlog/spdlog.h"

int main(int argc, char **argv) {
	// Configure the global logger
    spdlog::set_async_mode(1048576); //queue size must be power of 2
    spdlog::set_level(spdlog::level::info);

    // Create a stdout logger, multi threaded
	std::shared_ptr<spdlog::logger> logger = spdlog::stdout_logger_mt("Logger");
	logger->debug("{}: {}", "Main", "Logger initialised");

	// Open the spine and pass it out logger
	Spine *spine = new Spine;
	spine->setLogger(logger);

	// Now we create our ZMQ context for all our sockets
	zmq::context_t spineCtx(1);

	try {
		// Lets pass the context through and use it to open all out sockets
		// Then we subscribe the spine to the spine channel
		spine->setSocketContext(&spineCtx);
		logger->debug("{}: {}", "Main", "Spine open, ZMQ context set");
		spine->openSockets();
		spine->subscribe(spine->name());
	} catch (const zmq::error_t &e) {
		logger->info("Exception: {}", e.what());
 		return 1;
	}

	// Load all the modules we wrote
	spine->loadModules("./modules");
	logger->info("{}: {}", "Main", "Running spine");
	bool spineReturn = spine->run();

	delete spine;
	return spineReturn ? 0 : 1;
}
{'fraction_non_alphanumeric': 0.17270588235294118, 'fraction_numerical': 0.011294117647058824, 'mean_word_length': 4.639257294429708, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 27, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3346851', 'n_tokens_mistral': 780, 'n_tokens_neox': 743, 'n_words': 189}