<commit_before>#include <lms/framework.h>
#include <lms/executionmanager.h>
#include <pugixml.hpp>
#include <fstream>
#include <csignal>
#include <map>
#include <thread>
#include <cstdlib>
#include <cstring>
#include <algorithm>
#include "lms/extra/backtrace_formatter.h"
#include "lms/logger.h"
#include "lms/extra/time.h"
#include "lms/module_config.h"
#include "lms/extra/string.h"
#include "lms/xml_parser.h"
#include "lms/extra/colors.h"
#include "lms/definitions.h"
#include "lms/runtime.h"

namespace lms{

std::string Framework::externalDirectory = LMS_EXTERNAL;
std::string Framework::configsDirectory = LMS_CONFIGS;

Framework::Framework(const ArgumentHandler &arguments) :
    logger("lms.Framework"), argumentHandler(arguments),
    configMonitorEnabled(false), m_running(false) {

    logging::Context &ctx = logging::Context::getDefault();

    if(! arguments.argQuiet) {
        ctx.appendSink(new logging::ConsoleSink(std::cout));
    }

    if(! arguments.argLogFile.empty()) {
        ctx.appendSink(new logging::FileSink(arguments.argLogFile));
    }

    SignalHandler::getInstance()
            .addListener(SIGINT, this)
            .addListener(SIGSEGV, this);

    if(! argumentHandler.argProfilingFile.empty()) {
        logger.info() << "Enable profiling";
        m_profiler.enable(arguments.argProfilingFile);
    } else {
        logger.info() << "Disable profiling";
    }

    configMonitorEnabled = argumentHandler.argConfigMonitor;

    if(configMonitorEnabled) {
        logger.info() << "Enable config monitor";
    } else {
        logger.info() << "Disable config monitor";
    }

    logger.info() << "RunLevel " <<  arguments.argRunLevel;

    std::unique_ptr<logging::ThresholdFilter> filter;

    //parse framework config
    if(arguments.argRunLevel >= RunLevel::CONFIG) {
        logger.info() << "MODULES: " << LMS_MODULES;
        logger.info() << "CONFIGS: " << LMS_CONFIGS;

        Runtime* rt = new Runtime("default", *this);
        registerRuntime(rt);

        XmlParser parser(*this, rt, arguments);
        parser.parseConfig(XmlParser::LoadConfigFlag::LOAD_EVERYTHING, arguments.argLoadConfiguration);

        for(const auto& rt : runtimes) {
            logger.info("registerRuntime") << rt.first << " " <<
                lms::executionTypeName(rt.second->executionType());
        }

        filter = parser.filter();

        for(auto error : parser.errors()) {
            logger.error() << error;
        }

        for(auto file : parser.files()) {
            logger.debug("file") << file;
            configMonitor.watch(file);
        }
    }

    if(arguments.argRunLevel >= RunLevel::ENABLE) {
#ifdef __unix__
        m_moduleLoader.addModulePath("/usr/lib/lms");
        m_moduleLoader.addModulePath("/usr/local/lib/lms");
#endif
        m_moduleLoader.addModulePath(LMS_MODULES, 1);

        // enable modules after they were made available
        logger.info() << "Start enabling modules";

        for(auto& runtime : runtimes) {
            runtime.second->enableModules();
        }
    }

    if(arguments.argRunLevel >= RunLevel::CYCLE) {
        logger.info() << "Start running modules";

        if(! filter) { // check if filter == nullptr
            filter.reset(new logging::ThresholdFilter(arguments.argLoggingThreshold));
        }
        if(arguments.argDefinedLoggingThreshold) {
            filter->defaultThreshold(arguments.argLoggingThreshold);
        }
        ctx.filter(filter.release());
<commit_msg>Load services during ENABLE<commit_after>#include <lms/framework.h>
#include <lms/executionmanager.h>
#include <pugixml.hpp>
#include <fstream>
#include <csignal>
#include <map>
#include <thread>
#include <cstdlib>
#include <cstring>
#include <algorithm>
#include "lms/extra/backtrace_formatter.h"
#include "lms/logger.h"
#include "lms/extra/time.h"
#include "lms/module_config.h"
#include "lms/extra/string.h"
#include "lms/xml_parser.h"
#include "lms/extra/colors.h"
#include "lms/definitions.h"
#include "lms/runtime.h"

namespace lms{

std::string Framework::externalDirectory = LMS_EXTERNAL;
std::string Framework::configsDirectory = LMS_CONFIGS;

Framework::Framework(const ArgumentHandler &arguments) :
    logger("lms.Framework"), argumentHandler(arguments),
    configMonitorEnabled(false), m_running(false) {

    logging::Context &ctx = logging::Context::getDefault();

    if(! arguments.argQuiet) {
        ctx.appendSink(new logging::ConsoleSink(std::cout));
    }

    if(! arguments.argLogFile.empty()) {
        ctx.appendSink(new logging::FileSink(arguments.argLogFile));
    }

    SignalHandler::getInstance()
            .addListener(SIGINT, this)
            .addListener(SIGSEGV, this);

    if(! argumentHandler.argProfilingFile.empty()) {
        logger.info() << "Enable profiling";
        m_profiler.enable(arguments.argProfilingFile);
    } else {
        logger.info() << "Disable profiling";
    }

    configMonitorEnabled = argumentHandler.argConfigMonitor;

    if(configMonitorEnabled) {
        logger.info() << "Enable config monitor";
    } else {
        logger.info() << "Disable config monitor";
    }

    logger.info() << "RunLevel " <<  arguments.argRunLevel;

    std::unique_ptr<logging::ThresholdFilter> filter;

    //parse framework config
    if(arguments.argRunLevel >= RunLevel::CONFIG) {
        logger.info() << "MODULES: " << LMS_MODULES;
        logger.info() << "CONFIGS: " << LMS_CONFIGS;

        Runtime* rt = new Runtime("default", *this);
        registerRuntime(rt);

        XmlParser parser(*this, rt, arguments);
        parser.parseConfig(XmlParser::LoadConfigFlag::LOAD_EVERYTHING, arguments.argLoadConfiguration);

        for(const auto& rt : runtimes) {
            logger.info("registerRuntime") << rt.first << " " <<
                lms::executionTypeName(rt.second->executionType());
        }

        filter = parser.filter();

        for(auto error : parser.errors()) {
            logger.error() << error;
        }

        for(auto file : parser.files()) {
            logger.debug("file") << file;
            configMonitor.watch(file);
        }
    }

    if(arguments.argRunLevel >= RunLevel::ENABLE) {
#ifdef __unix__
        m_moduleLoader.addModulePath("/usr/lib/lms");
        m_moduleLoader.addModulePath("/usr/local/lib/lms");
        // TODO global service path
#endif
        m_moduleLoader.addModulePath(LMS_MODULES, 0);
        m_serviceLoader.addModulePath(LMS_SERVICES, 0);

        // enable modules after they were made available
        logger.info() << "Start enabling modules";

        for(auto& runtime : runtimes) {
            runtime.second->enableModules();
        }
    }

    if(arguments.argRunLevel >= RunLevel::CYCLE) {
        logger.info() << "Start running modules";

        if(! filter) { // check if filter == nullptr
            filter.reset(new logging::ThresholdFilter(arguments.argLoggingThreshold));
        }
        if(arguments.argDefinedLoggingThreshold) {
            filter->defaultThreshold(arguments.argLoggingThreshold);
        }
        ctx.filter(filter.release());

{'fraction_non_alphanumeric': 0.13985915492957746, 'fraction_numerical': 0.00042253521126760566, 'mean_word_length': 3.1284883720930234, 'pattern_counts': {'":': 0, '<': 93, '<?xml version=': 0, '>': 37, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 4}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6236980', 'n_tokens_mistral': 2145, 'n_tokens_neox': 2091, 'n_words': 369}