<commit_before>#include <stdexcept>
#include <algorithm>

#include "server/jobstorage.h"
#include "common/config.h"


using namespace Batyr;

JobStorage::JobStorage(std::chrono::duration<int> _maxAgeDoneJobs)
    :   logger(Poco::Logger::get("JobStorage")),
        maxAgeDoneJobs( _maxAgeDoneJobs )
{
    // start thread to cleanup finished jobs
    cleanupExitMutex.lock();
    auto storage = this;
    cleanupThread = std::thread([storage](){
        // run every few seconds
        while (!storage->cleanupExitMutex.try_lock_for( std::chrono::duration<int>( SERVER_JOB_CLEANUP_INTERVAL ) )) {
            // perform the cleaning
            std::lock_guard<std::mutex> lock(storage->mapModificationMutex);

            // calculate the min finishTime a job must have to be spared
            // from the cleaning
            auto minTime = std::chrono::system_clock::now() - storage->maxAgeDoneJobs;

            poco_debug(storage->logger, "Starting to remove deprecated jobs");

            size_t numRemovedJobs = 0;
            for (auto it = storage->jobMap.begin(), ite = storage->jobMap.end(); it != ite;) {
                if (it->second->isDone()) {
                    if (it->second->getTimeFinished() < minTime) {
                        it = storage->jobMap.erase(it);
                        numRemovedJobs++;
                    }
                    else {
                        ++it;
                    }
                }
                else {
                    ++it;
                }
            }

            // decrease verbosity
            if (numRemovedJobs > 0) {
                poco_information(storage->logger, "Removed " + std::to_string(numRemovedJobs) + " deprecated jobs");
            }
        }
        // locking succeded. time to exit
        storage->cleanupExitMutex.unlock();
        poco_debug(storage->logger, "Exiting cleanup thread");
    });

}
<commit_msg>remove debug output which makes debug output unreadable<commit_after>#include <stdexcept>
#include <algorithm>

#include "server/jobstorage.h"
#include "common/config.h"


using namespace Batyr;

JobStorage::JobStorage(std::chrono::duration<int> _maxAgeDoneJobs)
    :   logger(Poco::Logger::get("JobStorage")),
        maxAgeDoneJobs( _maxAgeDoneJobs )
{
    // start thread to cleanup finished jobs
    cleanupExitMutex.lock();
    auto storage = this;
    cleanupThread = std::thread([storage](){
        // run every few seconds
        while (!storage->cleanupExitMutex.try_lock_for( std::chrono::duration<int>( SERVER_JOB_CLEANUP_INTERVAL ) )) {
            // perform the cleaning
            std::lock_guard<std::mutex> lock(storage->mapModificationMutex);

            // calculate the min finishTime a job must have to be spared
            // from the cleaning
            auto minTime = std::chrono::system_clock::now() - storage->maxAgeDoneJobs;

            size_t numRemovedJobs = 0;
            for (auto it = storage->jobMap.begin(), ite = storage->jobMap.end(); it != ite;) {
                if (it->second->isDone()) {
                    if (it->second->getTimeFinished() < minTime) {
                        it = storage->jobMap.erase(it);
                        numRemovedJobs++;
                    }
                    else {
                        ++it;
                    }
                }
                else {
                    ++it;
                }
            }

            // decrease verbosity
            if (numRemovedJobs > 0) {
                poco_information(storage->logger, "Removed " + std::to_string(numRemovedJobs) + " deprecated jobs");
            }
        }
        // locking succeded. time to exit
        storage->cleanupExitMutex.unlock();
        poco_debug(storage->logger, "Exiting cleanup thread");
    });

}

{'fraction_non_alphanumeric': 0.11814345991561181, 'fraction_numerical': 0.0010548523206751054, 'mean_word_length': 1.822172619047619, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 42, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6687108', 'n_tokens_mistral': 1070, 'n_tokens_neox': 1026, 'n_words': 217}