<commit_before>#include "Precompiled.hpp"
#include "CrestConnectionPool.hpp"
#include "CrestCache.hpp"
#include "CppServers.hpp"
#include "http/CrestRequest.hpp"
#include "http/core/HttpParser.hpp"
#include "Gzip.hpp"
#include <iostream>

CrestConnectionPool::CrestConnectionPool(CrestCache *cache)
    : cache(cache)
    , exiting(false), mutex(), request_queued(), request_queue(), connections()
    , request_allowance(0)
    , request_allowance_time(0)
{
    for (auto i = 0; i < PCREST_MAX_CONNECTIONS; ++i)
    {
        connections.emplace_back(this);
    }
}

CrestConnectionPool::~CrestConnectionPool()
{
    exit();
}

void CrestConnectionPool::exit()
{
    {
        std::unique_lock<std::mutex> lock(mutex);
        exiting = true;
        request_queued.notify_all();
    }
    for (auto &con : connections)
    {
        con.thread.join();
    }
    connections.clear();
}

void CrestConnectionPool::queue(CrestHttpRequest *request)
{
    assert(!exiting);
    {
        std::unique_lock<std::mutex> lock(mutex);
        request_queue.push(request);
        request_queued.notify_one();
    }
}

void CrestConnectionPool::wait_for_request_allowance()
{
    if (--request_allowance >= 0) return;

    std::unique_lock<std::mutex> lock(mutex);
    //only one thread actively waits to reset the count, the others will block on the mutex and
    //then continue using this second atomic int decrement
    if (--request_allowance >= 0) return;


    using std::chrono::steady_clock;
    using std::chrono::seconds;
    auto now = steady_clock::now();
    auto nows = std::chrono::duration_cast<seconds>(now.time_since_epoch());

    log_info() << "CREST request allowance used. Renewing. "
        << " allowance_time: " << request_allowance_time.count()
        << " now: " << nows.count()
        << std::endl;

    
    if (request_allowance_time < nows)
    {
        request_allowance_time = nows;
    }
    else
    {
        request_allowance_time = nows + seconds(1);
        std::this_thread::sleep_until(steady_clock::time_point(request_allowance_time));

        nows = std::chrono::duration_cast<seconds>(steady_clock::now().time_since_epoch());
        log_info() << "CREST request allowance renewed at " << nows.count() << std::endl;
    }
    request_allowance = PCREST_MAX_REQS_PER_SEC - 1;
}

<commit_msg>Reduced logging for CREST request limit<commit_after>#include "Precompiled.hpp"
#include "CrestConnectionPool.hpp"
#include "CrestCache.hpp"
#include "CppServers.hpp"
#include "http/CrestRequest.hpp"
#include "http/core/HttpParser.hpp"
#include "Gzip.hpp"
#include <iostream>

CrestConnectionPool::CrestConnectionPool(CrestCache *cache)
    : cache(cache)
    , exiting(false), mutex(), request_queued(), request_queue(), connections()
    , request_allowance(0)
    , request_allowance_time(0)
{
    for (auto i = 0; i < PCREST_MAX_CONNECTIONS; ++i)
    {
        connections.emplace_back(this);
    }
}

CrestConnectionPool::~CrestConnectionPool()
{
    exit();
}

void CrestConnectionPool::exit()
{
    {
        std::unique_lock<std::mutex> lock(mutex);
        exiting = true;
        request_queued.notify_all();
    }
    for (auto &con : connections)
    {
        con.thread.join();
    }
    connections.clear();
}

void CrestConnectionPool::queue(CrestHttpRequest *request)
{
    assert(!exiting);
    {
        std::unique_lock<std::mutex> lock(mutex);
        request_queue.push(request);
        request_queued.notify_one();
    }
}

void CrestConnectionPool::wait_for_request_allowance()
{
    //atomic decrement, no need to lock if not managing the reset
    if (--request_allowance >= 0) return;
    //only one thread actively waits to reset the count, the others will block on the mutex and
    //then continue using this second int decrement
    std::unique_lock<std::mutex> lock(mutex);
    if (--request_allowance >= 0) return;

    //Make the thread sleep until the second after the previous allowance timestamp
    using std::chrono::steady_clock;
    using std::chrono::seconds;
    auto now = steady_clock::now();
    auto nows = std::chrono::duration_cast<seconds>(now.time_since_epoch());
    
    if (request_allowance_time < nows)
    {   // More than a second passed since the last allowance anyway
        request_allowance_time = nows;
    }
    else
    {
        request_allowance_time = nows + seconds(1);
        std::this_thread::sleep_until(steady_clock::time_point(request_allowance_time));

        nows = std::chrono::duration_cast<seconds>(steady_clock::now().time_since_epoch());
    }
    request_allowance = PCREST_MAX_REQS_PER_SEC - 1;
    log_info() << "CREST request allowance renewed at " << nows.count() << std::endl;
}


{'fraction_non_alphanumeric': 0.12930667800935772, 'fraction_numerical': 0.0029774564015312634, 'mean_word_length': 3.661050545094153, 'pattern_counts': {'":': 0, '<': 43, '<?xml version=': 0, '>': 19, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1781970', 'n_tokens_mistral': 1539, 'n_tokens_neox': 1507, 'n_words': 302}