<commit_before>//
// bucket.hpp
// **********
//
// Copyright (c) 2018 <NAME> (sharon at aegis dot gg)
//
// Distributed under the MIT License. (See accompanying file LICENSE)
//

#pragma once

#include "aegis/config.hpp"
#include <mutex>
#include <future>
#include <chrono>
#include <functional>
#include <string>
#include <queue>
#include <atomic>
#include "aegis/rest/rest_reply.hpp"
#include "aegis/snowflake.hpp"
#include <asio/io_context.hpp>
#include <asio/use_future.hpp>
#include <asio/post.hpp>
#if defined(AEGIS_HAS_STD_OPTIONAL)
#include <optional>
#else
#include "aegis/optional.hpp"
namespace std
{
using std::experimental::optional;
}
#endif

namespace aegis
{

using rest_call = std::function<rest::rest_reply(const std::string & path, const std::string & content, const std::string & method, const std::string & host)>;

namespace ratelimit
{

using namespace std::chrono;

/**
* Major parameter of REST API access
* Emoji is a partial major parameter and is ratelimited per guild across all members
*/
enum bucket_type
{
    Guild = 0,
    Channel = 1,
    Emoji = 2
};

/// Buckets store ratelimit data per major parameter
/**
 * Bucket class for tracking the ratelimits per snowflake per major parameter.
 * Each bucket tracks a single major parameter and a single snowflake
 * Current major parameters are GUILD, CHANNEL, and EMOJI
 */
template<typename Callable, typename Result>
class bucket
{
public:
    /**
     * Construct a bucket object for tracking ratelimits per major parameter of the REST API (guild/channel/emoji)
     */
    bucket(Callable & call, asio::io_context & _io_context, std::atomic<int64_t> & global_limit)
        : limit(0)
        , remaining(1)
        , reset(0)
        , _call(call)
        , _io_context(_io_context)
        , _global_limit(global_limit)
    {

    }

    std::atomic<int64_t> limit; /**< Rate limit current endpoint call limit */
    std::atomic<int64_t> remaining; /**< Rate limit remaining count */
    std::atomic<int64_t> reset; /**< Rate limit reset time */

    /// Check if globally ratelimited
    /**
     * @returns true if globally ratelimited
     */
    AEGIS_DECL bool is_global() const AEGIS_NOEXCEPT
    {
        return _global_limit > 0;
    }


    /// Check if bucket can send a message without hitting the ratelimit
    /**
     * @returns true if bucket ratelimits permit a message to be sent
     */
    AEGIS_DECL bool can_perform() const AEGIS_NOEXCEPT
    {
        if (ignore_rates)
            return true;
        if (limit.load(std::memory_order_relaxed) == 0)
            return true;
        if (remaining.load(std::memory_order_relaxed) > 0)
            return true;
        int64_t time = duration_cast<seconds>(system_clock::now().time_since_epoch()).count();
        if (time < reset.load(std::memory_order_relaxed))
            return false;
        return true;
    }

    AEGIS_DECL Result perform(const std::string & path, const std::string & content, const std::string & method, const std::string & host = "")
    {
        std::lock_guard<std::mutex> lock(m);
        while (!can_perform())
        {
            //TODO: find a better solution - wrap asio execution handling, poll ratelimit object to track ordering and execution
            // not an ideal scenario, but by current design rescheduling a message that would be ratelimited
            // would cause out of order messages
            std::this_thread::sleep_for(seconds((reset.load(std::memory_order_relaxed)
                                                 - std::chrono::duration_cast<seconds>(std::chrono::system_clock::now().time_since_epoch()).count()) + 1));
        }
        //std::queue<std::tuple<std::string, std::string, std::string, std::function<void(rest_response)>>> query;
        Result reply(_call(path, content, method, host));
        limit.store(reply.limit, std::memory_order_relaxed);
        remaining.store(reply.remaining, std::memory_order_relaxed);
        reset.store(reply.reset, std::memory_order_relaxed);
        return reply;
    }

    AEGIS_DECL std::future<Result> post_task(const std::string & path, const std::string & method = "POST", const std::string & obj = "", const std::string & host = "")
    {
        using result = asio::async_result<asio::use_future_t<>, void(Result)>;
        using handler = typename result::completion_handler_type;

        handler exec(std::forward<decltype(asio::use_future)>(asio::use_future));
        result ret(exec);

        asio::post(_io_context, [=]() mutable
        {
            exec(perform(path, obj, method, host));
        });
        return ret.get();
    }

    bool ignore_rates = false;
    std::mutex m;
    Callable & _call;
    std::queue<std::tuple<std::string, std::string, std::string, std::function<void(Result)>>> _queue;

private:
    asio::io_context & _io_context;
    std::atomic<int64_t> & _global_limit;
};

}

}
<commit_msg>Added missing bucket_type hash specialization<commit_after>//
// bucket.hpp
// **********
//
// Copyright (c) 2018 <NAME> (sharon at aegis dot gg)
//
// Distributed under the MIT License. (See accompanying file LICENSE)
//

#pragma once

#include "aegis/config.hpp"
#include <mutex>
#include <future>
#include <chrono>
#include <functional>
#include <string>
#include <queue>
#include <atomic>
#include "aegis/rest/rest_reply.hpp"
#include "aegis/snowflake.hpp"
#include <asio/io_context.hpp>
#include <asio/use_future.hpp>
#include <asio/post.hpp>
#if defined(AEGIS_HAS_STD_OPTIONAL)
#include <optional>
#else
#include "aegis/optional.hpp"
namespace std
{
using std::experimental::optional;
}
#endif

namespace aegis
{

using rest_call = std::function<rest::rest_reply(const std::string & path, const std::string & content, const std::string & method, const std::string & host)>;

namespace ratelimit
{

using namespace std::chrono;

/**
* Major parameter of REST API access
* Emoji is a partial major parameter and is ratelimited per guild across all members
*/
enum bucket_type
{
    Guild = 0,
    Channel = 1,
    Emoji = 2
};

/// Buckets store ratelimit data per major parameter
/**
 * Bucket class for tracking the ratelimits per snowflake per major parameter.
 * Each bucket tracks a single major parameter and a single snowflake
 * Current major parameters are GUILD, CHANNEL, and EMOJI
 */
template<typename Callable, typename Result>
class bucket
{
public:
    /**
     * Construct a bucket object for tracking ratelimits per major parameter of the REST API (guild/channel/emoji)
     */
    bucket(Callable & call, asio::io_context & _io_context, std::atomic<int64_t> & global_limit)
        : limit(0)
        , remaining(1)
        , reset(0)
        , _call(call)
        , _io_context(_io_context)
        , _global_limit(global_limit)
    {

    }

    std::atomic<int64_t> limit; /**< Rate limit current endpoint call limit */
    std::atomic<int64_t> remaining; /**< Rate limit remaining count */
    std::atomic<int64_t> reset; /**< Rate limit reset time */

    /// Check if globally ratelimited
    /**
     * @returns true if globally ratelimited
     */
    AEGIS_DECL bool is_global() const AEGIS_NOEXCEPT
    {
        return _global_limit > 0;
    }


    /// Check if bucket can send a message without hitting the ratelimit
    /**
     * @returns true if bucket ratelimits permit a message to be sent
     */
    AEGIS_DECL bool can_perform() const AEGIS_NOEXCEPT
    {
        if (ignore_rates)
            return true;
        if (limit.load(std::memory_order_relaxed) == 0)
            return true;
        if (remaining.load(std::memory_order_relaxed) > 0)
            return true;
        int64_t time = duration_cast<seconds>(system_clock::now().time_since_epoch()).count();
        if (time < reset.load(std::memory_order_relaxed))
            return false;
        return true;
    }

    AEGIS_DECL Result perform(const std::string & path, const std::string & content, const std::string & method, const std::string & host = "")
    {
        std::lock_guard<std::mutex> lock(m);
        while (!can_perform())
        {
            //TODO: find a better solution - wrap asio execution handling, poll ratelimit object to track ordering and execution
            // not an ideal scenario, but by current design rescheduling a message that would be ratelimited
            // would cause out of order messages
            std::this_thread::sleep_for(seconds((reset.load(std::memory_order_relaxed)
                                                 - std::chrono::duration_cast<seconds>(std::chrono::system_clock::now().time_since_epoch()).count()) + 1));
        }
        //std::queue<std::tuple<std::string, std::string, std::string, std::function<void(rest_response)>>> query;
        Result reply(_call(path, content, method, host));
        limit.store(reply.limit, std::memory_order_relaxed);
        remaining.store(reply.remaining, std::memory_order_relaxed);
        reset.store(reply.reset, std::memory_order_relaxed);
        return reply;
    }

    AEGIS_DECL std::future<Result> post_task(const std::string & path, const std::string & method = "POST", const std::string & obj = "", const std::string & host = "")
    {
        using result = asio::async_result<asio::use_future_t<>, void(Result)>;
        using handler = typename result::completion_handler_type;

        handler exec(std::forward<decltype(asio::use_future)>(asio::use_future));
        result ret(exec);

        asio::post(_io_context, [=]() mutable
        {
            exec(perform(path, obj, method, host));
        });
        return ret.get();
    }

    bool ignore_rates = false;
    std::mutex m;
    Callable & _call;
    std::queue<std::tuple<std::string, std::string, std::string, std::function<void(Result)>>> _queue;

private:
    asio::io_context & _io_context;
    std::atomic<int64_t> & _global_limit;
};

}

}

namespace std
{

template <>
struct hash<aegis::ratelimit::bucket_type>
{
    std::size_t operator()(const aegis::ratelimit::bucket_type& k) const
    {
        return hash<int64_t>()(k);
    }
};

}

{'fraction_non_alphanumeric': 0.1255342411291124, 'fraction_numerical': 0.0053672597157340225, 'mean_word_length': 3.1837837837837837, 'pattern_counts': {'":': 0, '<': 78, '<?xml version=': 0, '>': 74, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1537901', 'n_tokens_mistral': 3186, 'n_tokens_neox': 3036, 'n_words': 810}