<commit_before>#pragma once

#include <emscripten/emscripten.h>

#include <vector>
#include <future>
#include <cstdint>

namespace wonder_rabbit_project
{
  namespace wonderland
  {
    namespace loader
    {
      using buffer_t = std::vector<std::uint8_t>;
      using future_t = std::future<buffer_t>;

      namespace detail
      {
        using promise_t = std::promise<buffer_t>;
      }

      template < class T = void >
      auto load(const std::string& url)
        -> future_t
      {
        auto promise_ptr = new detail::promise_t();
        auto future = promise_ptr->get_future();

        constexpr auto request_type = "GET";
        constexpr auto request_parameter = "";
        auto argument = promise_ptr;
        constexpr auto free_after_on_load = true;

        auto on_load     = [](unsigned handle, void* user_data, void* data, unsigned size)
        {
          LOG << "load";
          auto promise_ptr = reinterpret_cast< detail::promise_t* >( user_data );
          auto typed_data = reinterpret_cast< std::uint8_t* >( data );
          buffer_t buffer( typed_data, typed_data + size);
          promise_ptr->set_value(std::move(buffer));
          delete promise_ptr;
        };
        auto on_error    = [](unsigned, void*, int, const char*)
        {
          LOG << "error";
          delete promise_ptr;
        };
        auto on_progress = [](unsigned, void*, int, int)
        {
          //LOG << "progress";
        };<commit_msg>remove debug code<commit_after>#pragma once

#include <emscripten/emscripten.h>

#include <vector>
#include <future>
#include <cstdint>

namespace wonder_rabbit_project
{
  namespace wonderland
  {
    namespace loader
    {
      using buffer_t = std::vector<std::uint8_t>;
      using future_t = std::future<buffer_t>;

      namespace detail
      {
        using promise_t = std::promise<buffer_t>;
      }

      template < class T = void >
      auto load(const std::string& url)
        -> future_t
      {
        auto promise_ptr = new detail::promise_t();
        auto future = promise_ptr->get_future();

        constexpr auto request_type = "GET";
        constexpr auto request_parameter = "";
        auto argument = promise_ptr;
        constexpr auto free_after_on_load = true;

        auto on_load     = [](unsigned handle, void* user_data, void* data, unsigned size)
        {
          auto promise_ptr = reinterpret_cast< detail::promise_t* >( user_data );
          auto typed_data = reinterpret_cast< std::uint8_t* >( data );
          buffer_t buffer( typed_data, typed_data + size);
          promise_ptr->set_value(std::move(buffer));
          delete promise_ptr;
        };
        
        auto on_error    = [](unsigned, void*, int, const char*)
        {
          delete promise_ptr;
        };
        
        auto on_progress = [](unsigned, void*, int, int)
        {
        };
{'fraction_non_alphanumeric': 0.1097645429362881, 'fraction_numerical': 0.0013850415512465374, 'mean_word_length': 2.2755102040816326, 'pattern_counts': {'":': 0, '<': 29, '<?xml version=': 0, '>': 29, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1373917', 'n_tokens_mistral': 859, 'n_tokens_neox': 850, 'n_words': 208}