<commit_before>#include "Precompiled.hpp"
#include "Errors.hpp"
#include "BulkMarketHistory.hpp"
#include "CrestBulkMarketOrders.hpp"
#include "CrestCache.hpp"
#include "lib/Params.hpp"
#include "http/HttpStatusErrors.hpp"
#include <iostream>
#include <chrono>
#include "json/JsonReader.hpp"
#include <rapidjson/stringbuffer.h>
#include <rapidjson/writer.h>
http::HttpResponse http_get_bulk_market_history(CrestCache &cache, http::HttpRequest &request)
{
    // Get params
    std::vector<int> regions = params_regions(request);
    std::vector<int> types = params_inv_types(request);

    // Build requests
    size_t count = regions.size() * types.size();
    log_info() << "GET /bulk-market-history with " << count << " histories" << std::endl;

    // Start cache lookups
    std::vector<CrestCacheEntry*> cache_entries;
    cache_entries.reserve(count);
    for (auto i : regions)
    {
        for (auto j : types)
        {
            std::stringstream ss;
            ss << "/market/" << i << "/types/" << j << "/history/";
            cache_entries.push_back(cache.get(ss.str()));
        }
    }
    //Results
    rapidjson::StringBuffer buffer;
    rapidjson::Writer<rapidjson::StringBuffer> json(buffer);
    json.StartArray();
    size_t k = 0;
    for (auto i : regions)
    {
        for (auto j : types)
        {
            auto &cache_entry = cache_entries[k];
            std::unique_lock<std::mutex> lock(cache_entry->mutex);
            cache_entry->wait(lock);
            // Process
            if (cache_entry->is_data_valid())
            {
                JsonReader doc(cache_entry->data);
                auto items = doc.as_object().get_array("items").get_native();

                json.StartObject();
                json.Key("region"); json.Int(i);
                json.Key("type");   json.Int(j);
                json.Key("items");
                items->Accept(json);

                json.EndObject();
            }

            ++k;
        }
    }
    json.EndArray();

    http::HttpResponse resp;
    resp.status_code = 200;
    resp.body.assign(
        (const uint8_t*)buffer.GetString(),
        (const uint8_t*)buffer.GetString() + buffer.GetSize());
    return resp;
}<commit_msg>Added performance timing to BulkMarketHistory.cpp<commit_after>#include "Precompiled.hpp"
#include "Errors.hpp"
#include "BulkMarketHistory.hpp"
#include "CrestBulkMarketOrders.hpp"
#include "CrestCache.hpp"
#include "lib/Params.hpp"
#include "http/HttpStatusErrors.hpp"
#include <iostream>
#include <chrono>
#include "json/JsonReader.hpp"
#include <rapidjson/stringbuffer.h>
#include <rapidjson/writer.h>
http::HttpResponse http_get_bulk_market_history(CrestCache &cache, http::HttpRequest &request)
{
    // Get params
    std::vector<int> regions = params_regions(request);
    std::vector<int> types = params_inv_types(request);

    // Build requests
    size_t count = regions.size() * types.size();
    log_info() << "GET /bulk-market-history with " << count << " histories" << std::endl;

    // Start cache lookups
    auto start = std::chrono::high_resolution_clock::now();

    std::vector<CrestCacheEntry*> cache_entries;
    cache_entries.reserve(count);
    for (auto i : regions)
    {
        for (auto j : types)
        {
            std::stringstream ss;
            ss << "/market/" << i << "/types/" << j << "/history/";
            cache_entries.push_back(cache.get(ss.str()));
        }
    }
    //Results
    rapidjson::StringBuffer buffer;
    rapidjson::Writer<rapidjson::StringBuffer> json(buffer);
    json.StartArray();
    size_t k = 0;
    for (auto i : regions)
    {
        for (auto j : types)
        {
            auto &cache_entry = cache_entries[k];
            std::unique_lock<std::mutex> lock(cache_entry->mutex);
            cache_entry->wait(lock);
            // Process
            if (cache_entry->is_data_valid())
            {
                JsonReader doc(cache_entry->data);
                auto items = doc.as_object().get_array("items").get_native();

                json.StartObject();
                json.Key("region"); json.Int(i);
                json.Key("type");   json.Int(j);
                json.Key("items");
                items->Accept(json);

                json.EndObject();
            }

            ++k;
        }
    }
    json.EndArray();
    auto end = std::chrono::high_resolution_clock::now();
    log_info() << "Cache lookup took "
        << std::chrono::duration_cast<std::chrono::milliseconds>(end - start).count()
        << "ms." << std::endl;

    http::HttpResponse resp;
    resp.status_code = 200;
    resp.body.assign(
        (const uint8_t*)buffer.GetString(),
        (const uint8_t*)buffer.GetString() + buffer.GetSize());
    return resp;
}
{'fraction_non_alphanumeric': 0.14843585975225698, 'fraction_numerical': 0.0025194205332773464, 'mean_word_length': 2.4521739130434783, 'pattern_counts': {'":': 0, '<': 66, '<?xml version=': 0, '>': 32, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4147814', 'n_tokens_mistral': 1493, 'n_tokens_neox': 1449, 'n_words': 270}