<commit_before>
#include <mutex>
#include <string>
#include <string.h>
#include <vector>
#include <thread>
#include <stdio.h>
#include <stdio.h>
#include <stdlib.h>
#include <iostream>
#include <cstdlib>
#include <curl/curl.h>
#include <sstream>
#include <queue>
#include <map>
#include "json.hpp"

/*
input(output, outputSize, function) ->
    "GET" -> addToWorkhorseQueue(QueueItem) - - -> Workhorse() -> mutex_lock() -> pop_front() -> mutex_unlock() -> fetchGETURLResult() -> 
    "POST" -> -> addToWorkhorseQueue(QueueItem) - - -> Workhorse() -> mutex_lock() -> pop_front() -> mutex_unlock() -> fetchPOSTURLResult() -> 
*/

class FetchURL
{
  public:
    struct FetchResult
    {
        std::string result;
        int status; //can be 0 = not completed yet; 1 = completed; 2 = error method, 3 = error on request
    };
    struct FetchQueueItem
    {
        int key;
        std::map<std::string, std::string> params;
    };

    void QueueWorkerThread();
    void InitializeThreads();
    int CallExtensionArgs(char *output, int outputSize, const char *function, const char **args, int argsCnt);
    int AddRequestToQueue(const char **args, int argsCnt);
    int GetStatus(int key);
    std::string GetResult(int key);
    std::string JsonToArray(std::string json);
  private:
    const char *cmdUrl = "#url";
    const char *cmdMethod = "#method";
    const char *cmdJSONDec = "#jsonToArray";
    const char *actionGET = "GET";
    const char *actionPUT = "PUT";
    const char *actionPOST = "POST";
    const char *actionPATCH = "PATH";
    const char *actionDELETE = "DELETE";
    const char *actionTRACE = "TRACE";
    const char *paramSep = "=";

    std::mutex queue_mutex;
    std::queue<FetchURL::FetchQueueItem> queue;
    std::mutex resList_mutex;
    std::map<int, FetchURL::FetchResult> resList;

    bool workerInitialized = false;

    std::map<std::string, std::string> parseRequestArgs(const char **args, int argsCnt);
    int addResultToQueue();
    void fetchResult(FetchURL::FetchQueueItem * qItem);
    std::string jsonToArray_object(nlohmann::json j);
    std::string jsonToArray_array(nlohmann::json j);
};
<commit_msg>renamed "PATH" to "PATCH"<commit_after>
#include <mutex>
#include <string>
#include <string.h>
#include <vector>
#include <thread>
#include <stdio.h>
#include <stdio.h>
#include <stdlib.h>
#include <iostream>
#include <cstdlib>
#include <curl/curl.h>
#include <sstream>
#include <queue>
#include <map>
#include "json.hpp"

/*
input(output, outputSize, function) ->
    "GET" -> addToWorkhorseQueue(QueueItem) - - -> Workhorse() -> mutex_lock() -> pop_front() -> mutex_unlock() -> fetchGETURLResult() -> 
    "POST" -> -> addToWorkhorseQueue(QueueItem) - - -> Workhorse() -> mutex_lock() -> pop_front() -> mutex_unlock() -> fetchPOSTURLResult() -> 
*/

class FetchURL
{
  public:
    struct FetchResult
    {
        std::string result;
        int status; //can be 0 = not completed yet; 1 = completed; 2 = error method, 3 = error on request
    };
    struct FetchQueueItem
    {
        int key;
        std::map<std::string, std::string> params;
    };

    void QueueWorkerThread();
    void InitializeThreads();
    int CallExtensionArgs(char *output, int outputSize, const char *function, const char **args, int argsCnt);
    int AddRequestToQueue(const char **args, int argsCnt);
    int GetStatus(int key);
    std::string GetResult(int key);
    std::string JsonToArray(std::string json);
  private:
    const char *cmdUrl = "#url";
    const char *cmdMethod = "#method";
    const char *cmdJSONDec = "#jsonToArray";
    const char *actionGET = "GET";
    const char *actionPUT = "PUT";
    const char *actionPOST = "POST";
    const char *actionPATCH = "PATCH";
    const char *actionDELETE = "DELETE";
    const char *actionTRACE = "TRACE";
    const char *paramSep = "=";

    std::mutex queue_mutex;
    std::queue<FetchURL::FetchQueueItem> queue;
    std::mutex resList_mutex;
    std::map<int, FetchURL::FetchResult> resList;

    bool workerInitialized = false;

    std::map<std::string, std::string> parseRequestArgs(const char **args, int argsCnt);
    int addResultToQueue();
    void fetchResult(FetchURL::FetchQueueItem * qItem);
    std::string jsonToArray_object(nlohmann::json j);
    std::string jsonToArray_array(nlohmann::json j);
};

{'fraction_non_alphanumeric': 0.1470656460218047, 'fraction_numerical': 0.001855717930874507, 'mean_word_length': 3.8668171557562077, 'pattern_counts': {'":': 0, '<': 39, '<?xml version=': 0, '>': 71, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '295449', 'n_tokens_mistral': 1417, 'n_tokens_neox': 1376, 'n_words': 365}