<commit_before>#include <curl/curl.h>
#include <memory>

#include "jload.h"

static bool wasCurlInit = false;

static size_t WriteCB(void *contents, size_t size, size_t nmemb, void *userp)
{
    std::string *userstr = (std::string*)userp;
    userstr->append((char*)contents, size * nmemb);
    return size * nmemb;
}

std::string getUrl(const char *url, const char* extraHeader)
{
    std::string resStr;

    CURL *curl;
    CURLcode res;

    initCurl();

    curl = curl_easy_init();
    if(!curl)
        return resStr;

    curl_easy_setopt(curl, CURLOPT_URL, url);
    curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 1L);
    curl_easy_setopt(curl, CURLOPT_SSL_VERIFYPEER, 0L);
    curl_easy_setopt(curl, CURLOPT_SSL_VERIFYHOST, 0L);
    curl_easy_setopt(curl, CURLOPT_NOSIGNAL, 1L);
    curl_easy_setopt(curl, CURLOPT_TIMEOUT, 10L);
    curl_easy_setopt(curl, CURLOPT_ACCEPT_ENCODING, "");
    curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 1L);
    curl_easy_setopt(curl, CURLOPT_MAXREDIRS, 10L);
    curl_easy_setopt(curl, CURLOPT_WRITEDATA, &resStr);
    curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, &WriteCB);

    struct curl_slist *hlist = nullptr;

    hlist = curl_slist_append(hlist, "Client-ID: deh0rnosabytmgde2jtn13k8mo899ye");

    if(extraHeader)
        hlist = curl_slist_append(hlist, extraHeader);

    curl_easy_setopt(curl, CURLOPT_HTTPHEADER, hlist);

    res = curl_easy_perform(curl);

    curl_slist_free_all(hlist);
    curl_easy_cleanup(curl);

    if(res != CURLE_OK)
        resStr.clear();

    return resStr;
}

Json::Value getJsonFromUrl(const char* url, const char* extraHeader)<commit_msg>Switch to Client-ID capable of using GraphQL<commit_after>#include <curl/curl.h>
#include <memory>

#include "jload.h"

static bool wasCurlInit = false;

static size_t WriteCB(void *contents, size_t size, size_t nmemb, void *userp)
{
    std::string *userstr = (std::string*)userp;
    userstr->append((char*)contents, size * nmemb);
    return size * nmemb;
}

std::string getUrl(const char *url, const char* extraHeader)
{
    std::string resStr;

    CURL *curl;
    CURLcode res;

    initCurl();

    curl = curl_easy_init();
    if(!curl)
        return resStr;

    curl_easy_setopt(curl, CURLOPT_URL, url);
    curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 1L);
    curl_easy_setopt(curl, CURLOPT_SSL_VERIFYPEER, 0L);
    curl_easy_setopt(curl, CURLOPT_SSL_VERIFYHOST, 0L);
    curl_easy_setopt(curl, CURLOPT_NOSIGNAL, 1L);
    curl_easy_setopt(curl, CURLOPT_TIMEOUT, 10L);
    curl_easy_setopt(curl, CURLOPT_ACCEPT_ENCODING, "");
    curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 1L);
    curl_easy_setopt(curl, CURLOPT_MAXREDIRS, 10L);
    curl_easy_setopt(curl, CURLOPT_WRITEDATA, &resStr);
    curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, &WriteCB);

    struct curl_slist *hlist = nullptr;

    hlist = curl_slist_append(hlist, "Client-ID: jzkbprff40iqj646a697cyrvl0zt2m6");

    if(extraHeader)
        hlist = curl_slist_append(hlist, extraHeader);

    curl_easy_setopt(curl, CURLOPT_HTTPHEADER, hlist);

    res = curl_easy_perform(curl);

    curl_slist_free_all(hlist);
    curl_easy_cleanup(curl);

    if(res != CURLE_OK)
        resStr.clear();

    return resStr;
}

Json::Value getJsonFromUrl(const char* url, const char* extraHeader)
{'fraction_non_alphanumeric': 0.11151625645700396, 'fraction_numerical': 0.011242783348526283, 'mean_word_length': 4.636986301369863, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6999621', 'n_tokens_mistral': 1403, 'n_tokens_neox': 1226, 'n_words': 227}