<commit_before>#include <string>
#include <vector>

#include <curl/curl.h>

#include "utf8string.h"

static size_t header_callback(char *ptr, size_t size, size_t nmemb, void *userdata)
{
    std::vector<utf8string> *headers = reinterpret_cast<std::vector<utf8string> *>(userdata);
    std::string s(ptr, size*nmemb);
    auto i = s.find_last_not_of("\r\n");
    headers->push_back(s.substr(0, i+1));
    return size*nmemb;
}

static size_t data_callback(char *ptr, size_t size, size_t nmemb, void *userdata)
{
    std::string *data = reinterpret_cast<std::string *>(userdata);
    data->append(std::string(ptr, size*nmemb));
    return size*nmemb;
}

namespace rtl {

std::string http$get(const std::string &url, std::vector<utf8string> *headers)
{
    std::string data;
    CURL *curl = curl_easy_init();
    curl_easy_setopt(curl, CURLOPT_URL, url.c_str());
    curl_easy_setopt(curl, CURLOPT_HEADERFUNCTION, header_callback);
    curl_easy_setopt(curl, CURLOPT_HEADERDATA, headers);
    curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, data_callback);
    curl_easy_setopt(curl, CURLOPT_WRITEDATA, &data);
    char error[CURL_ERROR_SIZE];
    curl_easy_setopt(curl, CURLOPT_ERRORBUFFER, error);
    CURLcode r = curl_easy_perform(curl);
    if (r == CURLE_OK) {
        long rc;
        curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &rc);
        //printf("rc %ld\n", rc);
    } else {
        printf("r %d error %s\n", r, error);
    }
    curl_easy_cleanup(curl);
    return data;
}
<commit_msg>Add User-Agent to curl http requests<commit_after>#include <string>
#include <vector>

#include <curl/curl.h>

#include "utf8string.h"

static size_t header_callback(char *ptr, size_t size, size_t nmemb, void *userdata)
{
    std::vector<utf8string> *headers = reinterpret_cast<std::vector<utf8string> *>(userdata);
    std::string s(ptr, size*nmemb);
    auto i = s.find_last_not_of("\r\n");
    headers->push_back(s.substr(0, i+1));
    return size*nmemb;
}

static size_t data_callback(char *ptr, size_t size, size_t nmemb, void *userdata)
{
    std::string *data = reinterpret_cast<std::string *>(userdata);
    data->append(std::string(ptr, size*nmemb));
    return size*nmemb;
}

namespace rtl {

std::string http$get(const std::string &url, std::vector<utf8string> *headers)
{
    std::string data;
    CURL *curl = curl_easy_init();
    curl_easy_setopt(curl, CURLOPT_URL, url.c_str());
    curl_easy_setopt(curl, CURLOPT_USERAGENT, "Neon/0.1");
    curl_easy_setopt(curl, CURLOPT_HEADERFUNCTION, header_callback);
    curl_easy_setopt(curl, CURLOPT_HEADERDATA, headers);
    curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, data_callback);
    curl_easy_setopt(curl, CURLOPT_WRITEDATA, &data);
    char error[CURL_ERROR_SIZE];
    curl_easy_setopt(curl, CURLOPT_ERRORBUFFER, error);
    CURLcode r = curl_easy_perform(curl);
    if (r == CURLE_OK) {
        long rc;
        curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &rc);
        //printf("rc %ld\n", rc);
    } else {
        printf("r %d error %s\n", r, error);
    }
    curl_easy_cleanup(curl);
    return data;
}

{'fraction_non_alphanumeric': 0.1396103896103896, 'fraction_numerical': 0.004545454545454545, 'mean_word_length': 4.581521739130435, 'pattern_counts': {'":': 0, '<': 19, '<?xml version=': 0, '>': 23, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1717794', 'n_tokens_mistral': 1218, 'n_tokens_neox': 1139, 'n_words': 222}