<commit_before>#include "curl/curl.h"
#include <string.h>
#include <stdlib.h>
#include "main.h"
#include "http.h"


size_t WriteMemoryCallback(void *ptr, size_t size, size_t nmemb, void *data) {
	size_t realsize = size * nmemb;
	struct MemoryStruct *mem = (struct MemoryStruct *)data;

	mem->memory = realloc(mem->memory, mem->size + realsize + 1);
	if (mem->memory) {
		memcpy(&(mem->memory[mem->size]), ptr, realsize);
		mem->size += realsize;
		mem->memory[mem->size] = 0;
	}
	return realsize;
}

int http_request(char *url, struct buffer *response) {
	CURL *curl;
	CURLcode res;
	struct MemoryStruct chunk;
	char curlerror[CURL_ERROR_SIZE];

	chunk.memory=NULL;
	chunk.size = 0;

	curl_global_init(CURL_GLOBAL_DEFAULT);
	curl = curl_easy_init();

	if(!curl) {
		printf("unable to load libcurl\n");
		return 1;
	}

	curl_easy_setopt(curl, CURLOPT_NOPROGRESS, 1);
	curl_easy_setopt(curl, CURLOPT_URL, url);
	curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, WriteMemoryCallback);
	curl_easy_setopt(curl, CURLOPT_WRITEDATA, (void *)&chunk);
	curl_easy_setopt(curl, CURLOPT_USERAGENT, "ChaosVPNclient/2.0");
	curl_easy_setopt(curl, CURLOPT_ERRORBUFFER, curlerror);

	res = curl_easy_perform(curl);

	if (res) {
		printf("\nrequest failed: %s\n", curlerror);
	}

	response->text = chunk.memory;<commit_msg>set missing CURLOPT_SSL_VERIFYHOST and CURLOPT_SSL_VERIFYPEER curl options, otherwise libcurl-gnutls does not like the www.vpn.hamburg.ccc.de certificate (the openssl libcurl accepts it)<commit_after>#include "curl/curl.h"
#include <string.h>
#include <stdlib.h>
#include "main.h"
#include "http.h"


size_t WriteMemoryCallback(void *ptr, size_t size, size_t nmemb, void *data) {
	size_t realsize = size * nmemb;
	struct MemoryStruct *mem = (struct MemoryStruct *)data;

	mem->memory = realloc(mem->memory, mem->size + realsize + 1);
	if (mem->memory) {
		memcpy(&(mem->memory[mem->size]), ptr, realsize);
		mem->size += realsize;
		mem->memory[mem->size] = 0;
	}
	return realsize;
}

int http_request(char *url, struct buffer *response) {
	CURL *curl;
	CURLcode res;
	struct MemoryStruct chunk;
	char curlerror[CURL_ERROR_SIZE];

	chunk.memory=NULL;
	chunk.size = 0;

	curl_global_init(CURL_GLOBAL_DEFAULT);
	curl = curl_easy_init();

	if(!curl) {
		printf("unable to load libcurl\n");
		return 1;
	}

	curl_easy_setopt(curl, CURLOPT_NOPROGRESS, 1);
	curl_easy_setopt(curl, CURLOPT_URL, url);
	curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, WriteMemoryCallback);
	curl_easy_setopt(curl, CURLOPT_WRITEDATA, (void *)&chunk);
	curl_easy_setopt(curl, CURLOPT_USERAGENT, "ChaosVPNclient/2.0");
	curl_easy_setopt(curl, CURLOPT_ERRORBUFFER, curlerror);
	curl_easy_setopt(curl, CURLOPT_SSL_VERIFYPEER, 0);
	curl_easy_setopt(curl, CURLOPT_SSL_VERIFYHOST, 0);

	res = curl_easy_perform(curl);

	if (res) {
		printf("\nrequest failed: %s\n", curlerror);
	}

	response->text = chunk.memory;
{'fraction_non_alphanumeric': 0.13685307398402222, 'fraction_numerical': 0.0055574852379298365, 'mean_word_length': 5.808510638297872, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 27, 'https://': 0, 'lorem ipsum': 0, 'www.': 1, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2151072', 'n_tokens_mistral': 1236, 'n_tokens_neox': 1094, 'n_words': 228}