<commit_before>#include <curl/curl.h>
#include <errno.h>
#include <gio/gio.h>
#include <glib/gstdio.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>

#include "network.h"

/* We need to make sure that we can support large bundles. */
G_STATIC_ASSERT(sizeof(curl_off_t) == 8);

typedef struct {
	const gchar *url;

	FILE *dl;

	curl_off_t pos;
	curl_off_t limit;

	gchar *err;
} RaucTransfer;

gboolean network_init(GError **error)
{
	CURLcode res;

	g_return_val_if_fail(error == NULL || *error == NULL, FALSE);

	res = curl_global_init(CURL_GLOBAL_ALL);
	if (res != CURLE_OK) {
		g_set_error(error, G_IO_ERROR, G_IO_ERROR_FAILED, "Initializing curl failed: %s", curl_easy_strerror(res));
		return FALSE;
	}

	return TRUE;
}

static size_t write_cb(char *ptr, size_t size, size_t nmemb, void *userdata)
{
	RaucTransfer *xfer = userdata;
	size_t res;

	/* check transfer limit */
	if (xfer->limit) {
		if ((guint64)(xfer->pos + size*nmemb) > (guint64)xfer->limit) {
			xfer->err = g_strdup("Maximum bundle download size exceeded. Download aborted.");
			return 0;
		}
	}

	res = fwrite(ptr, size, nmemb, xfer->dl);
	xfer->pos += size*res;

	return res;
}

static int xfer_cb(void *clientp, curl_off_t dltotal, curl_off_t dlnow,
		curl_off_t ultotal, curl_off_t ulnow)
{
	RaucTransfer *xfer = clientp;

	/* check transfer limit */
	if (xfer->limit) {
		if ((dlnow > xfer->limit)
		    || (dltotal > xfer->limit)) {
			xfer->err = g_strdup("Maximum bundle download size exceeded. Download aborted.");
			return 1;
		}
	}

	return 0;
}

static gboolean transfer(RaucTransfer *xfer, GError **error)
{
	CURL *curl = NULL;
	CURLcode r;
	char errbuf[CURL_ERROR_SIZE];
	gboolean res = FALSE;

	g_return_val_if_fail(xfer, FALSE);
	g_return_val_if_fail(error == NULL || *error == NULL, FALSE);

	curl = curl_easy_init();
	if (curl == NULL) {
		g_set_error(error, G_IO_ERROR, G_IO_ERROR_FAILED, "Unable to start libcurl easy session");
		goto out;
	}

	curl_easy_setopt(curl, CURLOPT_VERBOSE, 1L);
	curl_easy_setopt(curl, CURLOPT_NOSIGNAL, 1L); /* avoid signals for threading */
	curl_easy_setopt(curl, CURLOPT_URL, xfer->url);
	curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 1L);
	curl_easy_setopt(curl, CURLOPT_MAXREDIRS, 8L);
	//curl_easy_setopt(curl, CURLOPT_MAX_RECV_SPEED_LARGE, 1048576L); /* bytes per second */
	//curl_easy_setopt(curl,  CURLOPT_LOW_SPEED_LIMIT, 1024L);
	//curl_easy_setopt(curl,  CURLOPT_LOW_SPEED_TIME, 60L);
	curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, write_cb);
	curl_easy_setopt(curl, CURLOPT_WRITEDATA, xfer);
	curl_easy_setopt(curl, CURLOPT_XFERINFOFUNCTION, xfer_cb);
	curl_easy_setopt(curl, CURLOPT_XFERINFODATA, xfer);
	curl_easy_setopt(curl, CURLOPT_FAILONERROR, 1L);
	curl_easy_setopt(curl, CURLOPT_ERRORBUFFER, errbuf);
	curl_easy_setopt(curl, CURLOPT_MAXFILESIZE_LARGE, xfer->limit);
	/* decode all supported Accept-Encoding headers */
	curl_easy_setopt(curl, CURLOPT_ACCEPT_ENCODING, "");
	curl_easy_setopt(curl, CURLOPT_NETRC, CURL_NETRC_OPTIONAL);

	/* set error buffer empty before performing a request */<commit_msg>src/network: fix max bundle size not always being checked<commit_after>#include <curl/curl.h>
#include <errno.h>
#include <gio/gio.h>
#include <glib/gstdio.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>

#include "network.h"

/* We need to make sure that we can support large bundles. */
G_STATIC_ASSERT(sizeof(curl_off_t) == 8);

typedef struct {
	const gchar *url;

	FILE *dl;

	curl_off_t pos;
	curl_off_t limit;

	gchar *err;
} RaucTransfer;

gboolean network_init(GError **error)
{
	CURLcode res;

	g_return_val_if_fail(error == NULL || *error == NULL, FALSE);

	res = curl_global_init(CURL_GLOBAL_ALL);
	if (res != CURLE_OK) {
		g_set_error(error, G_IO_ERROR, G_IO_ERROR_FAILED, "Initializing curl failed: %s", curl_easy_strerror(res));
		return FALSE;
	}

	return TRUE;
}

static size_t write_cb(char *ptr, size_t size, size_t nmemb, void *userdata)
{
	RaucTransfer *xfer = userdata;
	size_t res;

	/* check transfer limit */
	if (xfer->limit) {
		if ((guint64)(xfer->pos + size*nmemb) > (guint64)xfer->limit) {
			xfer->err = g_strdup("Maximum bundle download size exceeded. Download aborted.");
			return 0;
		}
	}

	res = fwrite(ptr, size, nmemb, xfer->dl);
	xfer->pos += size*res;

	return res;
}

static int xfer_cb(void *clientp, curl_off_t dltotal, curl_off_t dlnow,
		curl_off_t ultotal, curl_off_t ulnow)
{
	RaucTransfer *xfer = clientp;

	/* check transfer limit */
	if (xfer->limit) {
		if ((dlnow > xfer->limit)
		    || (dltotal > xfer->limit)) {
			xfer->err = g_strdup("Maximum bundle download size exceeded. Download aborted.");
			return 1;
		}
	}

	return 0;
}

static gboolean transfer(RaucTransfer *xfer, GError **error)
{
	CURL *curl = NULL;
	CURLcode r;
	char errbuf[CURL_ERROR_SIZE];
	gboolean res = FALSE;

	g_return_val_if_fail(xfer, FALSE);
	g_return_val_if_fail(error == NULL || *error == NULL, FALSE);

	curl = curl_easy_init();
	if (curl == NULL) {
		g_set_error(error, G_IO_ERROR, G_IO_ERROR_FAILED, "Unable to start libcurl easy session");
		goto out;
	}

	curl_easy_setopt(curl, CURLOPT_VERBOSE, 1L);
	curl_easy_setopt(curl, CURLOPT_NOSIGNAL, 1L); /* avoid signals for threading */
	curl_easy_setopt(curl, CURLOPT_URL, xfer->url);
	curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 1L);
	curl_easy_setopt(curl, CURLOPT_MAXREDIRS, 8L);
	//curl_easy_setopt(curl, CURLOPT_MAX_RECV_SPEED_LARGE, 1048576L); /* bytes per second */
	//curl_easy_setopt(curl,  CURLOPT_LOW_SPEED_LIMIT, 1024L);
	//curl_easy_setopt(curl,  CURLOPT_LOW_SPEED_TIME, 60L);
	curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, write_cb);
	curl_easy_setopt(curl, CURLOPT_WRITEDATA, xfer);
	curl_easy_setopt(curl, CURLOPT_NOPROGRESS, 0); /* allow XFERINFOFUNCTION callbacks */
	curl_easy_setopt(curl, CURLOPT_XFERINFOFUNCTION, xfer_cb);
	curl_easy_setopt(curl, CURLOPT_XFERINFODATA, xfer);
	curl_easy_setopt(curl, CURLOPT_FAILONERROR, 1L);
	curl_easy_setopt(curl, CURLOPT_ERRORBUFFER, errbuf);
	curl_easy_setopt(curl, CURLOPT_MAXFILESIZE_LARGE, xfer->limit);
	/* decode all supported Accept-Encoding headers */
	curl_easy_setopt(curl, CURLOPT_ACCEPT_ENCODING, "");
	curl_easy_setopt(curl, CURLOPT_NETRC, CURL_NETRC_OPTIONAL);

	/* set error buffer empty before performing a request */
{'fraction_non_alphanumeric': 0.12478031634446397, 'fraction_numerical': 0.008467806358843265, 'mean_word_length': 5.394279877425945, 'pattern_counts': {'":': 0, '<': 17, '<?xml version=': 0, '>': 47, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 8, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7006304', 'n_tokens_mistral': 2794, 'n_tokens_neox': 2454, 'n_words': 532}