<commit_before>#include "output.h"

#include <errno.h>
#include <glib.h>
#include <stdlib.h>
#include <string.h>
#include <sys/stat.h>
#include <unistd.h>

static GOnce check_write_out_once = G_ONCE_INIT;

#ifdef UNIT_TESTING
extern gint fchmod_t();
extern gint mkstemp_t();
extern gint close_t();
#define fchmod(X, Y) fchmod_t(X, Y)
#define mkstemp(X) mkstemp_t(X)
#define close(X) close_t(X)
#endif

#include "cmd.h"

static const gint WSHC_MAX_OUT_LEN = 50;
static const gint WSHC_MAX_HOSTS = 20;
static const gint WSHC_SCORE_THRESHOLD = 2;
static const gsize WSHC_ALLOC_LEN = 4096;
static const gchar* WSHC_STDERR_TAIL = "stderr:\n";
static const gchar* WSHC_STDOUT_TAIL = "stdout:\n";
static const gsize WSHC_STDERR_TAIL_SIZE = 9;
static const gsize WSHC_STDOUT_TAIL_SIZE = 9;

struct collate {
	GSList* hosts;
	gchar** output;
	gchar** error;
	gint exit_code;
};

// Struct for final collate info
struct f_collate {
	gchar** out;
	gsize* size;
};

static void free_output(wshc_host_output_t* out) {
	if (out->error) g_strfreev(out->error);
	if (out->output) g_strfreev(out->output);
	g_slice_free(wshc_host_output_t, out);
}

void wshc_init_output(wshc_output_info_t** out, gboolean show_stdout) {
	g_assert(out);

	*out = g_slice_new0(wshc_output_info_t);

	(*out)->show_stdout = show_stdout;
	(*out)->mut = g_mutex_new();
	(*out)->output = g_hash_table_new_full(g_str_hash, g_str_equal,
		(GDestroyNotify)g_free, (GDestroyNotify)free_output);
}

void wshc_cleanup_output(wshc_output_info_t** out) {
	g_assert(*out);

	g_hash_table_destroy((*out)->output);
	g_mutex_free((*out)->mut);
	g_slice_free(wshc_output_info_t, *out);
	*out = NULL;
}

static gint write_output_mem(wshc_output_info_t* out, const gchar* hostname, const wsh_cmd_res_t* res) {
	// Freed on destruction
	wshc_host_output_t* host_out = g_slice_new0(wshc_host_output_t);
	host_out->error = g_strdupv(res->std_error);
	host_out->output = g_strdupv(res->std_output);
	host_out->exit_code = res->exit_status;

	g_mutex_lock(out->mut);
	g_hash_table_insert(out->output, g_strdup(hostname), host_out);
	g_mutex_unlock(out->mut);

	return EXIT_SUCCESS;
}

static gint write_output_file(wshc_output_info_t* out, const gchar* hostname, const wsh_cmd_res_t* res) {
	return 0;
}

/* The idea is to try and guess the amount of output that the 
 * whole set of hosts will have based on the first host and the number
 * of hosts.
 *
 * If there's too much to reasonably fit in memory, then we'll write it
 * out to a temp file.
 *
 * This is very static right now, and should be implemented more dynamically/
 * intelligently
 */
gint wshc_check_write_output(struct check_write_out_args* args) {
	g_assert(args);
	g_assert(args->num_hosts);
	g_assert(args->out);
	g_assert(args->res);

	gint score = 0;
	gint ret = EXIT_SUCCESS;
	gboolean write = FALSE;
	wshc_output_info_t* out = args->out;
	const wsh_cmd_res_t* res = args->res;
	guint num_hosts = args->num_hosts;

	if (res->std_error_len > WSHC_MAX_OUT_LEN) 						score++;
	if (out->show_stdout && res->std_output_len > WSHC_MAX_OUT_LEN)	score++;
	if (num_hosts > WSHC_MAX_HOSTS)									score++;

	g_mutex_lock(out->mut);
	if (score >= WSHC_SCORE_THRESHOLD) {
		if ((out->out_fd = mkstemp("/tmp/wshc_out.tmp.XXXXXX")) == -1) {
			g_printerr("Could not create temp output file: %s\n", strerror(errno));
			g_printerr("Continuing without\n");
			write = FALSE;
			ret = EXIT_FAILURE;
			goto wshc_check_write_output_error;
		}

		// This should be done by mkstemp on most UNIXes, but older versions
		// of Linux didn't so I'm just going to be careful
		if (fchmod(out->out_fd, 0x600)) {
			g_printerr("Could not chown temp output file: %s\n", strerror(errno));
			g_printerr("Continuing without\n");
			write = FALSE;
			(void)close(out->out_fd);
			ret = EXIT_FAILURE;
			goto wshc_check_write_output_error;
		}

		write = TRUE;
	}

wshc_check_write_output_error:
	out->write_out = write;
	g_mutex_unlock(out->mut);
	return ret;
}

gint wshc_write_output(wshc_output_info_t* out, guint num_hosts, const gchar* hostname, const wsh_cmd_res_t* res) {
	struct check_write_out_args args = {
		.out = out,
		.res = res,
		.num_hosts = num_hosts,
	};

	/* If there's an error, output it immediately */
	if (res->error_message) {
		g_printerr("%s: %s\n", hostname, res->error_message);
		return EXIT_SUCCESS;
	}

#ifdef UNIT_TESTING
	check_write_out_once.status = G_ONCE_STATUS_READY;
#endif

	g_once(&check_write_out_once, (GThreadFunc)wshc_check_write_output, &args);

	if (out->write_out) return write_output_file(out, hostname, res);
	else				return write_output_mem(out, hostname, res);
}

static void hash_compare(gchar* hostname, wshc_host_output_t* out, GSList** clist) {
	for (GSList* p = *clist; p != NULL && p->data != NULL; p = p->next) {
		if (((struct collate*)(p->data))->exit_code == out->exit_code) {
			((struct collate*)(p->data))->hosts =
				g_slist_prepend(((struct collate*)(p->data))->hosts, hostname);
			return;
		}
	}

	struct collate* c = g_slice_new0(struct collate);
	c->output = out->output;
	c->error = out->error;
	c->exit_code = out->exit_code;
	c->hosts = NULL;
	c->hosts = g_slist_prepend(c->hosts, hostname);

	*clist = g_slist_prepend(*clist, c);
}

static void construct_out(struct collate* c, struct f_collate* f) {
	gsize new_len = 0;

	gsize host_list_len = 0;
	for (GSList* host_list = c->hosts; host_list != NULL; host_list = host_list->next) {
		host_list_len += strlen(host_list->data) + 2;<commit_msg>Collation is more intelligent. Fixes #21<commit_after>#include "output.h"

#include <errno.h>
#include <glib.h>
#include <stdlib.h>
#include <string.h>
#include <sys/stat.h>
#include <unistd.h>

static GOnce check_write_out_once = G_ONCE_INIT;

#ifdef UNIT_TESTING
extern gint fchmod_t();
extern gint mkstemp_t();
extern gint close_t();
#define fchmod(X, Y) fchmod_t(X, Y)
#define mkstemp(X) mkstemp_t(X)
#define close(X) close_t(X)
#endif

#include "cmd.h"

static const gint WSHC_MAX_OUT_LEN = 50;
static const gint WSHC_MAX_HOSTS = 20;
static const gint WSHC_SCORE_THRESHOLD = 2;
static const gsize WSHC_ALLOC_LEN = 4096;
static const gchar* WSHC_STDERR_TAIL = "stderr:\n";
static const gchar* WSHC_STDOUT_TAIL = "stdout:\n";
static const gsize WSHC_STDERR_TAIL_SIZE = 9;
static const gsize WSHC_STDOUT_TAIL_SIZE = 9;

struct collate {
	GSList* hosts;
	gchar** output;
	gchar** error;
	gint exit_code;
};

// Struct for final collate info
struct f_collate {
	gchar** out;
	gsize* size;
};

static void free_output(wshc_host_output_t* out) {
	if (out->error) g_strfreev(out->error);
	if (out->output) g_strfreev(out->output);
	g_slice_free(wshc_host_output_t, out);
}

void wshc_init_output(wshc_output_info_t** out, gboolean show_stdout) {
	g_assert(out);

	*out = g_slice_new0(wshc_output_info_t);

	(*out)->show_stdout = show_stdout;
	(*out)->mut = g_mutex_new();
	(*out)->output = g_hash_table_new_full(g_str_hash, g_str_equal,
		(GDestroyNotify)g_free, (GDestroyNotify)free_output);
}

void wshc_cleanup_output(wshc_output_info_t** out) {
	g_assert(*out);

	g_hash_table_destroy((*out)->output);
	g_mutex_free((*out)->mut);
	g_slice_free(wshc_output_info_t, *out);
	*out = NULL;
}

static gint write_output_mem(wshc_output_info_t* out, const gchar* hostname, const wsh_cmd_res_t* res) {
	// Freed on destruction
	wshc_host_output_t* host_out = g_slice_new0(wshc_host_output_t);
	host_out->error = g_strdupv(res->std_error);
	host_out->output = g_strdupv(res->std_output);
	host_out->exit_code = res->exit_status;

	g_mutex_lock(out->mut);
	g_hash_table_insert(out->output, g_strdup(hostname), host_out);
	g_mutex_unlock(out->mut);

	return EXIT_SUCCESS;
}

static gint write_output_file(wshc_output_info_t* out, const gchar* hostname, const wsh_cmd_res_t* res) {
	return 0;
}

/* The idea is to try and guess the amount of output that the 
 * whole set of hosts will have based on the first host and the number
 * of hosts.
 *
 * If there's too much to reasonably fit in memory, then we'll write it
 * out to a temp file.
 *
 * This is very static right now, and should be implemented more dynamically/
 * intelligently
 */
gint wshc_check_write_output(struct check_write_out_args* args) {
	g_assert(args);
	g_assert(args->num_hosts);
	g_assert(args->out);
	g_assert(args->res);

	gint score = 0;
	gint ret = EXIT_SUCCESS;
	gboolean write = FALSE;
	wshc_output_info_t* out = args->out;
	const wsh_cmd_res_t* res = args->res;
	guint num_hosts = args->num_hosts;

	if (res->std_error_len > WSHC_MAX_OUT_LEN) 						score++;
	if (out->show_stdout && res->std_output_len > WSHC_MAX_OUT_LEN)	score++;
	if (num_hosts > WSHC_MAX_HOSTS)									score++;

	g_mutex_lock(out->mut);
	if (score >= WSHC_SCORE_THRESHOLD) {
		if ((out->out_fd = mkstemp("/tmp/wshc_out.tmp.XXXXXX")) == -1) {
			g_printerr("Could not create temp output file: %s\n", strerror(errno));
			g_printerr("Continuing without\n");
			write = FALSE;
			ret = EXIT_FAILURE;
			goto wshc_check_write_output_error;
		}

		// This should be done by mkstemp on most UNIXes, but older versions
		// of Linux didn't so I'm just going to be careful
		if (fchmod(out->out_fd, 0x600)) {
			g_printerr("Could not chown temp output file: %s\n", strerror(errno));
			g_printerr("Continuing without\n");
			write = FALSE;
			(void)close(out->out_fd);
			ret = EXIT_FAILURE;
			goto wshc_check_write_output_error;
		}

		write = TRUE;
	}

wshc_check_write_output_error:
	out->write_out = write;
	g_mutex_unlock(out->mut);
	return ret;
}

gint wshc_write_output(wshc_output_info_t* out, guint num_hosts, const gchar* hostname, const wsh_cmd_res_t* res) {
	struct check_write_out_args args = {
		.out = out,
		.res = res,
		.num_hosts = num_hosts,
	};

	/* If there's an error, output it immediately */
	if (res->error_message) {
		g_printerr("%s: %s\n", hostname, res->error_message);
		return EXIT_SUCCESS;
	}

#ifdef UNIT_TESTING
	check_write_out_once.status = G_ONCE_STATUS_READY;
#endif

	g_once(&check_write_out_once, (GThreadFunc)wshc_check_write_output, &args);

	if (out->write_out) return write_output_file(out, hostname, res);
	else				return write_output_mem(out, hostname, res);
}

static gboolean cmp(struct collate* col, wshc_host_output_t* out) {
	if (col->exit_code != out->exit_code)
		return FALSE;

	gsize i = 0;
	for (gchar** cur = col->error; *cur != NULL; cur++) {
		if (! out->error[i])
			return FALSE;

		if (strncmp(out->error[i], *cur, strlen(*cur)))
			return FALSE;

		i++;
	}

	i = 0;
	for (gchar** cur = col->output; *cur != NULL; cur++) {
		if (! out->output[i])
			return FALSE;

		if (strncmp(out->output[i], *cur, strlen(*cur)))
			return FALSE;

		i++;
	}

	return TRUE;
}

static void hash_compare(gchar* hostname, wshc_host_output_t* out, GSList** clist) {
	for (GSList* p = *clist; p != NULL && p->data != NULL; p = p->next) {
		if (cmp((struct collate*)(p->data), out)) {
			((struct collate*)(p->data))->hosts =
				g_slist_prepend(((struct collate*)(p->data))->hosts, hostname);
			return;
		}
	}

	struct collate* c = g_slice_new0(struct collate);
	c->output = out->output;
	c->error = out->error;
	c->exit_code = out->exit_code;
	c->hosts = NULL;
	c->hosts = g_slist_prepend(c->hosts, hostname);

	*clist = g_slist_prepend(*clist, c);
}

static void construct_out(struct collate* c, struct f_collate* f) {
	gsize new_len = 0;

	gsize host_list_len = 0;
	for (GSList* host_list = c->hosts; host_list != NULL; host_list = host_list->next) {
		host_list_len += strlen(host_list->data) + 2;
{'fraction_non_alphanumeric': 0.13448758307100384, 'fraction_numerical': 0.004547044421126268, 'mean_word_length': 4.9443866943866945, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 143, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5279434', 'n_tokens_mistral': 4869, 'n_tokens_neox': 4610, 'n_words': 1027}