<commit_before>/*
 * Copyright (c) 2015, <NAME>
 * All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions are met:
 *	 * Redistributions of source code must retain the above copyright
 *	   notice, this list of conditions and the following disclaimer.
 *	 * Redistributions in binary form must reproduce the above copyright
 *	   notice, this list of conditions and the following disclaimer in the
 *	   documentation and/or other materials provided with the distribution.
 *
 * THIS SOFTWARE IS PROVIDED BY AUTHOR ''AS IS'' AND ANY
 * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
 * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
 * DISCLAIMED. IN NO EVENT SHALL AUTHOR BE LIABLE FOR ANY
 * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
 * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
 * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
 * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 */

#include "config.h"
#include "libutil/util.h"
#include "libserver/cfg_file.h"
#include "libserver/cfg_rcl.h"
#include "libserver/worker_util.h"
#include "libserver/rspamd_control.h"
#include "unix-std.h"

#ifdef HAVE_GLOB_H
#include <glob.h>
#endif

static gpointer init_hs_helper (struct rspamd_config *cfg);
static void start_hs_helper (struct rspamd_worker *worker);

worker_t hs_helper_worker = {
		"hs_helper",                /* Name */
		init_hs_helper,             /* Init function */
		start_hs_helper,            /* Start function */
		FALSE,                      /* No socket */
		TRUE,                       /* Unique */
		FALSE,                      /* Non threaded */
		TRUE,                       /* Killable */
		SOCK_STREAM                 /* TCP socket */
};

const static gdouble default_max_time = 1.0;

/*
 * Worker's context
 */
struct hs_helper_ctx {
	gchar *hs_dir;
	gdouble max_time;
	struct rspamd_config *cfg;
	struct event_base *ev_base;
};

static gpointer
init_hs_helper (struct rspamd_config *cfg)
{
	struct hs_helper_ctx *ctx;
	GQuark type;

	type = g_quark_try_string ("hs_helper");
	ctx = g_malloc0 (sizeof (*ctx));

	ctx->cfg = cfg;
	ctx->hs_dir = RSPAMD_DBDIR "/";
	ctx->max_time = default_max_time;

	rspamd_rcl_register_worker_option (cfg,
			type,
			"cache_dir",
			rspamd_rcl_parse_struct_string,
			ctx,
			G_STRUCT_OFFSET (struct hs_helper_ctx, hs_dir),
			0,
			"Directory where to save hyperscan compiled expressions");
	rspamd_rcl_register_worker_option (cfg,
			type,
			"max_time",
			rspamd_rcl_parse_struct_time,
			ctx,
			G_STRUCT_OFFSET (struct hs_helper_ctx, max_time),
			RSPAMD_CL_FLAG_TIME_FLOAT,
			"Maximum time to wait for compilation of a single expression");
	rspamd_rcl_register_worker_option (cfg,
			type,
			"timeout",
			rspamd_rcl_parse_struct_time,
			ctx,
			G_STRUCT_OFFSET (struct hs_helper_ctx, max_time),
			RSPAMD_CL_FLAG_TIME_FLOAT,
			"Maximum time to wait for compilation of a single expression");

	return ctx;
}

/**
 * Clean
 */
static gboolean
rspamd_hs_helper_cleanup_dir (struct hs_helper_ctx *ctx, gboolean forced)
{
	struct stat st;
	glob_t globbuf;
	guint len, i;
	gint rc;
	gchar *pattern;
	gboolean ret = TRUE;

	if (stat (ctx->hs_dir, &st) == -1) {
		msg_err ("cannot stat path %s, %s",
				ctx->hs_dir,
				strerror (errno));
		return FALSE;
	}

	globbuf.gl_offs = 0;
	len = strlen (ctx->hs_dir) + 1 + sizeof ("*.hs");
	pattern = g_malloc (len);
	rspamd_snprintf (pattern, len, "%s%c%s", ctx->hs_dir, G_DIR_SEPARATOR, "*.hs");

	if ((rc = glob (pattern, GLOB_DOOFFS, NULL, &globbuf)) == 0) {
		for (i = 0; i < globbuf.gl_pathc; i++) {
			if (forced ||
					!rspamd_re_cache_is_valid_hyperscan_file (ctx->cfg->re_cache,
						globbuf.gl_pathv[i], TRUE)) {
				if (unlink (globbuf.gl_pathv[i]) == -1) {
					msg_err ("cannot unlink %s: %s", globbuf.gl_pathv[i],
							strerror (errno));
					ret = FALSE;
				}
			}
		}
	}
	else if (rc != GLOB_NOMATCH) {
		msg_err ("glob %s failed: %s", pattern, strerror (errno));
		ret = FALSE;
	}

	globfree (&globbuf);
	g_free (pattern);

	return ret;
}

static gboolean
rspamd_rs_compile (struct hs_helper_ctx *ctx, struct rspamd_worker *worker,
		gboolean forced)
{
	GError *err = NULL;
	static struct rspamd_srv_command srv_cmd;
	gint ncompiled;

	if (!rspamd_hs_helper_cleanup_dir (ctx, forced)) {
		msg_warn ("cannot cleanup cache dir '%s'", ctx->hs_dir);
	}

	if ((ncompiled = rspamd_re_cache_compile_hyperscan (ctx->cfg->re_cache,
			ctx->hs_dir, ctx->max_time,
			&err)) == -1) {
		msg_err ("failed to compile re cache: %e", err);
		g_error_free (err);

		return FALSE;
	}

	msg_info ("compiled %d regular expressions to the hyperscan tree",
			ncompiled);

	srv_cmd.type = RSPAMD_SRV_HYPERSCAN_LOADED;
	srv_cmd.cmd.hs_loaded.cache_dir = ctx->hs_dir;

	rspamd_srv_send_command (worker, ctx->ev_base, &srv_cmd, NULL, NULL);

	return TRUE;
}

static gboolean
rspamd_hs_helper_reload (struct rspamd_main *rspamd_main,
		struct rspamd_worker *worker, gint fd,
		struct rspamd_control_command *cmd,
		gpointer ud)<commit_msg>Add extra sleep to avoid race condition<commit_after>/*
 * Copyright (c) 2015, <NAME>
 * All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions are met:
 *	 * Redistributions of source code must retain the above copyright
 *	   notice, this list of conditions and the following disclaimer.
 *	 * Redistributions in binary form must reproduce the above copyright
 *	   notice, this list of conditions and the following disclaimer in the
 *	   documentation and/or other materials provided with the distribution.
 *
 * THIS SOFTWARE IS PROVIDED BY AUTHOR ''AS IS'' AND ANY
 * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
 * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
 * DISCLAIMED. IN NO EVENT SHALL AUTHOR BE LIABLE FOR ANY
 * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
 * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
 * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
 * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 */

#include "config.h"
#include "libutil/util.h"
#include "libserver/cfg_file.h"
#include "libserver/cfg_rcl.h"
#include "libserver/worker_util.h"
#include "libserver/rspamd_control.h"
#include "unix-std.h"

#ifdef HAVE_GLOB_H
#include <glob.h>
#endif

static gpointer init_hs_helper (struct rspamd_config *cfg);
static void start_hs_helper (struct rspamd_worker *worker);

worker_t hs_helper_worker = {
		"hs_helper",                /* Name */
		init_hs_helper,             /* Init function */
		start_hs_helper,            /* Start function */
		FALSE,                      /* No socket */
		TRUE,                       /* Unique */
		FALSE,                      /* Non threaded */
		TRUE,                       /* Killable */
		SOCK_STREAM                 /* TCP socket */
};

const static gdouble default_max_time = 1.0;

/*
 * Worker's context
 */
struct hs_helper_ctx {
	gchar *hs_dir;
	gdouble max_time;
	struct rspamd_config *cfg;
	struct event_base *ev_base;
};

static gpointer
init_hs_helper (struct rspamd_config *cfg)
{
	struct hs_helper_ctx *ctx;
	GQuark type;

	type = g_quark_try_string ("hs_helper");
	ctx = g_malloc0 (sizeof (*ctx));

	ctx->cfg = cfg;
	ctx->hs_dir = RSPAMD_DBDIR "/";
	ctx->max_time = default_max_time;

	rspamd_rcl_register_worker_option (cfg,
			type,
			"cache_dir",
			rspamd_rcl_parse_struct_string,
			ctx,
			G_STRUCT_OFFSET (struct hs_helper_ctx, hs_dir),
			0,
			"Directory where to save hyperscan compiled expressions");
	rspamd_rcl_register_worker_option (cfg,
			type,
			"max_time",
			rspamd_rcl_parse_struct_time,
			ctx,
			G_STRUCT_OFFSET (struct hs_helper_ctx, max_time),
			RSPAMD_CL_FLAG_TIME_FLOAT,
			"Maximum time to wait for compilation of a single expression");
	rspamd_rcl_register_worker_option (cfg,
			type,
			"timeout",
			rspamd_rcl_parse_struct_time,
			ctx,
			G_STRUCT_OFFSET (struct hs_helper_ctx, max_time),
			RSPAMD_CL_FLAG_TIME_FLOAT,
			"Maximum time to wait for compilation of a single expression");

	return ctx;
}

/**
 * Clean
 */
static gboolean
rspamd_hs_helper_cleanup_dir (struct hs_helper_ctx *ctx, gboolean forced)
{
	struct stat st;
	glob_t globbuf;
	guint len, i;
	gint rc;
	gchar *pattern;
	gboolean ret = TRUE;

	if (stat (ctx->hs_dir, &st) == -1) {
		msg_err ("cannot stat path %s, %s",
				ctx->hs_dir,
				strerror (errno));
		return FALSE;
	}

	globbuf.gl_offs = 0;
	len = strlen (ctx->hs_dir) + 1 + sizeof ("*.hs");
	pattern = g_malloc (len);
	rspamd_snprintf (pattern, len, "%s%c%s", ctx->hs_dir, G_DIR_SEPARATOR, "*.hs");

	if ((rc = glob (pattern, GLOB_DOOFFS, NULL, &globbuf)) == 0) {
		for (i = 0; i < globbuf.gl_pathc; i++) {
			if (forced ||
					!rspamd_re_cache_is_valid_hyperscan_file (ctx->cfg->re_cache,
						globbuf.gl_pathv[i], TRUE)) {
				if (unlink (globbuf.gl_pathv[i]) == -1) {
					msg_err ("cannot unlink %s: %s", globbuf.gl_pathv[i],
							strerror (errno));
					ret = FALSE;
				}
			}
		}
	}
	else if (rc != GLOB_NOMATCH) {
		msg_err ("glob %s failed: %s", pattern, strerror (errno));
		ret = FALSE;
	}

	globfree (&globbuf);
	g_free (pattern);

	return ret;
}

static gboolean
rspamd_rs_compile (struct hs_helper_ctx *ctx, struct rspamd_worker *worker,
		gboolean forced)
{
	GError *err = NULL;
	static struct rspamd_srv_command srv_cmd;
	gint ncompiled;

	if (!rspamd_hs_helper_cleanup_dir (ctx, forced)) {
		msg_warn ("cannot cleanup cache dir '%s'", ctx->hs_dir);
	}

	if ((ncompiled = rspamd_re_cache_compile_hyperscan (ctx->cfg->re_cache,
			ctx->hs_dir, ctx->max_time,
			&err)) == -1) {
		msg_err ("failed to compile re cache: %e", err);
		g_error_free (err);

		return FALSE;
	}

	msg_info ("compiled %d regular expressions to the hyperscan tree",
			ncompiled);

	/*
	 * Do not send notification unless all other workers are started
	 * XXX: now we just sleep for 5 seconds to ensure that
	 */
	sleep (5);

	srv_cmd.type = RSPAMD_SRV_HYPERSCAN_LOADED;
	srv_cmd.cmd.hs_loaded.cache_dir = ctx->hs_dir;

	rspamd_srv_send_command (worker, ctx->ev_base, &srv_cmd, NULL, NULL);

	return TRUE;
}

static gboolean
rspamd_hs_helper_reload (struct rspamd_main *rspamd_main,
		struct rspamd_worker *worker, gint fd,
		struct rspamd_control_command *cmd,
		gpointer ud)
{'fraction_non_alphanumeric': 0.10483870967741936, 'fraction_numerical': 0.002932551319648094, 'mean_word_length': 3.72628843655262, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 39, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3659193', 'n_tokens_mistral': 4109, 'n_tokens_neox': 3714, 'n_words': 1120}