<commit_before>/*
 * Copyright (c) 2009-2012, <NAME>
 * All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions are met:
 *     * Redistributions of source code must retain the above copyright
 *       notice, this list of conditions and the following disclaimer.
 *     * Redistributions in binary form must reproduce the above copyright
 *       notice, this list of conditions and the following disclaimer in the
 *       documentation and/or other materials provided with the distribution.
 *
 * THIS SOFTWARE IS PROVIDED BY AUTHOR ''AS IS'' AND ANY
 * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
 * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
 * DISCLAIMED. IN NO EVENT SHALL AUTHOR BE LIABLE FOR ANY
 * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
 * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
 * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
 * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 */

/***MODULE:regexp
 * rspamd module that implements different regexp rules
 */


#include "config.h"
#include "main.h"
#include "message.h"
#include "cfg_file.h"
#include "map.h"
#include "util.h"
#include "expressions.h"
#include "view.h"
#include "lua/lua_common.h"
#include "json/jansson.h"
#include "cfg_xml.h"


#define DEFAULT_STATFILE_PREFIX "./"

struct regexp_module_item {
	struct expression               *expr;
	gchar                           *symbol;
	guint32                         avg_time;
	gpointer                        lua_function;
};

struct autolearn_data {
	gchar                           *statfile_name;
	gchar                           *symbol;
	float                           weight;
};

struct regexp_ctx {
	gint                          (*filter) (struct worker_task * task);
	GHashTable                     *autolearn_symbols;
	gchar                          *statfile_prefix;

	memory_pool_t                  *regexp_pool;
	memory_pool_t                  *dynamic_pool;
	gsize                           max_size;
	gsize							max_threads;
	GThreadPool					   *workers;
};

struct regexp_json_buf {
	gchar                          *buf;
	gchar                          *pos;
	size_t                          buflen;
	struct config_file             *cfg;
};

/* Lua regexp module for checking rspamd regexps */
LUA_FUNCTION_DEF (regexp, match);

static const struct luaL_reg    regexplib_m[] = {
	LUA_INTERFACE_DEF (regexp, match),
	{"__tostring", lua_class_tostring},
	{NULL, NULL}
};

static struct regexp_ctx       *regexp_module_ctx = NULL;
static GMutex 				   *workers_mtx = NULL;

static gint                     regexp_common_filter (struct worker_task *task);
static void				     process_regexp_item_threaded (gpointer data, gpointer user_data);
static gboolean                 rspamd_regexp_match_number (struct worker_task *task, GList * args, void *unused);
static gboolean                 rspamd_raw_header_exists (struct worker_task *task, GList * args, void *unused);
static gboolean                 rspamd_check_smtp_data (struct worker_task *task, GList * args, void *unused);
static gboolean                 rspamd_regexp_occurs_number (struct worker_task *task, GList * args, void *unused);
static gboolean                 rspamd_content_type_is_type (struct worker_task * task, GList * args, void *unused);
static gboolean                 rspamd_content_type_is_subtype (struct worker_task *task, GList * args, void *unused);
static gboolean                 rspamd_content_type_has_param (struct worker_task * task, GList * args, void *unused);
static gboolean                 rspamd_content_type_compare_param (struct worker_task * task, GList * args, void *unused);
static gboolean                 rspamd_has_content_part (struct worker_task *task, GList * args, void *unused);
static gboolean                 rspamd_has_content_part_len (struct worker_task *task, GList * args, void *unused);
static void                    process_regexp_item (struct worker_task *task, void *user_data);


/* Initialization */
gint regexp_module_init (struct config_file *cfg, struct module_ctx **ctx);
gint regexp_module_config (struct config_file *cfg);
gint regexp_module_reconfig (struct config_file *cfg);

module_t regexp_module = {
	"regexp",
	regexp_module_init,
	regexp_module_config,
	regexp_module_reconfig
};

/* Task cache functions */
#if ((GLIB_MAJOR_VERSION == 2) && (GLIB_MINOR_VERSION <= 30))
static GStaticMutex task_cache_mtx =  G_STATIC_MUTEX_INIT;
#else
G_LOCK_DEFINE (task_cache_mtx);
#endif

void
task_cache_add (struct worker_task *task, struct rspamd_regexp *re, gint32 result)
{
	if (result == 0) {
		result = -1;
	}
	/* Avoid concurrenting inserting of results */
#if ((GLIB_MAJOR_VERSION == 2) && (GLIB_MINOR_VERSION <= 30))
	g_static_mutex_lock (&task_cache_mtx);
#else
	G_LOCK (task_cache_mtx);
#endif
	g_hash_table_insert (task->re_cache, re->regexp_text, GINT_TO_POINTER (result));
#if ((GLIB_MAJOR_VERSION == 2) && (GLIB_MINOR_VERSION <= 30))
	g_static_mutex_unlock (&task_cache_mtx);
#else
	G_UNLOCK (task_cache_mtx);
#endif
}

gint32
task_cache_check (struct worker_task *task, struct rspamd_regexp *re)
{
	gpointer                        res;
	gint32                          r;

#if ((GLIB_MAJOR_VERSION == 2) && (GLIB_MINOR_VERSION <= 30))
	g_static_mutex_lock (&task_cache_mtx);
#else
	G_LOCK (task_cache_mtx);
#endif
	if ((res = g_hash_table_lookup (task->re_cache, re->regexp_text)) != NULL) {
		r = GPOINTER_TO_INT (res);
#if ((GLIB_MAJOR_VERSION == 2) && (GLIB_MINOR_VERSION <= 30))
		g_static_mutex_unlock (&task_cache_mtx);
#else
		G_UNLOCK (task_cache_mtx);
#endif
		if (r == -1) {
			return 0;
		}
		return 1;
	}
#if ((GLIB_MAJOR_VERSION == 2) && (GLIB_MINOR_VERSION <= 30))
	g_static_mutex_unlock (&task_cache_mtx);
#else
	G_UNLOCK (task_cache_mtx);
#endif
	return -1;
}


static gint
luaopen_regexp (lua_State * L)
{
	luaL_register (L, "rspamd_regexp", regexplib_m);

	return 1;
}

static void 
regexp_dynamic_insert_result (struct worker_task *task, void *user_data)
{
	gchar                           *symbol = user_data;
		
	insert_result (task, symbol, 1, NULL);
}

/*
 * Utility functions for matching exact number of regexps
 */
typedef gboolean (*int_compare_func) (gint a, gint b);
static gboolean
op_equal (gint a, gint b)
{
	return a == b;
}
static gboolean
op_more (gint a, gint b)
{
	return a > b;
}
static gboolean
op_less (gint a, gint b)
{
	return a < b;
}
static gboolean
op_more_equal (gint a, gint b)
{
	return a >= b;
}
static gboolean
op_less_equal (gint a, gint b)
{
	return a <= b;
}

/*
 * Process ip and mask of dynamic regexp
 */
static gboolean
parse_regexp_ipmask (const gchar *begin, struct dynamic_map_item *addr)
{
	const gchar                    *pos;
	gchar                           ip_buf[sizeof ("255.255.255.255")], mask_buf[3], *p;
	gint                            state = 0, dots = 0;
	
	bzero (ip_buf, sizeof (ip_buf));
	bzero (mask_buf, sizeof (mask_buf));
	pos = begin;
	p = ip_buf;

	if (*pos == '!') {
		addr->negative = TRUE;
		pos ++;
	}
	else {
		addr->negative = FALSE;
	}

	while (*pos) {
		switch (state) {
			case 0:
				state = 1;
				p = ip_buf;
				dots = 0;
				break;
			case 1:
				/* Begin parse ip */
				if (p - ip_buf >= (gint)sizeof (ip_buf) || dots > 3) {
					return FALSE;
				}
				if (g_ascii_isdigit (*pos)) {
					*p ++ = *pos ++;
				}
				else if (*pos == '.') {
					*p ++ = *pos ++;
					dots ++;
				}
				else if (*pos == '/') {
					pos ++;
					p = mask_buf;
					state = 2;
				}
				else {
					/* Invalid character */
					return FALSE;
				}
				break;
			case 2:
				/* Parse mask */
				if (p - mask_buf > 2) {
					return FALSE;
				}
				if (g_ascii_isdigit (*pos)) {
					*p ++ = *pos ++;
				}
				else {
					return FALSE;
				}
				break;
		}
	}

	if (!inet_aton (ip_buf, &addr->addr)) {
		return FALSE;
	}
	if (state == 2) {
		/* Also parse mask */
		addr->mask = (mask_buf[0] - '0') * 10 + mask_buf[1] - '0';
		if (addr->mask > 32) {
			msg_info ("bad ipmask value: '%s'", begin);
			return FALSE;
		}
	}
	else {
		addr->mask = 32;
	}

	return TRUE;

}

/* Process regexp expression */
static                          gboolean
read_regexp_expression (memory_pool_t * pool, struct regexp_module_item *chain, gchar *symbol, gchar *line, gboolean raw_mode)
{
	struct expression              *e, *cur;

	e = parse_expression (pool, line);
	if (e == NULL) {
		msg_warn ("%s = \"%s\" is invalid regexp expression", symbol, line);
		return FALSE;
	}
	chain->expr = e;
	cur = e;
	while (cur) {
		if (cur->type == EXPR_REGEXP) {
			cur->content.operand = parse_regexp (pool, cur->content.operand, raw_mode);
			if (cur->content.operand == NULL) {
				msg_warn ("cannot parse regexp, skip expression %s = \"%s\"", symbol, line);
				return FALSE;
			}
			cur->type = EXPR_REGEXP_PARSED;
		}
		cur = cur->next;
	}

	return TRUE;
}


/* Callbacks for reading json dynamic rules */
gchar                         *
json_regexp_read_cb (memory_pool_t * pool, gchar * chunk, gint len, struct map_cb_data *data)
{
	struct regexp_json_buf                *jb;
	gint                            free, off;

	if (data->cur_data == NULL) {
		jb = g_malloc (sizeof (struct regexp_json_buf));
		jb->cfg = ((struct regexp_json_buf *)data->prev_data)->cfg;
		jb->buf = NULL;
		jb->pos = NULL;
		data->cur_data = jb;
	}
	else {
		jb = data->cur_data;
	}

	if (jb->buf == NULL) {
		/* Allocate memory for buffer */
		jb->buflen = len * 2;
		jb->buf = g_malloc (jb->buflen);
		jb->pos = jb->buf;
	}

	off = jb->pos - jb->buf;
	free = jb->buflen - off;

	if (free < len) {
		jb->buflen = MAX (jb->buflen * 2, jb->buflen + len * 2);
		jb->buf = g_realloc (jb->buf, jb->buflen);
		jb->pos = jb->buf + off;
	}

	memcpy (jb->pos, chunk, len);
	jb->pos += len;

	/* Say not to copy any part of this buffer */
	return NULL;
}

void
json_regexp_fin_cb (memory_pool_t * pool, struct map_cb_data *data)
{
	struct regexp_json_buf         *jb;
	guint                           nelts, i, j;
	json_t                         *js, *cur_elt, *cur_nm, *it_val;
	json_error_t                    je;
	gchar                           *cur_rule, *cur_symbol;
	double                          score;
	struct regexp_module_item      *cur_item;
	GList                          *cur_networks = NULL;
	struct dynamic_map_item        *cur_nitem;
	memory_pool_t                  *new_pool;

	if (data->prev_data) {
		jb = data->prev_data;
		/* Clean prev data */
		if (jb->buf) {
			g_free (jb->buf);
		}
		g_free (jb);
	}

	/* Now parse json */
	if (data->cur_data) {
		jb = data->cur_data;
	}
	else {
		msg_err ("no data read");
		return;
	}
	if (jb->buf == NULL) {
		msg_err ("no data read");
		return;
	}
	/* NULL terminate current buf */
	*jb->pos = '\0';

	js = json_loads (jb->buf, &je);
	if (!js) {
		msg_err ("cannot load json data: parse error %s, on line %d", je.text, je.line);
		return;
	}

	if (!json_is_array (js)) {
		json_decref (js);
		msg_err ("loaded json is not an array");
		return;
	}
	
	new_pool = memory_pool_new (memory_pool_get_size ());
		
	remove_dynamic_rules (jb->cfg->cache);
	if (regexp_module_ctx->dynamic_pool != NULL) {
		memory_pool_delete (regexp_module_ctx->dynamic_pool);
	}
	regexp_module_ctx->dynamic_pool = new_pool;

	nelts = json_array_size (js);
	for (i = 0; i < nelts; i++) {
		cur_networks = NULL;
		cur_rule = NULL;

		cur_elt = json_array_get (js, i);
		if (!cur_elt || !json_is_object (cur_elt)) {
			msg_err ("loaded json is not an object");
			continue;
		}
		/* Factor param */
		cur_nm = json_object_get (cur_elt, "factor");
		if (cur_nm == NULL || !json_is_number (cur_nm)) {
			msg_err ("factor is not a number or not exists, but is required");
			continue;
		}
		score = json_number_value (cur_nm); 
		/* Symbol param */
		cur_nm = json_object_get (cur_elt, "symbol");
		if (cur_nm == NULL || !json_is_string (cur_nm)) {
			msg_err ("symbol is not a string or not exists, but is required");
			continue;
		}
		cur_symbol = memory_pool_strdup (new_pool, json_string_value (cur_nm)); 
		/* Enabled flag */
		cur_nm = json_object_get (cur_elt, "enabled");
		if (cur_nm != NULL && json_is_boolean (cur_nm)) {
			if (json_is_false (cur_nm)) {
				msg_info ("rule %s is disabled in json", cur_symbol);
				continue;
			}
		}
		/* Now check other settings */
		/* Rule */
		cur_nm = json_object_get (cur_elt, "rule");
		if (cur_nm != NULL && json_is_string (cur_nm)) {
			cur_rule = memory_pool_strdup (new_pool, json_string_value (cur_nm));
		}
		/* Networks array */
		cur_nm = json_object_get (cur_elt, "networks");
		if (cur_nm != NULL && json_is_array (cur_nm)) {
			for (j = 0; j < json_array_size (cur_nm); j++) {
				it_val = json_array_get (cur_nm, i);
				if (it_val && json_is_string (it_val)) {
					cur_nitem = memory_pool_alloc (new_pool, sizeof (struct dynamic_map_item));
					if (parse_regexp_ipmask (json_string_value (it_val), cur_nitem)) {
						cur_networks = g_list_prepend (cur_networks, cur_nitem);
					}
				}
			}
		}
		if (cur_rule) {
			/* Dynamic rule has rule option */
			cur_item = memory_pool_alloc0 (new_pool, sizeof (struct regexp_module_item));
			cur_item->symbol = cur_symbol;
			if (read_regexp_expression (new_pool, cur_item, cur_symbol, cur_rule, jb->cfg->raw_mode)) {
				register_dynamic_symbol (new_pool, &jb->cfg->cache, cur_symbol, score, process_regexp_item, cur_item, cur_networks);
			}
			else {
				msg_warn ("cannot parse dynamic rule");
			}
		}
		else {
			/* Just rule that is allways true (for whitelisting for example) */
			register_dynamic_symbol (new_pool, &jb->cfg->cache, cur_symbol, score, regexp_dynamic_insert_result, cur_symbol, cur_networks);
		}
		if (cur_networks) {
			g_list_free (cur_networks);
		}
	}
	json_decref (js);
}

/* Init function */
gint
regexp_module_init (struct config_file *cfg, struct module_ctx **ctx)
{
	regexp_module_ctx = g_malloc (sizeof (struct regexp_ctx));

	regexp_module_ctx->filter = regexp_common_filter;
	regexp_module_ctx->regexp_pool = memory_pool_new (memory_pool_get_size ());
	regexp_module_ctx->dynamic_pool = NULL;
	regexp_module_ctx->autolearn_symbols = g_hash_table_new (rspamd_str_hash, rspamd_str_equal);
	regexp_module_ctx->workers = NULL;

	*ctx = (struct module_ctx *)regexp_module_ctx;
	register_expression_function ("regexp_match_number", rspamd_regexp_match_number, NULL);
	register_expression_function ("regexp_occurs_number", rspamd_regexp_occurs_number, NULL);
	register_expression_function ("raw_header_exists", rspamd_raw_header_exists, NULL);
	register_expression_function ("check_smtp_data", rspamd_check_smtp_data, NULL);
	register_expression_function ("content_type_is_type", rspamd_content_type_is_type, NULL);
	register_expression_function ("content_type_is_subtype", rspamd_content_type_is_subtype, NULL);
	register_expression_function ("content_type_has_param", rspamd_content_type_has_param, NULL);
	register_expression_function ("content_type_compare_param", rspamd_content_type_compare_param, NULL);
	register_expression_function ("has_content_part", rspamd_has_content_part, NULL);
	register_expression_function ("has_content_part_len", rspamd_has_content_part_len, NULL);

	(void)luaopen_regexp (cfg->lua_state);
	register_module_opt ("regexp", "dynamic_rules", MODULE_OPT_TYPE_STRING);
	register_module_opt ("regexp", "max_size", MODULE_OPT_TYPE_SIZE);
	register_module_opt ("regexp", "max_threads", MODULE_OPT_TYPE_SIZE);
	register_module_opt ("regexp", "/^\\S+$/", MODULE_OPT_TYPE_STRING);

	return 0;
}


/* 
 * Parse string in format:
 * SYMBOL:statfile:weight
 */
void
parse_autolearn_param (const gchar *param, const gchar *value, struct config_file *cfg)
{
	struct autolearn_data          *d;
	gchar                           *p;

	p = memory_pool_strdup (regexp_module_ctx->regexp_pool, value);
	d = memory_pool_alloc (regexp_module_ctx->regexp_pool, sizeof (struct autolearn_data));

	d->symbol = strsep (&p, ":");
	if (d->symbol) {
		d->statfile_name = strsep (&p, ":");
		if (d->statfile_name) {
			if (p != NULL && *p != '\0') {
				d->weight = strtod (p, NULL);
				g_hash_table_insert (regexp_module_ctx->autolearn_symbols, d->symbol, d);
			}
		}
		else {
			msg_warn ("cannot extract statfile name from %s", p);
		}
	}
	else {
		msg_warn ("cannot extract symbol name from %s", p);
	}
}

gint
regexp_module_config (struct config_file *cfg)
{
	GList                          *cur_opt = NULL;
	struct module_opt              *cur;
	struct regexp_module_item      *cur_item;
	gchar                          *value;
	gint                            res = TRUE;
	struct regexp_json_buf         *jb, **pjb;

	if ((value = get_module_opt (cfg, "regexp", "statfile_prefix")) != NULL) {
		regexp_module_ctx->statfile_prefix = memory_pool_strdup (regexp_module_ctx->regexp_pool, value);
	}
	else {
		regexp_module_ctx->statfile_prefix = DEFAULT_STATFILE_PREFIX;
	}
	if ((value = get_module_opt (cfg, "regexp", "max_size")) != NULL) {
		regexp_module_ctx->max_size = parse_limit (value, -1);
	}
	else {
		regexp_module_ctx->max_size = 0;
	}
	if ((value = get_module_opt (cfg, "regexp", "max_threads")) != NULL) {
		if (g_thread_supported ()) {
			regexp_module_ctx->max_threads = parse_limit (value, -1);
		}
		else {
			regexp_module_ctx->max_threads = 0;
		}
		regexp_module_ctx->workers = NULL;
	}
	else {
		regexp_module_ctx->max_threads = 0;
		regexp_module_ctx->workers = NULL;
	}
	if ((value = get_module_opt (cfg, "regexp", "dynamic_rules")) != NULL) {
		jb = g_malloc (sizeof (struct regexp_json_buf));
		pjb = g_malloc (sizeof (struct regexp_json_buf *));
		jb->buf = NULL;
		jb->cfg = cfg;
		*pjb = jb;
		if (!add_map (cfg, value, "Dynamic regexp rules", json_regexp_read_cb, json_regexp_fin_cb, (void **)pjb)) {
			msg_err ("cannot add map %s", value);
		}
	}


	cur_opt = g_hash_table_lookup (cfg->modules_opts, "regexp");
	while (cur_opt) {
		cur = cur_opt->data;
		/* Skip several options that are not regexp */
		if (g_ascii_strncasecmp (cur->param, "autolearn", sizeof ("autolearn") - 1) == 0) {
			parse_autolearn_param (cur->param, cur->value, cfg);
			cur_opt = g_list_next (cur_opt);
			continue;
		}
		else if (g_ascii_strncasecmp (cur->param, "dynamic_rules", sizeof ("dynamic_rules") - 1) == 0) {
			cur_opt = g_list_next (cur_opt);
			continue;
		}
		else if (g_ascii_strncasecmp (cur->param, "max_size", sizeof ("max_size") - 1) == 0) {
			cur_opt = g_list_next (cur_opt);
			continue;
		}
		else if (g_ascii_strncasecmp (cur->param, "max_threads", sizeof ("max_threads") - 1) == 0) {
			cur_opt = g_list_next (cur_opt);
			continue;
		}
		/* Handle regexps */
		cur_item = memory_pool_alloc0 (regexp_module_ctx->regexp_pool, sizeof (struct regexp_module_item));
		cur_item->symbol = cur->param;
		if (cur->is_lua && cur->lua_type == LUA_VAR_STRING) {
			if (!read_regexp_expression (regexp_module_ctx->regexp_pool, cur_item, cur->param, cur->actual_data, cfg->raw_mode)) {
				res = FALSE;
			}
		}
		else if (cur->is_lua && cur->lua_type == LUA_VAR_FUNCTION) {
			cur_item->lua_function = cur->actual_data;
		}
		else if (! cur->is_lua) {
			if (!read_regexp_expression (regexp_module_ctx->regexp_pool, cur_item, cur->param, cur->value, cfg->raw_mode)) {
				res = FALSE;
			}
		}
		else {
			msg_err ("unknown variable type for %s", cur->param);
			res = FALSE;
		}
		
		if ( !res) {
			/* Stop on errors */
			break;
		}
		
		register_symbol (&cfg->cache, cur->param, 1, process_regexp_item, cur_item);

		cur_opt = g_list_next (cur_opt);
	}


	return res;
}

gint
regexp_module_reconfig (struct config_file *cfg)
{
	memory_pool_delete (regexp_module_ctx->regexp_pool);
	regexp_module_ctx->regexp_pool = memory_pool_new (memory_pool_get_size ());

	return regexp_module_config (cfg);
}

struct url_regexp_param {
	struct worker_task             *task;
	GRegex                         *regexp;
	struct rspamd_regexp           *re;
	gboolean                        found;
};

static                          gboolean
tree_url_callback (gpointer key, gpointer value, void *data)
{<commit_msg>Fix regexp plugin.<commit_after>/*
 * Copyright (c) 2009-2012, <NAME>
 * All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions are met:
 *     * Redistributions of source code must retain the above copyright
 *       notice, this list of conditions and the following disclaimer.
 *     * Redistributions in binary form must reproduce the above copyright
 *       notice, this list of conditions and the following disclaimer in the
 *       documentation and/or other materials provided with the distribution.
 *
 * THIS SOFTWARE IS PROVIDED BY AUTHOR ''AS IS'' AND ANY
 * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
 * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
 * DISCLAIMED. IN NO EVENT SHALL AUTHOR BE LIABLE FOR ANY
 * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
 * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
 * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
 * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 */

/***MODULE:regexp
 * rspamd module that implements different regexp rules
 */


#include "config.h"
#include "main.h"
#include "message.h"
#include "cfg_file.h"
#include "map.h"
#include "util.h"
#include "expressions.h"
#include "view.h"
#include "lua/lua_common.h"
#include "json/jansson.h"
#include "cfg_xml.h"


#define DEFAULT_STATFILE_PREFIX "./"

struct regexp_module_item {
	struct expression               *expr;
	gchar                           *symbol;
	guint32                         avg_time;
	gpointer                        lua_function;
};

struct autolearn_data {
	gchar                           *statfile_name;
	gchar                           *symbol;
	float                           weight;
};

struct regexp_ctx {
	gint                          (*filter) (struct worker_task * task);
	GHashTable                     *autolearn_symbols;
	gchar                          *statfile_prefix;

	memory_pool_t                  *regexp_pool;
	memory_pool_t                  *dynamic_pool;
	gsize                           max_size;
	gsize							max_threads;
	GThreadPool					   *workers;
};

struct regexp_json_buf {
	gchar                          *buf;
	gchar                          *pos;
	size_t                          buflen;
	struct config_file             *cfg;
};

/* Lua regexp module for checking rspamd regexps */
LUA_FUNCTION_DEF (regexp, match);

static const struct luaL_reg    regexplib_m[] = {
	LUA_INTERFACE_DEF (regexp, match),
	{"__tostring", lua_class_tostring},
	{NULL, NULL}
};

static struct regexp_ctx       *regexp_module_ctx = NULL;
static GMutex 				   *workers_mtx = NULL;

static gint                     regexp_common_filter (struct worker_task *task);
static void				     process_regexp_item_threaded (gpointer data, gpointer user_data);
static gboolean                 rspamd_regexp_match_number (struct worker_task *task, GList * args, void *unused);
static gboolean                 rspamd_raw_header_exists (struct worker_task *task, GList * args, void *unused);
static gboolean                 rspamd_check_smtp_data (struct worker_task *task, GList * args, void *unused);
static gboolean                 rspamd_regexp_occurs_number (struct worker_task *task, GList * args, void *unused);
static gboolean                 rspamd_content_type_is_type (struct worker_task * task, GList * args, void *unused);
static gboolean                 rspamd_content_type_is_subtype (struct worker_task *task, GList * args, void *unused);
static gboolean                 rspamd_content_type_has_param (struct worker_task * task, GList * args, void *unused);
static gboolean                 rspamd_content_type_compare_param (struct worker_task * task, GList * args, void *unused);
static gboolean                 rspamd_has_content_part (struct worker_task *task, GList * args, void *unused);
static gboolean                 rspamd_has_content_part_len (struct worker_task *task, GList * args, void *unused);
static void                    process_regexp_item (struct worker_task *task, void *user_data);


/* Initialization */
gint regexp_module_init (struct config_file *cfg, struct module_ctx **ctx);
gint regexp_module_config (struct config_file *cfg);
gint regexp_module_reconfig (struct config_file *cfg);

module_t regexp_module = {
	"regexp",
	regexp_module_init,
	regexp_module_config,
	regexp_module_reconfig
};

/* Task cache functions */
#if ((GLIB_MAJOR_VERSION == 2) && (GLIB_MINOR_VERSION <= 30))
static GStaticMutex task_cache_mtx =  G_STATIC_MUTEX_INIT;
#else
G_LOCK_DEFINE (task_cache_mtx);
#endif

void
task_cache_add (struct worker_task *task, struct rspamd_regexp *re, gint32 result)
{
	if (result == 0) {
		result = -1;
	}
	/* Avoid concurrenting inserting of results */
#if ((GLIB_MAJOR_VERSION == 2) && (GLIB_MINOR_VERSION <= 30))
	g_static_mutex_lock (&task_cache_mtx);
#else
	G_LOCK (task_cache_mtx);
#endif
	g_hash_table_insert (task->re_cache, re->regexp_text, GINT_TO_POINTER (result));
#if ((GLIB_MAJOR_VERSION == 2) && (GLIB_MINOR_VERSION <= 30))
	g_static_mutex_unlock (&task_cache_mtx);
#else
	G_UNLOCK (task_cache_mtx);
#endif
}

gint32
task_cache_check (struct worker_task *task, struct rspamd_regexp *re)
{
	gpointer                        res;
	gint32                          r;

#if ((GLIB_MAJOR_VERSION == 2) && (GLIB_MINOR_VERSION <= 30))
	g_static_mutex_lock (&task_cache_mtx);
#else
	G_LOCK (task_cache_mtx);
#endif
	if ((res = g_hash_table_lookup (task->re_cache, re->regexp_text)) != NULL) {
		r = GPOINTER_TO_INT (res);
#if ((GLIB_MAJOR_VERSION == 2) && (GLIB_MINOR_VERSION <= 30))
		g_static_mutex_unlock (&task_cache_mtx);
#else
		G_UNLOCK (task_cache_mtx);
#endif
		if (r == -1) {
			return 0;
		}
		return 1;
	}
#if ((GLIB_MAJOR_VERSION == 2) && (GLIB_MINOR_VERSION <= 30))
	g_static_mutex_unlock (&task_cache_mtx);
#else
	G_UNLOCK (task_cache_mtx);
#endif
	return -1;
}


static gint
luaopen_regexp (lua_State * L)
{
	luaL_register (L, "rspamd_regexp", regexplib_m);

	return 1;
}

static void 
regexp_dynamic_insert_result (struct worker_task *task, void *user_data)
{
	gchar                           *symbol = user_data;
		
	insert_result (task, symbol, 1, NULL);
}

/*
 * Utility functions for matching exact number of regexps
 */
typedef gboolean (*int_compare_func) (gint a, gint b);
static gboolean
op_equal (gint a, gint b)
{
	return a == b;
}
static gboolean
op_more (gint a, gint b)
{
	return a > b;
}
static gboolean
op_less (gint a, gint b)
{
	return a < b;
}
static gboolean
op_more_equal (gint a, gint b)
{
	return a >= b;
}
static gboolean
op_less_equal (gint a, gint b)
{
	return a <= b;
}

/*
 * Process ip and mask of dynamic regexp
 */
static gboolean
parse_regexp_ipmask (const gchar *begin, struct dynamic_map_item *addr)
{
	const gchar                    *pos;
	gchar                           ip_buf[sizeof ("255.255.255.255")], mask_buf[3], *p;
	gint                            state = 0, dots = 0;
	
	bzero (ip_buf, sizeof (ip_buf));
	bzero (mask_buf, sizeof (mask_buf));
	pos = begin;
	p = ip_buf;

	if (*pos == '!') {
		addr->negative = TRUE;
		pos ++;
	}
	else {
		addr->negative = FALSE;
	}

	while (*pos) {
		switch (state) {
			case 0:
				state = 1;
				p = ip_buf;
				dots = 0;
				break;
			case 1:
				/* Begin parse ip */
				if (p - ip_buf >= (gint)sizeof (ip_buf) || dots > 3) {
					return FALSE;
				}
				if (g_ascii_isdigit (*pos)) {
					*p ++ = *pos ++;
				}
				else if (*pos == '.') {
					*p ++ = *pos ++;
					dots ++;
				}
				else if (*pos == '/') {
					pos ++;
					p = mask_buf;
					state = 2;
				}
				else {
					/* Invalid character */
					return FALSE;
				}
				break;
			case 2:
				/* Parse mask */
				if (p - mask_buf > 2) {
					return FALSE;
				}
				if (g_ascii_isdigit (*pos)) {
					*p ++ = *pos ++;
				}
				else {
					return FALSE;
				}
				break;
		}
	}

	if (!inet_aton (ip_buf, &addr->addr)) {
		return FALSE;
	}
	if (state == 2) {
		/* Also parse mask */
		addr->mask = (mask_buf[0] - '0') * 10 + mask_buf[1] - '0';
		if (addr->mask > 32) {
			msg_info ("bad ipmask value: '%s'", begin);
			return FALSE;
		}
	}
	else {
		addr->mask = 32;
	}

	return TRUE;

}

/* Process regexp expression */
static                          gboolean
read_regexp_expression (memory_pool_t * pool, struct regexp_module_item *chain,
		const gchar *symbol, const gchar *line, gboolean raw_mode)
{
	struct expression              *e, *cur;

	e = parse_expression (pool, (gchar *)line);
	if (e == NULL) {
		msg_warn ("%s = \"%s\" is invalid regexp expression", symbol, line);
		return FALSE;
	}
	chain->expr = e;
	cur = e;
	while (cur) {
		if (cur->type == EXPR_REGEXP) {
			cur->content.operand = parse_regexp (pool, cur->content.operand, raw_mode);
			if (cur->content.operand == NULL) {
				msg_warn ("cannot parse regexp, skip expression %s = \"%s\"", symbol, line);
				return FALSE;
			}
			cur->type = EXPR_REGEXP_PARSED;
		}
		cur = cur->next;
	}

	return TRUE;
}


/* Callbacks for reading json dynamic rules */
gchar                         *
json_regexp_read_cb (memory_pool_t * pool, gchar * chunk, gint len, struct map_cb_data *data)
{
	struct regexp_json_buf                *jb;
	gint                            free, off;

	if (data->cur_data == NULL) {
		jb = g_malloc (sizeof (struct regexp_json_buf));
		jb->cfg = ((struct regexp_json_buf *)data->prev_data)->cfg;
		jb->buf = NULL;
		jb->pos = NULL;
		data->cur_data = jb;
	}
	else {
		jb = data->cur_data;
	}

	if (jb->buf == NULL) {
		/* Allocate memory for buffer */
		jb->buflen = len * 2;
		jb->buf = g_malloc (jb->buflen);
		jb->pos = jb->buf;
	}

	off = jb->pos - jb->buf;
	free = jb->buflen - off;

	if (free < len) {
		jb->buflen = MAX (jb->buflen * 2, jb->buflen + len * 2);
		jb->buf = g_realloc (jb->buf, jb->buflen);
		jb->pos = jb->buf + off;
	}

	memcpy (jb->pos, chunk, len);
	jb->pos += len;

	/* Say not to copy any part of this buffer */
	return NULL;
}

void
json_regexp_fin_cb (memory_pool_t * pool, struct map_cb_data *data)
{
	struct regexp_json_buf         *jb;
	guint                           nelts, i, j;
	json_t                         *js, *cur_elt, *cur_nm, *it_val;
	json_error_t                    je;
	gchar                           *cur_rule, *cur_symbol;
	double                          score;
	struct regexp_module_item      *cur_item;
	GList                          *cur_networks = NULL;
	struct dynamic_map_item        *cur_nitem;
	memory_pool_t                  *new_pool;

	if (data->prev_data) {
		jb = data->prev_data;
		/* Clean prev data */
		if (jb->buf) {
			g_free (jb->buf);
		}
		g_free (jb);
	}

	/* Now parse json */
	if (data->cur_data) {
		jb = data->cur_data;
	}
	else {
		msg_err ("no data read");
		return;
	}
	if (jb->buf == NULL) {
		msg_err ("no data read");
		return;
	}
	/* NULL terminate current buf */
	*jb->pos = '\0';

	js = json_loads (jb->buf, &je);
	if (!js) {
		msg_err ("cannot load json data: parse error %s, on line %d", je.text, je.line);
		return;
	}

	if (!json_is_array (js)) {
		json_decref (js);
		msg_err ("loaded json is not an array");
		return;
	}
	
	new_pool = memory_pool_new (memory_pool_get_size ());
		
	remove_dynamic_rules (jb->cfg->cache);
	if (regexp_module_ctx->dynamic_pool != NULL) {
		memory_pool_delete (regexp_module_ctx->dynamic_pool);
	}
	regexp_module_ctx->dynamic_pool = new_pool;

	nelts = json_array_size (js);
	for (i = 0; i < nelts; i++) {
		cur_networks = NULL;
		cur_rule = NULL;

		cur_elt = json_array_get (js, i);
		if (!cur_elt || !json_is_object (cur_elt)) {
			msg_err ("loaded json is not an object");
			continue;
		}
		/* Factor param */
		cur_nm = json_object_get (cur_elt, "factor");
		if (cur_nm == NULL || !json_is_number (cur_nm)) {
			msg_err ("factor is not a number or not exists, but is required");
			continue;
		}
		score = json_number_value (cur_nm); 
		/* Symbol param */
		cur_nm = json_object_get (cur_elt, "symbol");
		if (cur_nm == NULL || !json_is_string (cur_nm)) {
			msg_err ("symbol is not a string or not exists, but is required");
			continue;
		}
		cur_symbol = memory_pool_strdup (new_pool, json_string_value (cur_nm)); 
		/* Enabled flag */
		cur_nm = json_object_get (cur_elt, "enabled");
		if (cur_nm != NULL && json_is_boolean (cur_nm)) {
			if (json_is_false (cur_nm)) {
				msg_info ("rule %s is disabled in json", cur_symbol);
				continue;
			}
		}
		/* Now check other settings */
		/* Rule */
		cur_nm = json_object_get (cur_elt, "rule");
		if (cur_nm != NULL && json_is_string (cur_nm)) {
			cur_rule = memory_pool_strdup (new_pool, json_string_value (cur_nm));
		}
		/* Networks array */
		cur_nm = json_object_get (cur_elt, "networks");
		if (cur_nm != NULL && json_is_array (cur_nm)) {
			for (j = 0; j < json_array_size (cur_nm); j++) {
				it_val = json_array_get (cur_nm, i);
				if (it_val && json_is_string (it_val)) {
					cur_nitem = memory_pool_alloc (new_pool, sizeof (struct dynamic_map_item));
					if (parse_regexp_ipmask (json_string_value (it_val), cur_nitem)) {
						cur_networks = g_list_prepend (cur_networks, cur_nitem);
					}
				}
			}
		}
		if (cur_rule) {
			/* Dynamic rule has rule option */
			cur_item = memory_pool_alloc0 (new_pool, sizeof (struct regexp_module_item));
			cur_item->symbol = cur_symbol;
			if (read_regexp_expression (new_pool, cur_item, cur_symbol, cur_rule, jb->cfg->raw_mode)) {
				register_dynamic_symbol (new_pool, &jb->cfg->cache, cur_symbol, score, process_regexp_item, cur_item, cur_networks);
			}
			else {
				msg_warn ("cannot parse dynamic rule");
			}
		}
		else {
			/* Just rule that is allways true (for whitelisting for example) */
			register_dynamic_symbol (new_pool, &jb->cfg->cache, cur_symbol, score, regexp_dynamic_insert_result, cur_symbol, cur_networks);
		}
		if (cur_networks) {
			g_list_free (cur_networks);
		}
	}
	json_decref (js);
}

/* Init function */
gint
regexp_module_init (struct config_file *cfg, struct module_ctx **ctx)
{
	regexp_module_ctx = g_malloc (sizeof (struct regexp_ctx));

	regexp_module_ctx->filter = regexp_common_filter;
	regexp_module_ctx->regexp_pool = memory_pool_new (memory_pool_get_size ());
	regexp_module_ctx->dynamic_pool = NULL;
	regexp_module_ctx->autolearn_symbols = g_hash_table_new (rspamd_str_hash, rspamd_str_equal);
	regexp_module_ctx->workers = NULL;

	*ctx = (struct module_ctx *)regexp_module_ctx;
	register_expression_function ("regexp_match_number", rspamd_regexp_match_number, NULL);
	register_expression_function ("regexp_occurs_number", rspamd_regexp_occurs_number, NULL);
	register_expression_function ("raw_header_exists", rspamd_raw_header_exists, NULL);
	register_expression_function ("check_smtp_data", rspamd_check_smtp_data, NULL);
	register_expression_function ("content_type_is_type", rspamd_content_type_is_type, NULL);
	register_expression_function ("content_type_is_subtype", rspamd_content_type_is_subtype, NULL);
	register_expression_function ("content_type_has_param", rspamd_content_type_has_param, NULL);
	register_expression_function ("content_type_compare_param", rspamd_content_type_compare_param, NULL);
	register_expression_function ("has_content_part", rspamd_has_content_part, NULL);
	register_expression_function ("has_content_part_len", rspamd_has_content_part_len, NULL);

	(void)luaopen_regexp (cfg->lua_state);
	register_module_opt ("regexp", "dynamic_rules", MODULE_OPT_TYPE_STRING);
	register_module_opt ("regexp", "max_size", MODULE_OPT_TYPE_SIZE);
	register_module_opt ("regexp", "max_threads", MODULE_OPT_TYPE_SIZE);
	register_module_opt ("regexp", "/^\\S+$/", MODULE_OPT_TYPE_STRING);

	return 0;
}


/* 
 * Parse string in format:
 * SYMBOL:statfile:weight
 */
void
parse_autolearn_param (const gchar *param, const gchar *value, struct config_file *cfg)
{
	struct autolearn_data          *d;
	gchar                           *p;

	p = memory_pool_strdup (regexp_module_ctx->regexp_pool, value);
	d = memory_pool_alloc (regexp_module_ctx->regexp_pool, sizeof (struct autolearn_data));

	d->symbol = strsep (&p, ":");
	if (d->symbol) {
		d->statfile_name = strsep (&p, ":");
		if (d->statfile_name) {
			if (p != NULL && *p != '\0') {
				d->weight = strtod (p, NULL);
				g_hash_table_insert (regexp_module_ctx->autolearn_symbols, d->symbol, d);
			}
		}
		else {
			msg_warn ("cannot extract statfile name from %s", p);
		}
	}
	else {
		msg_warn ("cannot extract symbol name from %s", p);
	}
}

gint
regexp_module_config (struct config_file *cfg)
{
	struct regexp_module_item      *cur_item;
	rspamd_cl_object_t              *sec, *value, *tmp;
	gint                            res = TRUE;
	struct regexp_json_buf         *jb, **pjb;


	HASH_FIND_STR (cfg->rcl_obj, "regexp", sec);
	if (sec == NULL) {
		msg_err ("regexp module enabled, but no rules are defined");
		return TRUE;
	}

	regexp_module_ctx->max_size = 0;
	regexp_module_ctx->max_threads = 0;
	regexp_module_ctx->workers = NULL;

	HASH_ITER (hh, sec, value, tmp) {
		if (g_ascii_strncasecmp (value->key, "autolearn", sizeof ("autolearn") - 1) == 0) {
			parse_autolearn_param (value->key, rspamd_cl_obj_tostring (value), cfg);
		}
		else if (g_ascii_strncasecmp (value->key, "dynamic_rules", sizeof ("dynamic_rules") - 1) == 0) {
			jb = g_malloc (sizeof (struct regexp_json_buf));
			pjb = g_malloc (sizeof (struct regexp_json_buf *));
			jb->buf = NULL;
			jb->cfg = cfg;
			*pjb = jb;
			if (!add_map (cfg, rspamd_cl_obj_tostring (value),
					"Dynamic regexp rules", json_regexp_read_cb, json_regexp_fin_cb,
					(void **)pjb)) {
				msg_err ("cannot add map %s", rspamd_cl_obj_tostring (value));
			}
		}
		else if (g_ascii_strncasecmp (value->key, "max_size", sizeof ("max_size") - 1) == 0) {
			regexp_module_ctx->max_size = rspamd_cl_obj_toint (value);
		}
		else if (g_ascii_strncasecmp (value->key, "max_threads", sizeof ("max_threads") - 1) == 0) {
			regexp_module_ctx->max_threads = rspamd_cl_obj_toint (value);
		}
		else if (value->type == RSPAMD_CL_STRING) {
			cur_item = memory_pool_alloc0 (regexp_module_ctx->regexp_pool, sizeof (struct regexp_module_item));
			cur_item->symbol = value->key;
			if (!read_regexp_expression (regexp_module_ctx->regexp_pool, cur_item, value->key,
					rspamd_cl_obj_tostring (value), cfg->raw_mode)) {
				res = FALSE;
			}
			register_symbol (&cfg->cache, cur_item->symbol, 1, process_regexp_item, cur_item);
		}
		else if (value->type == RSPAMD_CL_USERDATA) {
			cur_item = memory_pool_alloc0 (regexp_module_ctx->regexp_pool, sizeof (struct regexp_module_item));
			cur_item->symbol = value->key;
			cur_item->lua_function = value->value.ud;
			register_symbol (&cfg->cache, cur_item->symbol, 1, process_regexp_item, cur_item);
		}
		else {
			msg_warn ("unknown type of attribute %s for regexp module", value->key);
		}
	}

	return res;
}

gint
regexp_module_reconfig (struct config_file *cfg)
{
	memory_pool_delete (regexp_module_ctx->regexp_pool);
	regexp_module_ctx->regexp_pool = memory_pool_new (memory_pool_get_size ());

	return regexp_module_config (cfg);
}

struct url_regexp_param {
	struct worker_task             *task;
	GRegex                         *regexp;
	struct rspamd_regexp           *re;
	gboolean                        found;
};

static                          gboolean
tree_url_callback (gpointer key, gpointer value, void *data)
{
{'fraction_non_alphanumeric': 0.11989182692307693, 'fraction_numerical': 0.004982972756410256, 'mean_word_length': 3.1999158691765697, 'pattern_counts': {'":': 4, '<': 27, '<?xml version=': 0, '>': 253, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 2}, 'pii_count': 2, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4292756', 'n_tokens_mistral': 15455, 'n_tokens_neox': 14119, 'n_words': 3534}