<commit_before>/*-
 * Copyright 2016 <NAME>
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *   http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
#include "libmime/message.h"
#include "re_cache.h"
#include "xxhash.h"
#include "cryptobox.h"
#include "ref.h"
#include "libserver/url.h"
#include "libserver/task.h"
#include "libserver/cfg_file.h"
#include "libutil/util.h"
#include "libutil/regexp.h"
#ifdef WITH_HYPERSCAN
#include "hs.h"
#include "unix-std.h"
#include <signal.h>

#ifndef WITH_PCRE2
#include <pcre.h>
#else
#include <pcre2.h>
#endif

#ifdef HAVE_SYS_WAIT_H
#include <sys/wait.h>
#endif

#endif

#define msg_err_re_cache(...) rspamd_default_log_function (G_LOG_LEVEL_CRITICAL, \
        "re_cache", cache->hash, \
        G_STRFUNC, \
        __VA_ARGS__)
#define msg_warn_re_cache(...)   rspamd_default_log_function (G_LOG_LEVEL_WARNING, \
        "re_cache", cache->hash, \
        G_STRFUNC, \
        __VA_ARGS__)
#define msg_info_re_cache(...)   rspamd_default_log_function (G_LOG_LEVEL_INFO, \
        "re_cache", cache->hash, \
        G_STRFUNC, \
        __VA_ARGS__)
#define msg_debug_re_cache(...)  rspamd_default_log_function (G_LOG_LEVEL_DEBUG, \
        "re_cache", cache->hash, \
        G_STRFUNC, \
        __VA_ARGS__)

#ifdef WITH_HYPERSCAN
#define RSPAMD_HS_MAGIC_LEN (sizeof (rspamd_hs_magic))
static const guchar rspamd_hs_magic[] = {'r', 's', 'h', 's', 'r', 'e', '1', '1'},
		rspamd_hs_magic_vector[] = {'r', 's', 'h', 's', 'r', 'v', '1', '1'};
#endif

struct rspamd_re_class {
	guint64 id;
	enum rspamd_re_type type;
	gpointer type_data;
	gsize type_len;
	GHashTable *re;
	gchar hash[rspamd_cryptobox_HASHBYTES + 1];
	rspamd_cryptobox_hash_state_t *st;
#ifdef WITH_HYPERSCAN
	hs_database_t *hs_db;
	hs_scratch_t *hs_scratch;
	gint *hs_ids;
	guint nhs;
#endif
};

enum rspamd_re_cache_elt_match_type {
	RSPAMD_RE_CACHE_PCRE = 0,
	RSPAMD_RE_CACHE_HYPERSCAN,
	RSPAMD_RE_CACHE_HYPERSCAN_PRE
};

struct rspamd_re_cache_elt {
	rspamd_regexp_t *re;
	enum rspamd_re_cache_elt_match_type match_type;
};

struct rspamd_re_cache {
	GHashTable *re_classes;
	GPtrArray *re;
	ref_entry_t ref;
	guint nre;
	guint max_re_data;
	gchar hash[rspamd_cryptobox_HASHBYTES + 1];
#ifdef WITH_HYPERSCAN
	gboolean hyperscan_loaded;
	gboolean disable_hyperscan;
	gboolean vectorized_hyperscan;
	hs_platform_info_t plt;
#endif
};

struct rspamd_re_runtime {
	guchar *checked;
	guchar *results;
	struct rspamd_re_cache *cache;
	struct rspamd_re_cache_stat stat;
};

static GQuark
rspamd_re_cache_quark (void)
{
	return g_quark_from_static_string ("re_cache");
}

static guint64
rspamd_re_cache_class_id (enum rspamd_re_type type,
		gpointer type_data,
		gsize datalen)
{
	XXH64_state_t st;

	XXH64_reset (&st, 0xdeadbabe);
	XXH64_update (&st, &type, sizeof (type));

	if (datalen > 0) {
		XXH64_update (&st, type_data, datalen);
	}

	return XXH64_digest (&st);
}

static void
rspamd_re_cache_destroy (struct rspamd_re_cache *cache)
{
	GHashTableIter it;
	gpointer k, v;
	struct rspamd_re_class *re_class;

	g_assert (cache != NULL);
	g_hash_table_iter_init (&it, cache->re_classes);

	while (g_hash_table_iter_next (&it, &k, &v)) {
		re_class = v;
		g_hash_table_iter_steal (&it);
		g_hash_table_unref (re_class->re);
#ifdef WITH_HYPERSCAN
		if (re_class->hs_db) {
			hs_free_database (re_class->hs_db);
		}
		if (re_class->hs_scratch) {
			hs_free_scratch (re_class->hs_scratch);
		}
		if (re_class->hs_ids) {
			g_free (re_class->hs_ids);
		}
#endif
		g_slice_free1 (sizeof (*re_class), re_class);
	}

	g_hash_table_unref (cache->re_classes);
	g_ptr_array_free (cache->re, TRUE);
	g_slice_free1 (sizeof (*cache), cache);
}

static void
rspamd_re_cache_elt_dtor (gpointer e)
{
	struct rspamd_re_cache_elt *elt = e;

	rspamd_regexp_unref (elt->re);
	g_slice_free1 (sizeof (*elt), elt);
}

struct rspamd_re_cache *
rspamd_re_cache_new (void)
{
	struct rspamd_re_cache *cache;

	cache = g_slice_alloc (sizeof (*cache));
	cache->re_classes = g_hash_table_new (g_int64_hash, g_int64_equal);
	cache->nre = 0;
	cache->re = g_ptr_array_new_full (256, rspamd_re_cache_elt_dtor);
#ifdef WITH_HYPERSCAN
	cache->hyperscan_loaded = FALSE;
#endif
	REF_INIT_RETAIN (cache, rspamd_re_cache_destroy);

	return cache;
}

gboolean
rspamd_re_cache_is_hs_loaded (struct rspamd_re_cache *cache)
{
	g_assert (cache != NULL);

#ifdef WITH_HYPERSCAN
	return cache->hyperscan_loaded;
#else
	return FALSE;
#endif
}

rspamd_regexp_t *
rspamd_re_cache_add (struct rspamd_re_cache *cache, rspamd_regexp_t *re,
		enum rspamd_re_type type, gpointer type_data, gsize datalen)
{
	guint64 class_id;
	struct rspamd_re_class *re_class;
	rspamd_regexp_t *nre;
	struct rspamd_re_cache_elt *elt;

	g_assert (cache != NULL);
	g_assert (re != NULL);

	class_id = rspamd_re_cache_class_id (type, type_data, datalen);
	re_class = g_hash_table_lookup (cache->re_classes, &class_id);

	if (re_class == NULL) {
		re_class = g_slice_alloc0 (sizeof (*re_class));
		re_class->id = class_id;
		re_class->type_len = datalen;
		re_class->type = type;
		re_class->re = g_hash_table_new_full (rspamd_regexp_hash,
				rspamd_regexp_equal, NULL, (GDestroyNotify)rspamd_regexp_unref);

		if (datalen > 0) {
			re_class->type_data = g_slice_alloc (datalen);
			memcpy (re_class->type_data, type_data, datalen);
		}

		g_hash_table_insert (cache->re_classes, &re_class->id, re_class);
	}

	if ((nre = g_hash_table_lookup (re_class->re, rspamd_regexp_get_id (re)))
			== NULL) {
		/*
		 * We set re id based on the global position in the cache
		 */
		elt = g_slice_alloc0 (sizeof (*elt));
		/* One ref for re_class */
		nre = rspamd_regexp_ref (re);
		rspamd_regexp_set_cache_id (re, cache->nre++);
		/* One ref for cache */
		elt->re = rspamd_regexp_ref (re);
		g_ptr_array_add (cache->re, elt);
		rspamd_regexp_set_class (re, re_class);
		g_hash_table_insert (re_class->re, rspamd_regexp_get_id (nre), nre);
	}

	return nre;
}

void
rspamd_re_cache_replace (struct rspamd_re_cache *cache,
		rspamd_regexp_t *what,
		rspamd_regexp_t *with)
{
	guint64 re_id;
	struct rspamd_re_class *re_class;
	rspamd_regexp_t *src;
	struct rspamd_re_cache_elt *elt;

	g_assert (cache != NULL);
	g_assert (what != NULL);
	g_assert (with != NULL);

	re_class = rspamd_regexp_get_class (what);

	if (re_class != NULL) {
		re_id = rspamd_regexp_get_cache_id (what);

		g_assert (re_id != RSPAMD_INVALID_ID);
		src = g_hash_table_lookup (re_class->re, rspamd_regexp_get_id (what));
		elt = g_ptr_array_index (cache->re, re_id);
		g_assert (elt != NULL);
		g_assert (src != NULL);

		rspamd_regexp_set_cache_id (what, RSPAMD_INVALID_ID);
		rspamd_regexp_set_class (what, NULL);
		rspamd_regexp_set_cache_id (with, re_id);
		rspamd_regexp_set_class (with, re_class);
		/*
		 * On calling of this function, we actually unref old re (what)
		 */
		g_hash_table_insert (re_class->re,
				rspamd_regexp_get_id (what),
				rspamd_regexp_ref (with));

		rspamd_regexp_unref (elt->re);
		elt->re = rspamd_regexp_ref (with);
		/* XXX: do not touch match type here */
	}
}

static gint
rspamd_re_cache_sort_func (gconstpointer a, gconstpointer b)
{
	struct rspamd_re_cache_elt * const *re1 = a, * const *re2 = b;

	return rspamd_regexp_cmp (rspamd_regexp_get_id ((*re1)->re),
			rspamd_regexp_get_id ((*re2)->re));
}

void
rspamd_re_cache_init (struct rspamd_re_cache *cache, struct rspamd_config *cfg)
{
	guint i, fl;
	GHashTableIter it;
	gpointer k, v;
	struct rspamd_re_class *re_class;
	rspamd_cryptobox_hash_state_t st_global;
	rspamd_regexp_t *re;
	struct rspamd_re_cache_elt *elt;
	guchar hash_out[rspamd_cryptobox_HASHBYTES];

	g_assert (cache != NULL);

	rspamd_cryptobox_hash_init (&st_global, NULL, 0);
	/* Resort all regexps */
	g_ptr_array_sort (cache->re, rspamd_re_cache_sort_func);

	for (i = 0; i < cache->re->len; i ++) {
		elt = g_ptr_array_index (cache->re, i);
		re = elt->re;
		re_class = rspamd_regexp_get_class (re);
		g_assert (re_class != NULL);
		rspamd_regexp_set_cache_id (re, i);

		if (re_class->st == NULL) {
			re_class->st = g_slice_alloc (sizeof (*re_class->st));
			rspamd_cryptobox_hash_init (re_class->st, NULL, 0);
		}

		/* Update hashes */
		rspamd_cryptobox_hash_update (re_class->st, (gpointer) &re_class->id,
				sizeof (re_class->id));
		rspamd_cryptobox_hash_update (&st_global, (gpointer) &re_class->id,
				sizeof (re_class->id));
		rspamd_cryptobox_hash_update (re_class->st, rspamd_regexp_get_id (re),
				rspamd_cryptobox_HASHBYTES);
		rspamd_cryptobox_hash_update (&st_global, rspamd_regexp_get_id (re),
				rspamd_cryptobox_HASHBYTES);
		fl = rspamd_regexp_get_pcre_flags (re);
		rspamd_cryptobox_hash_update (re_class->st, (const guchar *)&fl,
				sizeof (fl));
		rspamd_cryptobox_hash_update (&st_global, (const guchar *) &fl,
				sizeof (fl));
		fl = rspamd_regexp_get_flags (re);
		rspamd_cryptobox_hash_update (re_class->st, (const guchar *) &fl,
				sizeof (fl));
		rspamd_cryptobox_hash_update (&st_global, (const guchar *) &fl,
				sizeof (fl));
		fl = rspamd_regexp_get_maxhits (re);
		rspamd_cryptobox_hash_update (re_class->st, (const guchar *) &fl,
				sizeof (fl));
		rspamd_cryptobox_hash_update (&st_global, (const guchar *) &fl,
				sizeof (fl));
	}

	rspamd_cryptobox_hash_final (&st_global, hash_out);
	rspamd_snprintf (cache->hash, sizeof (cache->hash), "%*xs",
			(gint) rspamd_cryptobox_HASHBYTES, hash_out);

	/* Now finalize all classes */
	g_hash_table_iter_init (&it, cache->re_classes);

	while (g_hash_table_iter_next (&it, &k, &v)) {
		re_class = v;

		if (re_class->st) {
			/*
			 * We finally update all classes with the number of expressions
			 * in the cache to ensure that if even a single re has been changed
			 * we won't be broken due to id mismatch
			 */
			rspamd_cryptobox_hash_update (re_class->st,
					(gpointer)&cache->re->len,
					sizeof (cache->re->len));
			rspamd_cryptobox_hash_final (re_class->st, hash_out);
			rspamd_snprintf (re_class->hash, sizeof (re_class->hash), "%*xs",
					(gint) rspamd_cryptobox_HASHBYTES, hash_out);
			g_slice_free1 (sizeof (*re_class->st), re_class->st);
			re_class->st = NULL;
		}
	}

#ifdef WITH_HYPERSCAN
	const gchar *platform = "generic";
	rspamd_fstring_t *features = rspamd_fstring_new ();

	cache->disable_hyperscan = cfg->disable_hyperscan;
	cache->vectorized_hyperscan = cfg->vectorized_hyperscan;

	g_assert (hs_populate_platform (&cache->plt) == HS_SUCCESS);

	/* Now decode what we do have */
	switch (cache->plt.tune) {
	case HS_TUNE_FAMILY_HSW:
		platform = "haswell";
		break;
	case HS_TUNE_FAMILY_SNB:
		platform = "sandy";
		break;
	case HS_TUNE_FAMILY_BDW:
		platform = "broadwell";
		break;
	case HS_TUNE_FAMILY_IVB:
		platform = "ivy";
		break;
	default:
		break;
	}

	if (cache->plt.cpu_features & HS_CPU_FEATURES_AVX2) {
		features = rspamd_fstring_append (features, "AVX2", 4);
	}

	hs_set_allocator (g_malloc, g_free);

	msg_info_re_cache ("loaded hyperscan engine witch cpu tune '%s' and features '%V'",
			platform, features);

	rspamd_fstring_free (features);
#endif
}

struct rspamd_re_runtime *
rspamd_re_cache_runtime_new (struct rspamd_re_cache *cache)
{
	struct rspamd_re_runtime *rt;
	g_assert (cache != NULL);

	rt = g_slice_alloc0 (sizeof (*rt));
	rt->cache = cache;
	REF_RETAIN (cache);
	rt->checked = g_slice_alloc0 (NBYTES (cache->nre));
	rt->results = g_slice_alloc0 (cache->nre);
	rt->stat.regexp_total = cache->nre;

	return rt;
}

const struct rspamd_re_cache_stat *
rspamd_re_cache_get_stat (struct rspamd_re_runtime *rt)
{
	g_assert (rt != NULL);

	return &rt->stat;
}

static guint
rspamd_re_cache_process_pcre (struct rspamd_re_runtime *rt,
		rspamd_regexp_t *re, rspamd_mempool_t *pool,
		const guchar *in, gsize len,
		gboolean is_raw)
{
	guint r = 0;
	const gchar *start = NULL, *end = NULL;
	guint max_hits = rspamd_regexp_get_maxhits (re);
	guint64 id = rspamd_regexp_get_cache_id (re);
	gdouble t1, t2;
	const gdouble slow_time = 0.1;

	if (in == NULL) {
		return rt->results[id];
	}

	if (len == 0) {
		len = strlen (in);
	}

	if (rt->cache->max_re_data > 0 && len > rt->cache->max_re_data) {
		len = rt->cache->max_re_data;
	}

	r = rt->results[id];

	if (max_hits == 0 || r < max_hits) {
		t1 = rspamd_get_ticks ();

		while (rspamd_regexp_search (re,
				in,
				len,
				&start,
				&end,
				is_raw,
				NULL)) {
			r++;

			if (max_hits > 0 && r >= max_hits) {
				break;
			}
		}

		rt->stat.regexp_checked++;
		rt->stat.bytes_scanned_pcre += len;
		rt->stat.bytes_scanned += len;

		if (r > 0) {
			rt->stat.regexp_matched += r;
		}

		t2 = rspamd_get_ticks ();

		if (t2 - t1 > slow_time) {
			msg_info_pool ("regexp '%16s' took %.2f seconds to execute",
					rspamd_regexp_get_pattern (re), t2 - t1);
		}
	}

	return r;
}

#ifdef WITH_HYPERSCAN
struct rspamd_re_hyperscan_cbdata {
	struct rspamd_re_runtime *rt;
	const guchar **ins;
	const guint *lens;
	guint count;
	rspamd_regexp_t *re;
	rspamd_mempool_t *pool;
};

static gint
rspamd_re_cache_hyperscan_cb (unsigned int id,
		unsigned long long from,
		unsigned long long to,
		unsigned int flags,
		void *ud)
{
	struct rspamd_re_hyperscan_cbdata *cbdata = ud;
	struct rspamd_re_runtime *rt;
	struct rspamd_re_cache_elt *pcre_elt;
	guint ret, maxhits, i, processed;

	rt = cbdata->rt;
	pcre_elt = g_ptr_array_index (rt->cache->re, id);
	maxhits = rspamd_regexp_get_maxhits (pcre_elt->re);

	if (pcre_elt->match_type == RSPAMD_RE_CACHE_HYPERSCAN) {
		ret = 1;
		setbit (rt->checked, id);

		if (maxhits == 0 || rt->results[id] < maxhits) {
			rt->results[id] += ret;
			rt->stat.regexp_matched++;
		}
	}
	else {
		if (!isset (rt->checked, id)) {

			processed = 0;

			for (i = 0; i < cbdata->count; i ++) {
				ret = rspamd_re_cache_process_pcre (rt,
						pcre_elt->re,
						cbdata->pool,
						cbdata->ins[i],
						cbdata->lens[i],
						FALSE);
				rt->results[id] = ret;
				setbit (rt->checked, id);

				processed += cbdata->lens[i];

				if (processed >= to) {
					break;
				}
			}
		}
	}

	return 0;
}
#endif

static guint
rspamd_re_cache_process_regexp_data (struct rspamd_re_runtime *rt,
		rspamd_regexp_t *re, rspamd_mempool_t *pool,
		const guchar **in, guint *lens,
		guint count,
		gboolean is_raw)
{

	guint64 re_id;
	guint ret = 0;
	guint i;

	re_id = rspamd_regexp_get_cache_id (re);

	if (count == 0 || in == NULL) {
		/* We assume this as absence of the specified data */
		setbit (rt->checked, re_id);
		rt->results[re_id] = ret;
		return ret;
	}

#ifndef WITH_HYPERSCAN
	for (i = 0; i < count; i++) {
		ret = rspamd_re_cache_process_pcre (rt,
				re,
				pool,
				in[i],
				lens[i],
				is_raw);
		rt->results[re_id] = ret;
	}

	setbit (rt->checked, re_id);
#else
	struct rspamd_re_cache_elt *elt;
	struct rspamd_re_class *re_class;
	struct rspamd_re_hyperscan_cbdata cbdata;

	elt = g_ptr_array_index (rt->cache->re, re_id);
	re_class = rspamd_regexp_get_class (re);

	if (rt->cache->disable_hyperscan || elt->match_type == RSPAMD_RE_CACHE_PCRE) {
		for (i = 0; i < count; i++) {
			ret = rspamd_re_cache_process_pcre (rt,
					re,
					pool,
					in[i],
					lens[i],
					is_raw);
			rt->results[re_id] = ret;
		}

		setbit (rt->checked, re_id);
	}
	else {
		for (i = 0; i < count; i ++) {
			if (rt->cache->max_re_data > 0 && lens[i] > rt->cache->max_re_data) {
				lens[i] = rt->cache->max_re_data;
			}
			rt->stat.bytes_scanned += lens[i];
		}

		g_assert (re_class->hs_scratch != NULL);
		g_assert (re_class->hs_db != NULL);

		/* Go through hyperscan API */
		if (!rt->cache->vectorized_hyperscan) {
			for (i = 0; i < count; i++) {
				cbdata.ins = &in[i];
				cbdata.re = re;
				cbdata.rt = rt;
				cbdata.lens = &lens[i];
				cbdata.count = 1;
				cbdata.pool = pool;

				if ((hs_scan (re_class->hs_db, in[i], lens[i], 0,
						re_class->hs_scratch,
						rspamd_re_cache_hyperscan_cb, &cbdata)) != HS_SUCCESS) {
					ret = 0;
				}
				else {
					ret = rt->results[re_id];
				}
			}
		}
		else {
			cbdata.ins = in;
			cbdata.re = re;
			cbdata.rt = rt;
			cbdata.lens = lens;
			cbdata.count = 1;
			cbdata.pool = pool;

			if ((hs_scan_vector (re_class->hs_db, (const char **)in, lens, count, 0,
					re_class->hs_scratch,
					rspamd_re_cache_hyperscan_cb, &cbdata)) != HS_SUCCESS) {
				ret = 0;
			}
			else {
				ret = rt->results[re_id];
			}
		}
	}
#endif

	return ret;
}

static void
rspamd_re_cache_finish_class (struct rspamd_re_runtime *rt,
		struct rspamd_re_class *re_class)
{
#ifdef WITH_HYPERSCAN
	guint i;
	guint64 re_id;

	/* Set all bits unchecked */
	for (i = 0; i < re_class->nhs; i++) {
		re_id = re_class->hs_ids[i];

		if (!isset (rt->checked, re_id)) {
			g_assert (rt->results[re_id] == 0);
			rt->results[re_id] = 0;
			setbit (rt->checked, re_id);
		}
	}
#endif
}

/*
 * Calculates the specified regexp for the specified class if it's not calculated
 */
static guint
rspamd_re_cache_exec_re (struct rspamd_task *task,
		struct rspamd_re_runtime *rt,
		rspamd_regexp_t *re,
		struct rspamd_re_class *re_class,
		gboolean is_strong)
{
	guint ret = 0, i, re_id;
	GPtrArray *headerlist;
	GList *slist;
	GHashTableIter it;
	struct raw_header *rh;
	const gchar *in, *end;
	const guchar **scvec;
	guint *lenvec;
	gboolean raw = FALSE;
	struct mime_text_part *part;
	struct mime_part *mime_part;
	struct rspamd_url *url;
	struct rspamd_re_cache *cache = rt->cache;
	gpointer k, v;
	guint len, cnt;

	msg_debug_re_cache ("get to the slow path for re type: %s: %s",
			rspamd_re_cache_type_to_string (re_class->type),
			rspamd_regexp_get_pattern (re));
	re_id = rspamd_regexp_get_cache_id (re);

	switch (re_class->type) {
	case RSPAMD_RE_HEADER:
	case RSPAMD_RE_RAWHEADER:
		/* Get list of specified headers */
		headerlist = rspamd_message_get_header_array (task,
				re_class->type_data,
				is_strong);

		if (headerlist) {
			scvec = g_malloc (sizeof (*scvec) * headerlist->len);
			lenvec = g_malloc (sizeof (*lenvec) * headerlist->len);

			for (i = 0; i < headerlist->len; i ++) {
				rh = g_ptr_array_index (headerlist, i);

				if (re_class->type == RSPAMD_RE_RAWHEADER) {
					in = rh->value;
					raw = TRUE;
					lenvec[i] = strlen (rh->value);
				}
				else {
					in = rh->decoded;
					/* Validate input */
					if (!in || !g_utf8_validate (in, -1, &end)) {
						lenvec[i] = 0;
						scvec[i] = (guchar *)"";
						continue;
					}
					lenvec[i] = end - in;
				}

				scvec[i] = (guchar *)in;
			}

			ret = rspamd_re_cache_process_regexp_data (rt, re,
					task->task_pool, scvec, lenvec, headerlist->len, raw);
			debug_task ("checking header %s regexp: %s -> %d",
					re_class->type_data,
					rspamd_regexp_get_pattern (re), ret);
			g_free (scvec);
			g_free (lenvec);
		}
		break;
	case RSPAMD_RE_ALLHEADER:
		raw = TRUE;
		in = task->raw_headers_content.begin;
		len = task->raw_headers_content.len;
		ret = rspamd_re_cache_process_regexp_data (rt, re,
				task->task_pool, (const guchar **)&in, &len, 1, raw);
		debug_task ("checking allheader regexp: %s -> %d",
				rspamd_regexp_get_pattern (re), ret);
		break;
	case RSPAMD_RE_MIMEHEADER:
		headerlist = rspamd_message_get_mime_header_array (task,
				re_class->type_data,
				is_strong);

		if (headerlist) {
			scvec = g_malloc (sizeof (*scvec) * headerlist->len);
			lenvec = g_malloc (sizeof (*lenvec) * headerlist->len);

			for (i = 0; i < headerlist->len; i ++) {
				rh = g_ptr_array_index (headerlist, i);

				if (re_class->type == RSPAMD_RE_RAWHEADER) {
					in = rh->value;
					raw = TRUE;
					lenvec[i] = strlen (rh->value);
				}
				else {
					in = rh->decoded;
					/* Validate input */
					if (!in || !g_utf8_validate (in, -1, &end)) {
						lenvec[i] = 0;
						scvec[i] = (guchar *)"";
						continue;
					}
					lenvec[i] = end - in;
				}

				scvec[i] = (guchar *)in;
			}

			ret = rspamd_re_cache_process_regexp_data (rt, re,
					task->task_pool, scvec, lenvec, headerlist->len, raw);
			debug_task ("checking mime header %s regexp: %s -> %d",
					re_class->type_data,
					rspamd_regexp_get_pattern (re), ret);
			g_free (scvec);
			g_free (lenvec);
		}
		break;
	case RSPAMD_RE_MIME:
	case RSPAMD_RE_RAWMIME:
		/* Iterate through text parts */
		if (task->text_parts->len > 0) {
			cnt = task->text_parts->len;
			scvec = g_malloc (sizeof (*scvec) * cnt);
			lenvec = g_malloc (sizeof (*lenvec) * cnt);

			for (i = 0; i < task->text_parts->len; i++) {
				part = g_ptr_array_index (task->text_parts, i);

				/* Skip empty parts */
				if (IS_PART_EMPTY (part)) {
					lenvec[i] = 0;
					scvec[i] = (guchar *) "";
					continue;
				}

				/* Check raw flags */
				if (!IS_PART_UTF (part)) {
					raw = TRUE;
				}
				/* Select data for regexp */
				if (re_class->type == RSPAMD_RE_RAWMIME) {
					in = part->orig->data;
					len = part->orig->len;
					raw = TRUE;
				}
				else {
					in = part->content->data;
					len = part->content->len;
				}

				scvec[i] = (guchar *) in;
				lenvec[i] = len;
			}

			ret = rspamd_re_cache_process_regexp_data (rt, re,
					task->task_pool, scvec, lenvec, cnt, raw);
			debug_task ("checking mime regexp: %s -> %d",
					rspamd_regexp_get_pattern (re), ret);
			g_free (scvec);
			g_free (lenvec);
		}
		break;
	case RSPAMD_RE_URL:
		cnt = g_hash_table_size (task->urls) + g_hash_table_size (task->emails);

		if (cnt > 0) {
			scvec = g_malloc (sizeof (*scvec) * cnt);
			lenvec = g_malloc (sizeof (*lenvec) * cnt);
			g_hash_table_iter_init (&it, task->urls);
			i = 0;

			while (g_hash_table_iter_next (&it, &k, &v)) {
				url = v;
				in = url->string;
				len = url->urllen;
				raw = FALSE;

				scvec[i] = (guchar *)in;
				lenvec[i++] = len;
			}

			g_hash_table_iter_init (&it, task->emails);

			while (g_hash_table_iter_next (&it, &k, &v)) {
				url = v;
				in = url->string;
				len = url->urllen;
				raw = FALSE;

				scvec[i] = (guchar *) in;
				lenvec[i++] = len;
			}

			g_assert (i == cnt);

			ret = rspamd_re_cache_process_regexp_data (rt, re,
					task->task_pool, scvec, lenvec, i, raw);
			debug_task ("checking url regexp: %s -> %d",
					rspamd_regexp_get_pattern (re), ret);
			g_free (scvec);
			g_free (lenvec);
		}
		break;
	case RSPAMD_RE_BODY:
		raw = TRUE;
		in = task->msg.begin;
		len = task->msg.len;

		ret = rspamd_re_cache_process_regexp_data (rt, re, task->task_pool,
				(const guchar **)&in, &len, 1, raw);
		debug_task ("checking rawbody regexp: %s -> %d",
				rspamd_regexp_get_pattern (re), ret);
		break;
	case RSPAMD_RE_SABODY:
		/*
		 * For SA body we get all parts and extract headers + body from them
		 *
		 */
		if (task->parts->len > 0) {
			cnt = task->parts->len * 2 + 1;
			scvec = g_malloc (sizeof (*scvec) * cnt);
			lenvec = g_malloc (sizeof (*lenvec) * cnt);

			/*
			 * Body rules also include the Subject as the first line
			 * of the body content.
			 */

			slist = rspamd_message_get_header (task, "Subject", FALSE);

			if (slist) {
				rh = slist->data;

				scvec[0] = (guchar *)rh->decoded;
				lenvec[0] = strlen (rh->decoded);
			}
			else {
				scvec[0] = (guchar *)"";
				lenvec[0] = 0;
			}

			for (i = 0; i < task->parts->len; i++) {
				mime_part = g_ptr_array_index (task->parts, i);

				if (mime_part->raw_headers_str) {
					scvec[i * 2 + 1] = (guchar *)mime_part->raw_headers_str;
					lenvec[i * 2 + 1] = strlen (mime_part->raw_headers_str);
				}
				else {
					scvec[i * 2 + 1] = (guchar *)"";
					lenvec[i * 2 + 1] = 0;
				}

				if (mime_part->content) {
					scvec[i * 2 + 2] = (guchar *)mime_part->content->data;
					lenvec[i * 2 + 2] = mime_part->content->len;
				}
				else {
					scvec[i * 2 + 2] = (guchar *)"";
					lenvec[i * 2 + 2] = 0;
				}
			}

			ret = rspamd_re_cache_process_regexp_data (rt, re,
					task->task_pool, scvec, lenvec, cnt, TRUE);
			debug_task ("checking sa body regexp: %s -> %d",
					rspamd_regexp_get_pattern (re), ret);
			g_free (scvec);
			g_free (lenvec);
		}
		break;
	case RSPAMD_RE_MAX:
		msg_err_task ("regexp of class invalid has been called: %s",
				rspamd_regexp_get_pattern (re));
		break;
	}

#if WITH_HYPERSCAN
	if (!rt->cache->disable_hyperscan) {
		rspamd_re_cache_finish_class (rt, re_class);
	}
#endif

	setbit (rt->checked, re_id);

	return rt->results[re_id];
}

gint
rspamd_re_cache_process (struct rspamd_task *task,
		struct rspamd_re_runtime *rt,
		rspamd_regexp_t *re,
		enum rspamd_re_type type,
		gpointer type_data,
		gsize datalen,
		gboolean is_strong)
{
	guint64 re_id;
	struct rspamd_re_class *re_class;
	struct rspamd_re_cache *cache;

	g_assert (rt != NULL);
	g_assert (task != NULL);
	g_assert (re != NULL);

	cache = rt->cache;
	re_id = rspamd_regexp_get_cache_id (re);

	if (re_id == RSPAMD_INVALID_ID || re_id > cache->nre) {
		msg_err_task ("re '%s' has no valid id for the cache",
				rspamd_regexp_get_pattern (re));
		return 0;
	}

	if (isset (rt->checked, re_id)) {
		/* Fast path */
		rt->stat.regexp_fast_cached ++;
		return rt->results[re_id];
	}
	else {
		/* Slow path */
		re_class = rspamd_regexp_get_class (re);

		if (re_class == NULL) {
			msg_err_task ("cannot find re class for regexp '%s'",
					rspamd_regexp_get_pattern (re));
			return 0;
		}

		return rspamd_re_cache_exec_re (task, rt, re, re_class,
				is_strong);
	}

	return 0;
}

void
rspamd_re_cache_runtime_destroy (struct rspamd_re_runtime *rt)
{
	g_assert (rt != NULL);

	g_slice_free1 (NBYTES (rt->cache->nre), rt->checked);
	g_slice_free1 (rt->cache->nre, rt->results);
	REF_RELEASE (rt->cache);
	g_slice_free1 (sizeof (*rt), rt);
}

void
rspamd_re_cache_unref (struct rspamd_re_cache *cache)
{
	if (cache) {
		REF_RELEASE (cache);
	}
}

struct rspamd_re_cache *
rspamd_re_cache_ref (struct rspamd_re_cache *cache)
{
	if (cache) {
		REF_RETAIN (cache);
	}

	return cache;
}

guint
rspamd_re_cache_set_limit (struct rspamd_re_cache *cache, guint limit)
{
	guint old;

	g_assert (cache != NULL);

	old = cache->max_re_data;
	cache->max_re_data = limit;

	return old;
}

const gchar *
rspamd_re_cache_type_to_string (enum rspamd_re_type type)
{
	const gchar *ret = "unknown";

	switch (type) {
	case RSPAMD_RE_HEADER:
		ret = "header";
		break;
	case RSPAMD_RE_RAWHEADER:
		ret = "raw header";
		break;
	case RSPAMD_RE_MIMEHEADER:
		ret = "mime header";
		break;
	case RSPAMD_RE_ALLHEADER:
		ret = "all headers";
		break;
	case RSPAMD_RE_MIME:
		ret = "part";
		break;
	case RSPAMD_RE_RAWMIME:
		ret = "raw part";
		break;
	case RSPAMD_RE_BODY:
		ret = "rawbody";
		break;
	case RSPAMD_RE_URL:
		ret = "url";
		break;
	case RSPAMD_RE_SABODY:
		ret = "sa body";
		break;
	case RSPAMD_RE_MAX:
		ret = "invalid class";
		break;
	}

	return ret;
}

enum rspamd_re_type
rspamd_re_cache_type_from_string (const char *str)
{
	enum rspamd_re_type ret = RSPAMD_RE_MAX;

	if (str != NULL) {
		if (strcmp (str, "header") == 0) {
			ret = RSPAMD_RE_HEADER;
		}
		else if (strcmp (str, "rawheader") == 0) {
			ret = RSPAMD_RE_RAWHEADER;
		}
		else if (strcmp (str, "mime") == 0) {
			ret = RSPAMD_RE_MIME;
		}
		else if (strcmp (str, "rawmime") == 0) {
			ret = RSPAMD_RE_RAWMIME;
		}
		else if (strcmp (str, "body") == 0) {
			ret = RSPAMD_RE_BODY;
		}
		else if (strcmp (str, "url") == 0) {
			ret = RSPAMD_RE_URL;
		}
		else if (strcmp (str, "allheader") == 0) {
			ret = RSPAMD_RE_ALLHEADER;
		}
		else if (strcmp (str, "mimeheader") == 0) {
			ret = RSPAMD_RE_MIMEHEADER;
		}
		else if (strcmp (str, "sabody") == 0) {
			ret = RSPAMD_RE_SABODY;
		}
	}

	return ret;
}

#ifdef WITH_HYPERSCAN
static gboolean
rspamd_re_cache_is_finite (struct rspamd_re_cache *cache,
		rspamd_regexp_t *re, gint flags, gdouble max_time)
{
	pid_t cld;
	gint status;
	struct timespec ts;
	hs_compile_error_t *hs_errors;
	hs_database_t *test_db;
	gdouble wait_time;<commit_msg>[Feature] Optimize rspamd_re_cache_type_from_string<commit_after>/*-
 * Copyright 2016 <NAME>
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *   http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
#include "libmime/message.h"
#include "re_cache.h"
#include "xxhash.h"
#include "cryptobox.h"
#include "ref.h"
#include "libserver/url.h"
#include "libserver/task.h"
#include "libserver/cfg_file.h"
#include "libutil/util.h"
#include "libutil/regexp.h"
#ifdef WITH_HYPERSCAN
#include "hs.h"
#include "unix-std.h"
#include <signal.h>

#ifndef WITH_PCRE2
#include <pcre.h>
#else
#include <pcre2.h>
#endif

#ifdef HAVE_SYS_WAIT_H
#include <sys/wait.h>
#endif

#endif

#define msg_err_re_cache(...) rspamd_default_log_function (G_LOG_LEVEL_CRITICAL, \
        "re_cache", cache->hash, \
        G_STRFUNC, \
        __VA_ARGS__)
#define msg_warn_re_cache(...)   rspamd_default_log_function (G_LOG_LEVEL_WARNING, \
        "re_cache", cache->hash, \
        G_STRFUNC, \
        __VA_ARGS__)
#define msg_info_re_cache(...)   rspamd_default_log_function (G_LOG_LEVEL_INFO, \
        "re_cache", cache->hash, \
        G_STRFUNC, \
        __VA_ARGS__)
#define msg_debug_re_cache(...)  rspamd_default_log_function (G_LOG_LEVEL_DEBUG, \
        "re_cache", cache->hash, \
        G_STRFUNC, \
        __VA_ARGS__)

#ifdef WITH_HYPERSCAN
#define RSPAMD_HS_MAGIC_LEN (sizeof (rspamd_hs_magic))
static const guchar rspamd_hs_magic[] = {'r', 's', 'h', 's', 'r', 'e', '1', '1'},
		rspamd_hs_magic_vector[] = {'r', 's', 'h', 's', 'r', 'v', '1', '1'};
#endif

struct rspamd_re_class {
	guint64 id;
	enum rspamd_re_type type;
	gpointer type_data;
	gsize type_len;
	GHashTable *re;
	gchar hash[rspamd_cryptobox_HASHBYTES + 1];
	rspamd_cryptobox_hash_state_t *st;
#ifdef WITH_HYPERSCAN
	hs_database_t *hs_db;
	hs_scratch_t *hs_scratch;
	gint *hs_ids;
	guint nhs;
#endif
};

enum rspamd_re_cache_elt_match_type {
	RSPAMD_RE_CACHE_PCRE = 0,
	RSPAMD_RE_CACHE_HYPERSCAN,
	RSPAMD_RE_CACHE_HYPERSCAN_PRE
};

struct rspamd_re_cache_elt {
	rspamd_regexp_t *re;
	enum rspamd_re_cache_elt_match_type match_type;
};

struct rspamd_re_cache {
	GHashTable *re_classes;
	GPtrArray *re;
	ref_entry_t ref;
	guint nre;
	guint max_re_data;
	gchar hash[rspamd_cryptobox_HASHBYTES + 1];
#ifdef WITH_HYPERSCAN
	gboolean hyperscan_loaded;
	gboolean disable_hyperscan;
	gboolean vectorized_hyperscan;
	hs_platform_info_t plt;
#endif
};

struct rspamd_re_runtime {
	guchar *checked;
	guchar *results;
	struct rspamd_re_cache *cache;
	struct rspamd_re_cache_stat stat;
};

static GQuark
rspamd_re_cache_quark (void)
{
	return g_quark_from_static_string ("re_cache");
}

static guint64
rspamd_re_cache_class_id (enum rspamd_re_type type,
		gpointer type_data,
		gsize datalen)
{
	XXH64_state_t st;

	XXH64_reset (&st, 0xdeadbabe);
	XXH64_update (&st, &type, sizeof (type));

	if (datalen > 0) {
		XXH64_update (&st, type_data, datalen);
	}

	return XXH64_digest (&st);
}

static void
rspamd_re_cache_destroy (struct rspamd_re_cache *cache)
{
	GHashTableIter it;
	gpointer k, v;
	struct rspamd_re_class *re_class;

	g_assert (cache != NULL);
	g_hash_table_iter_init (&it, cache->re_classes);

	while (g_hash_table_iter_next (&it, &k, &v)) {
		re_class = v;
		g_hash_table_iter_steal (&it);
		g_hash_table_unref (re_class->re);
#ifdef WITH_HYPERSCAN
		if (re_class->hs_db) {
			hs_free_database (re_class->hs_db);
		}
		if (re_class->hs_scratch) {
			hs_free_scratch (re_class->hs_scratch);
		}
		if (re_class->hs_ids) {
			g_free (re_class->hs_ids);
		}
#endif
		g_slice_free1 (sizeof (*re_class), re_class);
	}

	g_hash_table_unref (cache->re_classes);
	g_ptr_array_free (cache->re, TRUE);
	g_slice_free1 (sizeof (*cache), cache);
}

static void
rspamd_re_cache_elt_dtor (gpointer e)
{
	struct rspamd_re_cache_elt *elt = e;

	rspamd_regexp_unref (elt->re);
	g_slice_free1 (sizeof (*elt), elt);
}

struct rspamd_re_cache *
rspamd_re_cache_new (void)
{
	struct rspamd_re_cache *cache;

	cache = g_slice_alloc (sizeof (*cache));
	cache->re_classes = g_hash_table_new (g_int64_hash, g_int64_equal);
	cache->nre = 0;
	cache->re = g_ptr_array_new_full (256, rspamd_re_cache_elt_dtor);
#ifdef WITH_HYPERSCAN
	cache->hyperscan_loaded = FALSE;
#endif
	REF_INIT_RETAIN (cache, rspamd_re_cache_destroy);

	return cache;
}

gboolean
rspamd_re_cache_is_hs_loaded (struct rspamd_re_cache *cache)
{
	g_assert (cache != NULL);

#ifdef WITH_HYPERSCAN
	return cache->hyperscan_loaded;
#else
	return FALSE;
#endif
}

rspamd_regexp_t *
rspamd_re_cache_add (struct rspamd_re_cache *cache, rspamd_regexp_t *re,
		enum rspamd_re_type type, gpointer type_data, gsize datalen)
{
	guint64 class_id;
	struct rspamd_re_class *re_class;
	rspamd_regexp_t *nre;
	struct rspamd_re_cache_elt *elt;

	g_assert (cache != NULL);
	g_assert (re != NULL);

	class_id = rspamd_re_cache_class_id (type, type_data, datalen);
	re_class = g_hash_table_lookup (cache->re_classes, &class_id);

	if (re_class == NULL) {
		re_class = g_slice_alloc0 (sizeof (*re_class));
		re_class->id = class_id;
		re_class->type_len = datalen;
		re_class->type = type;
		re_class->re = g_hash_table_new_full (rspamd_regexp_hash,
				rspamd_regexp_equal, NULL, (GDestroyNotify)rspamd_regexp_unref);

		if (datalen > 0) {
			re_class->type_data = g_slice_alloc (datalen);
			memcpy (re_class->type_data, type_data, datalen);
		}

		g_hash_table_insert (cache->re_classes, &re_class->id, re_class);
	}

	if ((nre = g_hash_table_lookup (re_class->re, rspamd_regexp_get_id (re)))
			== NULL) {
		/*
		 * We set re id based on the global position in the cache
		 */
		elt = g_slice_alloc0 (sizeof (*elt));
		/* One ref for re_class */
		nre = rspamd_regexp_ref (re);
		rspamd_regexp_set_cache_id (re, cache->nre++);
		/* One ref for cache */
		elt->re = rspamd_regexp_ref (re);
		g_ptr_array_add (cache->re, elt);
		rspamd_regexp_set_class (re, re_class);
		g_hash_table_insert (re_class->re, rspamd_regexp_get_id (nre), nre);
	}

	return nre;
}

void
rspamd_re_cache_replace (struct rspamd_re_cache *cache,
		rspamd_regexp_t *what,
		rspamd_regexp_t *with)
{
	guint64 re_id;
	struct rspamd_re_class *re_class;
	rspamd_regexp_t *src;
	struct rspamd_re_cache_elt *elt;

	g_assert (cache != NULL);
	g_assert (what != NULL);
	g_assert (with != NULL);

	re_class = rspamd_regexp_get_class (what);

	if (re_class != NULL) {
		re_id = rspamd_regexp_get_cache_id (what);

		g_assert (re_id != RSPAMD_INVALID_ID);
		src = g_hash_table_lookup (re_class->re, rspamd_regexp_get_id (what));
		elt = g_ptr_array_index (cache->re, re_id);
		g_assert (elt != NULL);
		g_assert (src != NULL);

		rspamd_regexp_set_cache_id (what, RSPAMD_INVALID_ID);
		rspamd_regexp_set_class (what, NULL);
		rspamd_regexp_set_cache_id (with, re_id);
		rspamd_regexp_set_class (with, re_class);
		/*
		 * On calling of this function, we actually unref old re (what)
		 */
		g_hash_table_insert (re_class->re,
				rspamd_regexp_get_id (what),
				rspamd_regexp_ref (with));

		rspamd_regexp_unref (elt->re);
		elt->re = rspamd_regexp_ref (with);
		/* XXX: do not touch match type here */
	}
}

static gint
rspamd_re_cache_sort_func (gconstpointer a, gconstpointer b)
{
	struct rspamd_re_cache_elt * const *re1 = a, * const *re2 = b;

	return rspamd_regexp_cmp (rspamd_regexp_get_id ((*re1)->re),
			rspamd_regexp_get_id ((*re2)->re));
}

void
rspamd_re_cache_init (struct rspamd_re_cache *cache, struct rspamd_config *cfg)
{
	guint i, fl;
	GHashTableIter it;
	gpointer k, v;
	struct rspamd_re_class *re_class;
	rspamd_cryptobox_hash_state_t st_global;
	rspamd_regexp_t *re;
	struct rspamd_re_cache_elt *elt;
	guchar hash_out[rspamd_cryptobox_HASHBYTES];

	g_assert (cache != NULL);

	rspamd_cryptobox_hash_init (&st_global, NULL, 0);
	/* Resort all regexps */
	g_ptr_array_sort (cache->re, rspamd_re_cache_sort_func);

	for (i = 0; i < cache->re->len; i ++) {
		elt = g_ptr_array_index (cache->re, i);
		re = elt->re;
		re_class = rspamd_regexp_get_class (re);
		g_assert (re_class != NULL);
		rspamd_regexp_set_cache_id (re, i);

		if (re_class->st == NULL) {
			re_class->st = g_slice_alloc (sizeof (*re_class->st));
			rspamd_cryptobox_hash_init (re_class->st, NULL, 0);
		}

		/* Update hashes */
		rspamd_cryptobox_hash_update (re_class->st, (gpointer) &re_class->id,
				sizeof (re_class->id));
		rspamd_cryptobox_hash_update (&st_global, (gpointer) &re_class->id,
				sizeof (re_class->id));
		rspamd_cryptobox_hash_update (re_class->st, rspamd_regexp_get_id (re),
				rspamd_cryptobox_HASHBYTES);
		rspamd_cryptobox_hash_update (&st_global, rspamd_regexp_get_id (re),
				rspamd_cryptobox_HASHBYTES);
		fl = rspamd_regexp_get_pcre_flags (re);
		rspamd_cryptobox_hash_update (re_class->st, (const guchar *)&fl,
				sizeof (fl));
		rspamd_cryptobox_hash_update (&st_global, (const guchar *) &fl,
				sizeof (fl));
		fl = rspamd_regexp_get_flags (re);
		rspamd_cryptobox_hash_update (re_class->st, (const guchar *) &fl,
				sizeof (fl));
		rspamd_cryptobox_hash_update (&st_global, (const guchar *) &fl,
				sizeof (fl));
		fl = rspamd_regexp_get_maxhits (re);
		rspamd_cryptobox_hash_update (re_class->st, (const guchar *) &fl,
				sizeof (fl));
		rspamd_cryptobox_hash_update (&st_global, (const guchar *) &fl,
				sizeof (fl));
	}

	rspamd_cryptobox_hash_final (&st_global, hash_out);
	rspamd_snprintf (cache->hash, sizeof (cache->hash), "%*xs",
			(gint) rspamd_cryptobox_HASHBYTES, hash_out);

	/* Now finalize all classes */
	g_hash_table_iter_init (&it, cache->re_classes);

	while (g_hash_table_iter_next (&it, &k, &v)) {
		re_class = v;

		if (re_class->st) {
			/*
			 * We finally update all classes with the number of expressions
			 * in the cache to ensure that if even a single re has been changed
			 * we won't be broken due to id mismatch
			 */
			rspamd_cryptobox_hash_update (re_class->st,
					(gpointer)&cache->re->len,
					sizeof (cache->re->len));
			rspamd_cryptobox_hash_final (re_class->st, hash_out);
			rspamd_snprintf (re_class->hash, sizeof (re_class->hash), "%*xs",
					(gint) rspamd_cryptobox_HASHBYTES, hash_out);
			g_slice_free1 (sizeof (*re_class->st), re_class->st);
			re_class->st = NULL;
		}
	}

#ifdef WITH_HYPERSCAN
	const gchar *platform = "generic";
	rspamd_fstring_t *features = rspamd_fstring_new ();

	cache->disable_hyperscan = cfg->disable_hyperscan;
	cache->vectorized_hyperscan = cfg->vectorized_hyperscan;

	g_assert (hs_populate_platform (&cache->plt) == HS_SUCCESS);

	/* Now decode what we do have */
	switch (cache->plt.tune) {
	case HS_TUNE_FAMILY_HSW:
		platform = "haswell";
		break;
	case HS_TUNE_FAMILY_SNB:
		platform = "sandy";
		break;
	case HS_TUNE_FAMILY_BDW:
		platform = "broadwell";
		break;
	case HS_TUNE_FAMILY_IVB:
		platform = "ivy";
		break;
	default:
		break;
	}

	if (cache->plt.cpu_features & HS_CPU_FEATURES_AVX2) {
		features = rspamd_fstring_append (features, "AVX2", 4);
	}

	hs_set_allocator (g_malloc, g_free);

	msg_info_re_cache ("loaded hyperscan engine witch cpu tune '%s' and features '%V'",
			platform, features);

	rspamd_fstring_free (features);
#endif
}

struct rspamd_re_runtime *
rspamd_re_cache_runtime_new (struct rspamd_re_cache *cache)
{
	struct rspamd_re_runtime *rt;
	g_assert (cache != NULL);

	rt = g_slice_alloc0 (sizeof (*rt));
	rt->cache = cache;
	REF_RETAIN (cache);
	rt->checked = g_slice_alloc0 (NBYTES (cache->nre));
	rt->results = g_slice_alloc0 (cache->nre);
	rt->stat.regexp_total = cache->nre;

	return rt;
}

const struct rspamd_re_cache_stat *
rspamd_re_cache_get_stat (struct rspamd_re_runtime *rt)
{
	g_assert (rt != NULL);

	return &rt->stat;
}

static guint
rspamd_re_cache_process_pcre (struct rspamd_re_runtime *rt,
		rspamd_regexp_t *re, rspamd_mempool_t *pool,
		const guchar *in, gsize len,
		gboolean is_raw)
{
	guint r = 0;
	const gchar *start = NULL, *end = NULL;
	guint max_hits = rspamd_regexp_get_maxhits (re);
	guint64 id = rspamd_regexp_get_cache_id (re);
	gdouble t1, t2;
	const gdouble slow_time = 0.1;

	if (in == NULL) {
		return rt->results[id];
	}

	if (len == 0) {
		len = strlen (in);
	}

	if (rt->cache->max_re_data > 0 && len > rt->cache->max_re_data) {
		len = rt->cache->max_re_data;
	}

	r = rt->results[id];

	if (max_hits == 0 || r < max_hits) {
		t1 = rspamd_get_ticks ();

		while (rspamd_regexp_search (re,
				in,
				len,
				&start,
				&end,
				is_raw,
				NULL)) {
			r++;

			if (max_hits > 0 && r >= max_hits) {
				break;
			}
		}

		rt->stat.regexp_checked++;
		rt->stat.bytes_scanned_pcre += len;
		rt->stat.bytes_scanned += len;

		if (r > 0) {
			rt->stat.regexp_matched += r;
		}

		t2 = rspamd_get_ticks ();

		if (t2 - t1 > slow_time) {
			msg_info_pool ("regexp '%16s' took %.2f seconds to execute",
					rspamd_regexp_get_pattern (re), t2 - t1);
		}
	}

	return r;
}

#ifdef WITH_HYPERSCAN
struct rspamd_re_hyperscan_cbdata {
	struct rspamd_re_runtime *rt;
	const guchar **ins;
	const guint *lens;
	guint count;
	rspamd_regexp_t *re;
	rspamd_mempool_t *pool;
};

static gint
rspamd_re_cache_hyperscan_cb (unsigned int id,
		unsigned long long from,
		unsigned long long to,
		unsigned int flags,
		void *ud)
{
	struct rspamd_re_hyperscan_cbdata *cbdata = ud;
	struct rspamd_re_runtime *rt;
	struct rspamd_re_cache_elt *pcre_elt;
	guint ret, maxhits, i, processed;

	rt = cbdata->rt;
	pcre_elt = g_ptr_array_index (rt->cache->re, id);
	maxhits = rspamd_regexp_get_maxhits (pcre_elt->re);

	if (pcre_elt->match_type == RSPAMD_RE_CACHE_HYPERSCAN) {
		ret = 1;
		setbit (rt->checked, id);

		if (maxhits == 0 || rt->results[id] < maxhits) {
			rt->results[id] += ret;
			rt->stat.regexp_matched++;
		}
	}
	else {
		if (!isset (rt->checked, id)) {

			processed = 0;

			for (i = 0; i < cbdata->count; i ++) {
				ret = rspamd_re_cache_process_pcre (rt,
						pcre_elt->re,
						cbdata->pool,
						cbdata->ins[i],
						cbdata->lens[i],
						FALSE);
				rt->results[id] = ret;
				setbit (rt->checked, id);

				processed += cbdata->lens[i];

				if (processed >= to) {
					break;
				}
			}
		}
	}

	return 0;
}
#endif

static guint
rspamd_re_cache_process_regexp_data (struct rspamd_re_runtime *rt,
		rspamd_regexp_t *re, rspamd_mempool_t *pool,
		const guchar **in, guint *lens,
		guint count,
		gboolean is_raw)
{

	guint64 re_id;
	guint ret = 0;
	guint i;

	re_id = rspamd_regexp_get_cache_id (re);

	if (count == 0 || in == NULL) {
		/* We assume this as absence of the specified data */
		setbit (rt->checked, re_id);
		rt->results[re_id] = ret;
		return ret;
	}

#ifndef WITH_HYPERSCAN
	for (i = 0; i < count; i++) {
		ret = rspamd_re_cache_process_pcre (rt,
				re,
				pool,
				in[i],
				lens[i],
				is_raw);
		rt->results[re_id] = ret;
	}

	setbit (rt->checked, re_id);
#else
	struct rspamd_re_cache_elt *elt;
	struct rspamd_re_class *re_class;
	struct rspamd_re_hyperscan_cbdata cbdata;

	elt = g_ptr_array_index (rt->cache->re, re_id);
	re_class = rspamd_regexp_get_class (re);

	if (rt->cache->disable_hyperscan || elt->match_type == RSPAMD_RE_CACHE_PCRE) {
		for (i = 0; i < count; i++) {
			ret = rspamd_re_cache_process_pcre (rt,
					re,
					pool,
					in[i],
					lens[i],
					is_raw);
			rt->results[re_id] = ret;
		}

		setbit (rt->checked, re_id);
	}
	else {
		for (i = 0; i < count; i ++) {
			if (rt->cache->max_re_data > 0 && lens[i] > rt->cache->max_re_data) {
				lens[i] = rt->cache->max_re_data;
			}
			rt->stat.bytes_scanned += lens[i];
		}

		g_assert (re_class->hs_scratch != NULL);
		g_assert (re_class->hs_db != NULL);

		/* Go through hyperscan API */
		if (!rt->cache->vectorized_hyperscan) {
			for (i = 0; i < count; i++) {
				cbdata.ins = &in[i];
				cbdata.re = re;
				cbdata.rt = rt;
				cbdata.lens = &lens[i];
				cbdata.count = 1;
				cbdata.pool = pool;

				if ((hs_scan (re_class->hs_db, in[i], lens[i], 0,
						re_class->hs_scratch,
						rspamd_re_cache_hyperscan_cb, &cbdata)) != HS_SUCCESS) {
					ret = 0;
				}
				else {
					ret = rt->results[re_id];
				}
			}
		}
		else {
			cbdata.ins = in;
			cbdata.re = re;
			cbdata.rt = rt;
			cbdata.lens = lens;
			cbdata.count = 1;
			cbdata.pool = pool;

			if ((hs_scan_vector (re_class->hs_db, (const char **)in, lens, count, 0,
					re_class->hs_scratch,
					rspamd_re_cache_hyperscan_cb, &cbdata)) != HS_SUCCESS) {
				ret = 0;
			}
			else {
				ret = rt->results[re_id];
			}
		}
	}
#endif

	return ret;
}

static void
rspamd_re_cache_finish_class (struct rspamd_re_runtime *rt,
		struct rspamd_re_class *re_class)
{
#ifdef WITH_HYPERSCAN
	guint i;
	guint64 re_id;

	/* Set all bits unchecked */
	for (i = 0; i < re_class->nhs; i++) {
		re_id = re_class->hs_ids[i];

		if (!isset (rt->checked, re_id)) {
			g_assert (rt->results[re_id] == 0);
			rt->results[re_id] = 0;
			setbit (rt->checked, re_id);
		}
	}
#endif
}

/*
 * Calculates the specified regexp for the specified class if it's not calculated
 */
static guint
rspamd_re_cache_exec_re (struct rspamd_task *task,
		struct rspamd_re_runtime *rt,
		rspamd_regexp_t *re,
		struct rspamd_re_class *re_class,
		gboolean is_strong)
{
	guint ret = 0, i, re_id;
	GPtrArray *headerlist;
	GList *slist;
	GHashTableIter it;
	struct raw_header *rh;
	const gchar *in, *end;
	const guchar **scvec;
	guint *lenvec;
	gboolean raw = FALSE;
	struct mime_text_part *part;
	struct mime_part *mime_part;
	struct rspamd_url *url;
	struct rspamd_re_cache *cache = rt->cache;
	gpointer k, v;
	guint len, cnt;

	msg_debug_re_cache ("get to the slow path for re type: %s: %s",
			rspamd_re_cache_type_to_string (re_class->type),
			rspamd_regexp_get_pattern (re));
	re_id = rspamd_regexp_get_cache_id (re);

	switch (re_class->type) {
	case RSPAMD_RE_HEADER:
	case RSPAMD_RE_RAWHEADER:
		/* Get list of specified headers */
		headerlist = rspamd_message_get_header_array (task,
				re_class->type_data,
				is_strong);

		if (headerlist) {
			scvec = g_malloc (sizeof (*scvec) * headerlist->len);
			lenvec = g_malloc (sizeof (*lenvec) * headerlist->len);

			for (i = 0; i < headerlist->len; i ++) {
				rh = g_ptr_array_index (headerlist, i);

				if (re_class->type == RSPAMD_RE_RAWHEADER) {
					in = rh->value;
					raw = TRUE;
					lenvec[i] = strlen (rh->value);
				}
				else {
					in = rh->decoded;
					/* Validate input */
					if (!in || !g_utf8_validate (in, -1, &end)) {
						lenvec[i] = 0;
						scvec[i] = (guchar *)"";
						continue;
					}
					lenvec[i] = end - in;
				}

				scvec[i] = (guchar *)in;
			}

			ret = rspamd_re_cache_process_regexp_data (rt, re,
					task->task_pool, scvec, lenvec, headerlist->len, raw);
			debug_task ("checking header %s regexp: %s -> %d",
					re_class->type_data,
					rspamd_regexp_get_pattern (re), ret);
			g_free (scvec);
			g_free (lenvec);
		}
		break;
	case RSPAMD_RE_ALLHEADER:
		raw = TRUE;
		in = task->raw_headers_content.begin;
		len = task->raw_headers_content.len;
		ret = rspamd_re_cache_process_regexp_data (rt, re,
				task->task_pool, (const guchar **)&in, &len, 1, raw);
		debug_task ("checking allheader regexp: %s -> %d",
				rspamd_regexp_get_pattern (re), ret);
		break;
	case RSPAMD_RE_MIMEHEADER:
		headerlist = rspamd_message_get_mime_header_array (task,
				re_class->type_data,
				is_strong);

		if (headerlist) {
			scvec = g_malloc (sizeof (*scvec) * headerlist->len);
			lenvec = g_malloc (sizeof (*lenvec) * headerlist->len);

			for (i = 0; i < headerlist->len; i ++) {
				rh = g_ptr_array_index (headerlist, i);

				if (re_class->type == RSPAMD_RE_RAWHEADER) {
					in = rh->value;
					raw = TRUE;
					lenvec[i] = strlen (rh->value);
				}
				else {
					in = rh->decoded;
					/* Validate input */
					if (!in || !g_utf8_validate (in, -1, &end)) {
						lenvec[i] = 0;
						scvec[i] = (guchar *)"";
						continue;
					}
					lenvec[i] = end - in;
				}

				scvec[i] = (guchar *)in;
			}

			ret = rspamd_re_cache_process_regexp_data (rt, re,
					task->task_pool, scvec, lenvec, headerlist->len, raw);
			debug_task ("checking mime header %s regexp: %s -> %d",
					re_class->type_data,
					rspamd_regexp_get_pattern (re), ret);
			g_free (scvec);
			g_free (lenvec);
		}
		break;
	case RSPAMD_RE_MIME:
	case RSPAMD_RE_RAWMIME:
		/* Iterate through text parts */
		if (task->text_parts->len > 0) {
			cnt = task->text_parts->len;
			scvec = g_malloc (sizeof (*scvec) * cnt);
			lenvec = g_malloc (sizeof (*lenvec) * cnt);

			for (i = 0; i < task->text_parts->len; i++) {
				part = g_ptr_array_index (task->text_parts, i);

				/* Skip empty parts */
				if (IS_PART_EMPTY (part)) {
					lenvec[i] = 0;
					scvec[i] = (guchar *) "";
					continue;
				}

				/* Check raw flags */
				if (!IS_PART_UTF (part)) {
					raw = TRUE;
				}
				/* Select data for regexp */
				if (re_class->type == RSPAMD_RE_RAWMIME) {
					in = part->orig->data;
					len = part->orig->len;
					raw = TRUE;
				}
				else {
					in = part->content->data;
					len = part->content->len;
				}

				scvec[i] = (guchar *) in;
				lenvec[i] = len;
			}

			ret = rspamd_re_cache_process_regexp_data (rt, re,
					task->task_pool, scvec, lenvec, cnt, raw);
			debug_task ("checking mime regexp: %s -> %d",
					rspamd_regexp_get_pattern (re), ret);
			g_free (scvec);
			g_free (lenvec);
		}
		break;
	case RSPAMD_RE_URL:
		cnt = g_hash_table_size (task->urls) + g_hash_table_size (task->emails);

		if (cnt > 0) {
			scvec = g_malloc (sizeof (*scvec) * cnt);
			lenvec = g_malloc (sizeof (*lenvec) * cnt);
			g_hash_table_iter_init (&it, task->urls);
			i = 0;

			while (g_hash_table_iter_next (&it, &k, &v)) {
				url = v;
				in = url->string;
				len = url->urllen;
				raw = FALSE;

				scvec[i] = (guchar *)in;
				lenvec[i++] = len;
			}

			g_hash_table_iter_init (&it, task->emails);

			while (g_hash_table_iter_next (&it, &k, &v)) {
				url = v;
				in = url->string;
				len = url->urllen;
				raw = FALSE;

				scvec[i] = (guchar *) in;
				lenvec[i++] = len;
			}

			g_assert (i == cnt);

			ret = rspamd_re_cache_process_regexp_data (rt, re,
					task->task_pool, scvec, lenvec, i, raw);
			debug_task ("checking url regexp: %s -> %d",
					rspamd_regexp_get_pattern (re), ret);
			g_free (scvec);
			g_free (lenvec);
		}
		break;
	case RSPAMD_RE_BODY:
		raw = TRUE;
		in = task->msg.begin;
		len = task->msg.len;

		ret = rspamd_re_cache_process_regexp_data (rt, re, task->task_pool,
				(const guchar **)&in, &len, 1, raw);
		debug_task ("checking rawbody regexp: %s -> %d",
				rspamd_regexp_get_pattern (re), ret);
		break;
	case RSPAMD_RE_SABODY:
		/*
		 * For SA body we get all parts and extract headers + body from them
		 *
		 */
		if (task->parts->len > 0) {
			cnt = task->parts->len * 2 + 1;
			scvec = g_malloc (sizeof (*scvec) * cnt);
			lenvec = g_malloc (sizeof (*lenvec) * cnt);

			/*
			 * Body rules also include the Subject as the first line
			 * of the body content.
			 */

			slist = rspamd_message_get_header (task, "Subject", FALSE);

			if (slist) {
				rh = slist->data;

				scvec[0] = (guchar *)rh->decoded;
				lenvec[0] = strlen (rh->decoded);
			}
			else {
				scvec[0] = (guchar *)"";
				lenvec[0] = 0;
			}

			for (i = 0; i < task->parts->len; i++) {
				mime_part = g_ptr_array_index (task->parts, i);

				if (mime_part->raw_headers_str) {
					scvec[i * 2 + 1] = (guchar *)mime_part->raw_headers_str;
					lenvec[i * 2 + 1] = strlen (mime_part->raw_headers_str);
				}
				else {
					scvec[i * 2 + 1] = (guchar *)"";
					lenvec[i * 2 + 1] = 0;
				}

				if (mime_part->content) {
					scvec[i * 2 + 2] = (guchar *)mime_part->content->data;
					lenvec[i * 2 + 2] = mime_part->content->len;
				}
				else {
					scvec[i * 2 + 2] = (guchar *)"";
					lenvec[i * 2 + 2] = 0;
				}
			}

			ret = rspamd_re_cache_process_regexp_data (rt, re,
					task->task_pool, scvec, lenvec, cnt, TRUE);
			debug_task ("checking sa body regexp: %s -> %d",
					rspamd_regexp_get_pattern (re), ret);
			g_free (scvec);
			g_free (lenvec);
		}
		break;
	case RSPAMD_RE_MAX:
		msg_err_task ("regexp of class invalid has been called: %s",
				rspamd_regexp_get_pattern (re));
		break;
	}

#if WITH_HYPERSCAN
	if (!rt->cache->disable_hyperscan) {
		rspamd_re_cache_finish_class (rt, re_class);
	}
#endif

	setbit (rt->checked, re_id);

	return rt->results[re_id];
}

gint
rspamd_re_cache_process (struct rspamd_task *task,
		struct rspamd_re_runtime *rt,
		rspamd_regexp_t *re,
		enum rspamd_re_type type,
		gpointer type_data,
		gsize datalen,
		gboolean is_strong)
{
	guint64 re_id;
	struct rspamd_re_class *re_class;
	struct rspamd_re_cache *cache;

	g_assert (rt != NULL);
	g_assert (task != NULL);
	g_assert (re != NULL);

	cache = rt->cache;
	re_id = rspamd_regexp_get_cache_id (re);

	if (re_id == RSPAMD_INVALID_ID || re_id > cache->nre) {
		msg_err_task ("re '%s' has no valid id for the cache",
				rspamd_regexp_get_pattern (re));
		return 0;
	}

	if (isset (rt->checked, re_id)) {
		/* Fast path */
		rt->stat.regexp_fast_cached ++;
		return rt->results[re_id];
	}
	else {
		/* Slow path */
		re_class = rspamd_regexp_get_class (re);

		if (re_class == NULL) {
			msg_err_task ("cannot find re class for regexp '%s'",
					rspamd_regexp_get_pattern (re));
			return 0;
		}

		return rspamd_re_cache_exec_re (task, rt, re, re_class,
				is_strong);
	}

	return 0;
}

void
rspamd_re_cache_runtime_destroy (struct rspamd_re_runtime *rt)
{
	g_assert (rt != NULL);

	g_slice_free1 (NBYTES (rt->cache->nre), rt->checked);
	g_slice_free1 (rt->cache->nre, rt->results);
	REF_RELEASE (rt->cache);
	g_slice_free1 (sizeof (*rt), rt);
}

void
rspamd_re_cache_unref (struct rspamd_re_cache *cache)
{
	if (cache) {
		REF_RELEASE (cache);
	}
}

struct rspamd_re_cache *
rspamd_re_cache_ref (struct rspamd_re_cache *cache)
{
	if (cache) {
		REF_RETAIN (cache);
	}

	return cache;
}

guint
rspamd_re_cache_set_limit (struct rspamd_re_cache *cache, guint limit)
{
	guint old;

	g_assert (cache != NULL);

	old = cache->max_re_data;
	cache->max_re_data = limit;

	return old;
}

const gchar *
rspamd_re_cache_type_to_string (enum rspamd_re_type type)
{
	const gchar *ret = "unknown";

	switch (type) {
	case RSPAMD_RE_HEADER:
		ret = "header";
		break;
	case RSPAMD_RE_RAWHEADER:
		ret = "raw header";
		break;
	case RSPAMD_RE_MIMEHEADER:
		ret = "mime header";
		break;
	case RSPAMD_RE_ALLHEADER:
		ret = "all headers";
		break;
	case RSPAMD_RE_MIME:
		ret = "part";
		break;
	case RSPAMD_RE_RAWMIME:
		ret = "raw part";
		break;
	case RSPAMD_RE_BODY:
		ret = "rawbody";
		break;
	case RSPAMD_RE_URL:
		ret = "url";
		break;
	case RSPAMD_RE_SABODY:
		ret = "sa body";
		break;
	case RSPAMD_RE_MAX:
		ret = "invalid class";
		break;
	}

	return ret;
}

enum rspamd_re_type
rspamd_re_cache_type_from_string (const char *str)
{
	enum rspamd_re_type ret;
	guint64 h;

	/*
	 * To optimize this function, we apply hash to input string and
	 * pre-select it from the values
	 */

	if (str != NULL) {
		h = XXH64 (str, strlen (str), 0xdeadbabe);

		switch (h) {
		case 0x298b9c8a58887d44LLU:
			ret = RSPAMD_RE_HEADER;
			break;
		case 0x467bfb5cd7ddf890LLU:
			ret = RSPAMD_RE_RAWHEADER;
			break;
		case 0xda081341fb600389LLU:
			ret = RSPAMD_RE_MIME;
			break;
		case 0xc35831e067a8221dLLU:
			ret = RSPAMD_RE_RAWMIME;
			break;
		case 0xc625e13dbe636de2LLU:
			ret = RSPAMD_RE_BODY;
			break;
		case 0x286edbe164c791d2LLU:
			ret = RSPAMD_RE_URL;
			break;
		case 0x796d62205a8778c7LLU:
			ret = RSPAMD_RE_ALLHEADER;
			break;
		case 0xa3c6c153b3b00a5eLLU:
			ret = RSPAMD_RE_MIMEHEADER;
			break;
		case 0x7794501506e604e9LLU:
			ret = RSPAMD_RE_SABODY;
			break;
		default:
			ret = RSPAMD_RE_MAX;
			break;
		}
	}
	else {
		ret = RSPAMD_RE_MAX;
	}

	return ret;
}

#ifdef WITH_HYPERSCAN
static gboolean
rspamd_re_cache_is_finite (struct rspamd_re_cache *cache,
		rspamd_regexp_t *re, gint flags, gdouble max_time)
{
	pid_t cld;
	gint status;
	struct timespec ts;
	hs_compile_error_t *hs_errors;
	hs_database_t *test_db;
	gdouble wait_time;
{'fraction_non_alphanumeric': 0.13337013819632426, 'fraction_numerical': 0.007942726884171535, 'mean_word_length': 3.7692373025310006, 'pattern_counts': {'":': 0, '<': 39, '<?xml version=': 0, '>': 551, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2326613', 'n_tokens_mistral': 26229, 'n_tokens_neox': 23324, 'n_words': 5033}