<commit_before>/*-
 * Copyright 2016 <NAME>
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *   http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
#include "lua_common.h"
#include "message.h"
#include "libutil/multipattern.h"

/***
 * @module rspamd_trie
 * Rspamd trie module provides the data structure suitable for searching of many
 * patterns in arbitrary texts (or binary chunks). The algorithmic complexity of
 * this algorithm is at most O(n + m + z), where `n` is the length of text, `m` is a length of pattern and `z` is a number of patterns in the text.
 *
 * Here is a typical example of trie usage:
 * @example
local rspamd_trie = require "rspamd_trie"
local patterns = {'aab', 'ab', 'bcd\0ef'}

local trie = rspamd_trie.create(patterns)

local function trie_callback(number, pos)
	print('Matched pattern number ' .. tostring(number) .. ' at pos: ' .. tostring(pos))
end

trie:match('some big text', trie_callback)
 */

/* Suffix trie */
LUA_FUNCTION_DEF (trie, create);
LUA_FUNCTION_DEF (trie, match);
LUA_FUNCTION_DEF (trie, search_mime);
LUA_FUNCTION_DEF (trie, search_rawmsg);
LUA_FUNCTION_DEF (trie, destroy);

static const struct luaL_reg trielib_m[] = {
	LUA_INTERFACE_DEF (trie, match),
	LUA_INTERFACE_DEF (trie, search_mime),
	LUA_INTERFACE_DEF (trie, search_rawmsg),
	{"__tostring", rspamd_lua_class_tostring},
	{"__gc", lua_trie_destroy},
	{NULL, NULL}
};
static const struct luaL_reg trielib_f[] = {
	LUA_INTERFACE_DEF (trie, create),
	{NULL, NULL}
};

static struct rspamd_multipattern *
lua_check_trie (lua_State * L, gint idx)
{
	void *ud = luaL_checkudata (L, 1, "rspamd{trie}");

	luaL_argcheck (L, ud != NULL, 1, "'trie' expected");
	return ud ? *((struct rspamd_multipattern **)ud) : NULL;
}

static gint
lua_trie_destroy (lua_State *L)
{
	struct rspamd_multipattern *trie = lua_check_trie (L, 1);

	if (trie) {
		rspamd_multipattern_destroy (trie);
	}

	return 0;
}

/***
 * function trie.create(patterns)
 * Creates new trie data structure
 * @param {table} array of string patterns
 * @return {trie} new trie object
 */
static gint
lua_trie_create (lua_State *L)
{
	struct rspamd_multipattern *trie, **ptrie;
	gint npat = 0, flags = RSPAMD_MULTIPATTERN_ICASE|RSPAMD_MULTIPATTERN_GLOB;
	GError *err = NULL;

	if (!lua_istable (L, 1)) {
		msg_err ("lua trie expects array of patterns for now");
		lua_pushnil (L);
	}
	else {
		lua_pushvalue (L, 1);
		lua_pushnil (L);

		while (lua_next (L, -2) != 0) {
			if (lua_isstring (L, -1)) {
				npat ++;
			}

			lua_pop (L, 1);
		}

		trie = rspamd_multipattern_create_sized (npat, flags);
		lua_pushnil (L);

		while (lua_next (L, -2) != 0) {
			if (lua_isstring (L, -1)) {
				rspamd_multipattern_add_pattern (trie, lua_tostring (L, -1),
						flags);
			}

			lua_pop (L, 1);
		}

		lua_pop (L, 1); /* table */

		if (!rspamd_multipattern_compile (trie, &err)) {
			msg_err ("cannot compile multipattern: %e", err);
			g_error_free (err);
			rspamd_multipattern_destroy (trie);
			lua_pushnil (L);<commit_msg>[Feature] Allow binary patterns in lua_trie<commit_after>/*-
 * Copyright 2016 <NAME>
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *   http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
#include "lua_common.h"
#include "message.h"
#include "libutil/multipattern.h"

/***
 * @module rspamd_trie
 * Rspamd trie module provides the data structure suitable for searching of many
 * patterns in arbitrary texts (or binary chunks). The algorithmic complexity of
 * this algorithm is at most O(n + m + z), where `n` is the length of text, `m` is a length of pattern and `z` is a number of patterns in the text.
 *
 * Here is a typical example of trie usage:
 * @example
local rspamd_trie = require "rspamd_trie"
local patterns = {'aab', 'ab', 'bcd\0ef'}

local trie = rspamd_trie.create(patterns)

local function trie_callback(number, pos)
	print('Matched pattern number ' .. tostring(number) .. ' at pos: ' .. tostring(pos))
end

trie:match('some big text', trie_callback)
 */

/* Suffix trie */
LUA_FUNCTION_DEF (trie, create);
LUA_FUNCTION_DEF (trie, match);
LUA_FUNCTION_DEF (trie, search_mime);
LUA_FUNCTION_DEF (trie, search_rawmsg);
LUA_FUNCTION_DEF (trie, destroy);

static const struct luaL_reg trielib_m[] = {
	LUA_INTERFACE_DEF (trie, match),
	LUA_INTERFACE_DEF (trie, search_mime),
	LUA_INTERFACE_DEF (trie, search_rawmsg),
	{"__tostring", rspamd_lua_class_tostring},
	{"__gc", lua_trie_destroy},
	{NULL, NULL}
};
static const struct luaL_reg trielib_f[] = {
	LUA_INTERFACE_DEF (trie, create),
	{NULL, NULL}
};

static struct rspamd_multipattern *
lua_check_trie (lua_State * L, gint idx)
{
	void *ud = luaL_checkudata (L, 1, "rspamd{trie}");

	luaL_argcheck (L, ud != NULL, 1, "'trie' expected");
	return ud ? *((struct rspamd_multipattern **)ud) : NULL;
}

static gint
lua_trie_destroy (lua_State *L)
{
	struct rspamd_multipattern *trie = lua_check_trie (L, 1);

	if (trie) {
		rspamd_multipattern_destroy (trie);
	}

	return 0;
}

/***
 * function trie.create(patterns)
 * Creates new trie data structure
 * @param {table} array of string patterns
 * @return {trie} new trie object
 */
static gint
lua_trie_create (lua_State *L)
{
	struct rspamd_multipattern *trie, **ptrie;
	gint npat = 0, flags = RSPAMD_MULTIPATTERN_ICASE|RSPAMD_MULTIPATTERN_GLOB;
	GError *err = NULL;

	if (!lua_istable (L, 1)) {
		msg_err ("lua trie expects array of patterns for now");
		lua_pushnil (L);
	}
	else {
		lua_pushvalue (L, 1);
		lua_pushnil (L);

		while (lua_next (L, -2) != 0) {
			if (lua_isstring (L, -1)) {
				npat ++;
			}

			lua_pop (L, 1);
		}

		trie = rspamd_multipattern_create_sized (npat, flags);
		lua_pushnil (L);

		while (lua_next (L, -2) != 0) {
			if (lua_isstring (L, -1)) {
				const gchar *pat;
				gsize patlen;

				pat = lua_tolstring (L, -1, &patlen);
				rspamd_multipattern_add_pattern_len (trie, pat, patlen, flags);
			}

			lua_pop (L, 1);
		}

		lua_pop (L, 1); /* table */

		if (!rspamd_multipattern_compile (trie, &err)) {
			msg_err ("cannot compile multipattern: %e", err);
			g_error_free (err);
			rspamd_multipattern_destroy (trie);
			lua_pushnil (L);
{'fraction_non_alphanumeric': 0.12239062053188447, 'fraction_numerical': 0.007434944237918215, 'mean_word_length': 4.279245283018868, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '306951', 'n_tokens_mistral': 2726, 'n_tokens_neox': 2576, 'n_words': 785}