<commit_before>--- Memory management and DPDK mempools
local mod = {}

local libmoon  = require "libmoon"
local ffi     = require "ffi"
local dpdkc   = require "dpdkc"
local dpdk    = require "dpdk"
local ns      = require "namespaces"
local serpent = require "Serpent"
local log     = require "log"
local libmoon  = require "libmoon"

local function loadAllocator()
	local ok, jemalloc = pcall(ffi.load, "libjemalloc.so")
	if ok then
		return jemalloc, "jemalloc"
	end
	local ok, jemalloc = pcall(ffi.load, "libjemalloc.so.1")
	if ok then
		return jemalloc, "jemalloc"
	end
	return ffi.C, "system malloc"
end

local allocator, allocatorName = loadAllocator()

ffi.cdef [[
	void* malloc(size_t size);
	void free(void* buf);
	void* alloc_huge(size_t size);
	int free_huge(void* ptr, size_t size);
]]

local C = ffi.C
local cast = ffi.cast

local warningShown = false
local function testMalloc(size)
	local mem = allocator.malloc(size)
	local ptr = ffi.cast("uintptr_t", mem)
	if ptr < 2^32 and not warningShown then
		if size < 2^20 then
			log:info("malloc() allocates small objects < 1 MiB from LuaJIT memory space, install libjemalloc if you encounter out of memory errors.")
		else
			log:warn("malloc() allocates objects >= 1 MiB from LuaJIT memory space.")<commit_msg>add support for libjemalloc.so.2<commit_after>--- Memory management and DPDK mempools
local mod = {}

local libmoon  = require "libmoon"
local ffi     = require "ffi"
local dpdkc   = require "dpdkc"
local dpdk    = require "dpdk"
local ns      = require "namespaces"
local serpent = require "Serpent"
local log     = require "log"
local libmoon  = require "libmoon"

local function loadAllocator()
	local ok, jemalloc = pcall(ffi.load, "libjemalloc.so")
	if ok then
		return jemalloc, "jemalloc"
	end
	local ok, jemalloc = pcall(ffi.load, "libjemalloc.so.1")
	if ok then
		return jemalloc, "jemalloc"
	end
	local ok, jemalloc = pcall(ffi.load, "libjemalloc.so.2")
	if ok then
		return jemalloc, "jemalloc"
	end
	return ffi.C, "system malloc"
end

local allocator, allocatorName = loadAllocator()

ffi.cdef [[
	void* malloc(size_t size);
	void free(void* buf);
	void* alloc_huge(size_t size);
	int free_huge(void* ptr, size_t size);
]]

local C = ffi.C
local cast = ffi.cast

local warningShown = false
local function testMalloc(size)
	local mem = allocator.malloc(size)
	local ptr = ffi.cast("uintptr_t", mem)
	if ptr < 2^32 and not warningShown then
		if size < 2^20 then
			log:info("malloc() allocates small objects < 1 MiB from LuaJIT memory space, install libjemalloc if you encounter out of memory errors.")
		else
			log:warn("malloc() allocates objects >= 1 MiB from LuaJIT memory space.")
{'fraction_non_alphanumeric': 0.10449438202247191, 'fraction_numerical': 0.00749063670411985, 'mean_word_length': 4.2066276803118905, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1722879', 'n_tokens_mistral': 984, 'n_tokens_neox': 946, 'n_words': 314}