<commit_before>local mod = {}

local log        = require "log"
local namespaces = require "namespaces"
local dpdkc      = require "dpdkc"
local ffi        = require "ffi"
local serpent    = require "Serpent"

mod.config = namespaces:get()
mod.config.appName = "phobos"

local function checkCore()
	if PHOBOS_TASK_NAME ~= "master" then
		log:fatal("This function is only available on the master task.", 2)
	end
end

function mod.setupPaths()
	-- looks like this: ;$BASE/lua/lib/?/init.lua, see task.cpp
	if not mod.config.basePath then
		mod.config.basePath = package.path:match(";([^;]+)/lua/lib/%?/init.lua$") .. "/"
	end
	local base = mod.config.basePath
	package.path = package.path .. ";" .. mod.config.userscript:match("(.-)/([^/]+)$") .. "/?"
	local ffiPath = ""
	for path in package.path:gmatch("([^;]+)") do
		if not path:match("init.lua$") then
			ffiPath = ffiPath .. ";" .. path:gsub("%?%.lua", "?")
		end
	end
	package.ffipath = ffiPath
	local load = ffi.load
	ffi.load = function(name)
		local ok, resLoad = pcall(load, name)
		if ok then
			return resLoad
		end
		-- handle paths, package.searchpath can't do that for us because of the pesky lib prefix
		if name:match("/") then
			local first, second = name:match("(.+)/([^/]+)$")
			name = first .. "/lib" .. second
		else
			name = "lib" .. name
		end
		local path, err =  package.searchpath(name .. ".so", package.ffipath, "/")
		if path then<commit_msg>handle scripts in the current directory properly<commit_after>local mod = {}

local log        = require "log"
local namespaces = require "namespaces"
local dpdkc      = require "dpdkc"
local ffi        = require "ffi"
local serpent    = require "Serpent"

mod.config = namespaces:get()
mod.config.appName = "phobos"

local function checkCore()
	if PHOBOS_TASK_NAME ~= "master" then
		log:fatal("This function is only available on the master task.", 2)
	end
end

function mod.setupPaths()
	-- looks like this: ;$BASE/lua/lib/?/init.lua, see task.cpp
	if not mod.config.basePath then
		mod.config.basePath = package.path:match(";([^;]+)/lua/lib/%?/init.lua$") .. "/"
	end
	local base = mod.config.basePath
	local userscriptPath = mod.config.userscript:match("(.-)/([^/]+)$")
	if userscriptPath then
		package.path = package.path .. ";" .. userscriptPath .. "/?.lua"
	end -- else: script is ./ which is already in the path
	local ffiPath = ""
	for path in package.path:gmatch("([^;]+)") do
		if not path:match("init.lua$") then
			ffiPath = ffiPath .. ";" .. path:gsub("%?%.lua", "?")
		end
	end
	package.ffipath = ffiPath
	local load = ffi.load
	ffi.load = function(name)
		local ok, resLoad = pcall(load, name)
		if ok then
			return resLoad
		end
		-- handle paths, package.searchpath can't do that for us because of the pesky lib prefix
		if name:match("/") then
			local first, second = name:match("(.+)/([^/]+)$")
			name = first .. "/lib" .. second
		else
			name = "lib" .. name
		end
		local path, err =  package.searchpath(name .. ".so", package.ffipath, "/")
		if path then
{'fraction_non_alphanumeric': 0.16738768718801997, 'fraction_numerical': 0.0006655574043261231, 'mean_word_length': 3.275960170697013, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6460314', 'n_tokens_mistral': 1106, 'n_tokens_neox': 1042, 'n_words': 300}