<commit_before>
function table_last(t)
	return t[#t]
end

function split_path(path)
	local name, extension = string.match(path, "^(%.?%.?.*)%.([^%.\\/]*)$")
	return (name or path), extension
end

function dirname(path)
	path = trim_trailing_slash(path)
	local dir, _ = string.match(path, "^(.+)/([^/]*)$")
	return dir or path
end

function trim_trailing_slash(path)
	if string.sub(path, -1) == "/" then
		path = string.sub(path, 1, -2)
	end
	return path
end

function make_inverse_table(t, value)
	local it = {}
	for k, v in pairs(t) do
		it[v] = value or k
	end
	return it
end

function printf(msg, ...)
	print(string.format(msg, ...))
end

function iterate_dir(dir, select_only, max_depth)
	require("lfs")
	assert(dir and dir ~= "", "directory parameter is missing or empty")
	dir = trim_trailing_slash(dir)

	local function yield_tree(base, path, depth)
		if max_depth ~= nil and depth > max_depth then
			return
		end
		for entry in lfs.dir(base .. path) do
			if entry ~= "." and entry ~= ".." then
				entry = path .. "/" .. entry
				local attr = lfs.attributes(base .. entry)
				if select_only == nil or attr.mode == select_only then
					coroutine.yield(string.sub(entry, 2), attr)
				end
				if attr.mode == "directory" then
					yield_tree(base, entry, depth + 1)
				end
			end
		end
	end

	return coroutine.wrap(
		function()
			yield_tree(dir, "", 1)
		end
	)
end

function togo_libraries()
	return {
		"core",
		"image",
		"window",
		"game",
	}
end

function togo_tools()
	return {
		"res_build",
	}
end
<commit_msg>scripts/common: use math.huge when max_depth is nil.<commit_after>
function table_last(t)
	return t[#t]
end

function split_path(path)
	local name, extension = string.match(path, "^(%.?%.?.*)%.([^%.\\/]*)$")
	return (name or path), extension
end

function dirname(path)
	path = trim_trailing_slash(path)
	local dir, _ = string.match(path, "^(.+)/([^/]*)$")
	return dir or path
end

function trim_trailing_slash(path)
	if string.sub(path, -1) == "/" then
		path = string.sub(path, 1, -2)
	end
	return path
end

function make_inverse_table(t, value)
	local it = {}
	for k, v in pairs(t) do
		it[v] = value or k
	end
	return it
end

function printf(msg, ...)
	print(string.format(msg, ...))
end

function iterate_dir(dir, select_only, max_depth)
	require("lfs")
	assert(dir and dir ~= "", "directory parameter is missing or empty")
	dir = trim_trailing_slash(dir)
	max_depth = max_depth or math.huge

	local function yield_tree(base, path, depth)
		if depth > max_depth then
			return
		end
		for entry in lfs.dir(base .. path) do
			if entry ~= "." and entry ~= ".." then
				entry = path .. "/" .. entry
				local attr = lfs.attributes(base .. entry)
				if select_only == nil or attr.mode == select_only then
					coroutine.yield(string.sub(entry, 2), attr)
				end
				if attr.mode == "directory" then
					yield_tree(base, entry, depth + 1)
				end
			end
		end
	end

	return coroutine.wrap(
		function()
			yield_tree(dir, "", 1)
		end
	)
end

function togo_libraries()
	return {
		"core",
		"image",
		"window",
		"game",
	}
end

function togo_tools()
	return {
		"res_build",
	}
end

{'fraction_non_alphanumeric': 0.15054452274183217, 'fraction_numerical': 0.003843689942344651, 'mean_word_length': 3.392405063291139, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1259444', 'n_tokens_mistral': 1269, 'n_tokens_neox': 1149, 'n_words': 323}