<commit_before>module(..., package.seeall)

function split(fn)
	local dir, file = fn:match("^(.*)[/\\]([^\\/]*)$")
	if not dir then
		return ".", fn
	else
		return dir, file
	end
end

function normalize(fn)
	local pieces = {}
	for piece in string.gmatch(fn, "[^/\\]+") do
		if piece == ".." then
			table.remove(pieces)
		elseif piece == "." then
			-- nop
		else
			pieces[#pieces + 1] = piece
		end
	end
	return table.concat(pieces, '/')
end

function join(dir, fn)
	return normalize(dir .. '/' .. fn)
end

function get_filename_dir(fn)
	return select(2, split(fn))
end

function get_extension(fn)
	return fn:match("(%.[^.]+)$") or ""
end

function drop_suffix(fn)
	return fn:match("^([^.]*)%.[^.]+") or fn<commit_msg>Fixed drop_suffix() was broken for names with multiple dots<commit_after>module(..., package.seeall)

function split(fn)
	local dir, file = fn:match("^(.*)[/\\]([^\\/]*)$")
	if not dir then
		return ".", fn
	else
		return dir, file
	end
end

function normalize(fn)
	local pieces = {}
	for piece in string.gmatch(fn, "[^/\\]+") do
		if piece == ".." then
			table.remove(pieces)
		elseif piece == "." then
			-- nop
		else
			pieces[#pieces + 1] = piece
		end
	end
	return table.concat(pieces, '/')
end

function join(dir, fn)
	return normalize(dir .. '/' .. fn)
end

function get_filename_dir(fn)
	return select(2, split(fn))
end

function get_extension(fn)
	return fn:match("(%.[^.]+)$") or ""
end

function drop_suffix(fn)
	return fn:match("^(.*)%.[^./\\]+$") or fn
{'fraction_non_alphanumeric': 0.20376597175521183, 'fraction_numerical': 0.0026899798251513113, 'mean_word_length': 3.4819277108433737, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5796220', 'n_tokens_mistral': 603, 'n_tokens_neox': 544, 'n_words': 149}