<commit_before>--[[---------------------------------------]
   [ Starfall Library                      ]
   [ By <NAME>                 ]
   [---------------------------------------]]

AddCSLuaFile("SFLib.lua")
SFLib = SFLib or {}

function SFLib.limitString(text, length)
	if #text <= length then
		return text
	else
		return string.sub(text, 1, length) .. "..."
	end
end

-- TODO: Load this
SFLib.builtins = {}

-- Operators
SFLib.optree_inv = {
	add  = "+",
	sub  = "-",
	mul  = "*",
	div  = "/",
	mod  = "%",
	exp  = "^",
	
	ass  = "=",
	aadd = "+=",
	asub = "-=",
	amul = "*=",
	adiv = "/=",
	
	inc  = "++",
	dec  = "--",
	
	eq   = "==",
	neq  = "!=",
	lth  = "<",
	geq  = ">=",
	leq  = "<=",
	gth  = ">",
	
	["not"] = "!",
	["and"] = "&&",
	["or"] = "||",
	
	qsm  = "?",
	col  = ":",
	def  = "?:",
	com  = ",",
	
	lpa  = "(",
	rpa  = ")",
	rcb  = "{",
	lcb  = "}",
	lsb  = "[",
	rsb  = "]",
	
	bwa = "&",
	bwo = "|",
	indx = ".",
}

SFLib.optree = {}
for token,op in pairs(SFLib.optree_inv) do
	local current = SFLib.optree
	for i = 1,#op do
		local c = op:sub(i,i)
		local nxt = current[c]
		if not nxt then
			nxt = {}
			current[c] = nxt
		end

		if i == #op then
			nxt[1] = token
		else<commit_msg>[SF] Fixed bug in SFLib.<commit_after>--[[---------------------------------------]
   [ Starfall Library                      ]
   [ By <NAME>                 ]
   [---------------------------------------]]

AddCSLuaFile("SFLib.lua")
SFLib = SFLib or {}

function SFLib.limitString(text, length)
	if #text <= length then
		return text
	else
		return string.sub(text, 1, length) .. "..."
	end
end

-- TODO: Load this
SFLib.builtins = {}

-- Operators
SFLib.optree_inv = {
	add  = "+",
	sub  = "-",
	mul  = "*",
	div  = "/",
	mod  = "%",
	exp  = "^",
	
	ass  = "=",
	aadd = "+=",
	asub = "-=",
	amul = "*=",
	adiv = "/=",
	
	inc  = "++",
	dec  = "--",
	
	eq   = "==",
	neq  = "!=",
	lth  = "<",
	geq  = ">=",
	leq  = "<=",
	gth  = ">",
	
	["not"] = "!",
	["and"] = "&&",
	["or"] = "||",
	
	qsm  = "?",
	col  = ":",
	def  = "?:",
	com  = ",",
	
	lpa  = "(",
	rpa  = ")",
	rcb  = "}",
	lcb  = "{",
	lsb  = "[",
	rsb  = "]",
	
	bwa = "&",
	bwo = "|",
	indx = ".",
}

SFLib.optree = {}
for token,op in pairs(SFLib.optree_inv) do
	local current = SFLib.optree
	for i = 1,#op do
		local c = op:sub(i,i)
		local nxt = current[c]
		if not nxt then
			nxt = {}
			current[c] = nxt
		end

		if i == #op then
			nxt[1] = token
		else
{'fraction_non_alphanumeric': 0.3042404281597365, 'fraction_numerical': 0.002470152326060107, 'mean_word_length': 1.6355748373101953, 'pattern_counts': {'":': 2, '<': 11, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5538588', 'n_tokens_mistral': 1140, 'n_tokens_neox': 1069, 'n_words': 203}