<commit_before>local byte = string.byte
local format = string.format
local find = string.find
local gsub = string.gsub
local rawequal = rawequal
local tostring = tostring
local type = type
local inf = 1/0

local function encode(v, nullv)
	local i = 1
	local builder = {}
	local visited = {}

	local function f_tostring(v)
		builder[i] = tostring(v)
		i = i+1
	end

	local radixmark = find(tostring(0.5), '[^0-9]')
	local delimmark = find(tostring(123456789.123456789), '[^0-9' .. radixmark .. ']')
	if radixmark == '.' then
		radixmark == nil
	end

	local f_number = f_tostring
	if radixmark or and delimmark then
		if radixmark and find(radixmark, '%W') then
			radixmark = '%' .. radixmark
		end
		if delimmark and find(selimmark, '%W') then
			delimmark = '%' .. delimmark
		end
		f_number = function(n)
			local s = tostring(n)
			if delimmark then
				s = gsub(s, delimmark, '', )
			end
			if radixmark then
				s = gsub(s, radixmark, '.')
			end
			builder[i] = s
			i = i+1
		end
	end

	local dodecode

	local f_string_subst = {
		['"'] = '\\"',
		['\\'] = '\\\\',
		['\b'] = '\\b',
		['\f'] = '\\f',
		['\n'] = '\\n',
		['\r'] = '\\r',
		['\t'] = '\\t',
		__index = function(_, c)
			return format('\\u00%02X', tostring(byte(c)))
		end
	}
	setmetatable(f_string_subst, f_string_subst)

	local function f_string(s)
		builder[i] = '"'
		-- use the cluttered pattern because lua 5.1 does not handle \0 in a pattern correctly
		builder[i+1] = gsub(s, '[^\32-\33\35-\91\93-\255]', f_string_subst)
		builder[i+2] = '"'
		i = i+3
	end

	local function f_table(o)
		if visited[o] then
			error("loop detected")
		end
		visited[o] = true

		local alen = o[0]
		if type(alen) == 'number' then
			builder[i] = '['
			i = i+1
			for j = 1, alen do
				dodecode(o[j])
				builder[i] = ','
				i = i+1
			end
			if alen > 0 then
				i = i-1
			end
			builder[i] = ']'
			i = i+1
			return
		end

		local v = o[1]
		if v ~= nil then
			builder[i] = '['
			i = i+1
			local j = 2
			repeat
				dodecode(v)
				v = o[j]
				if v == nil then
					break
				end
				j = j+1
				builder[i] = ','
				i = i+1
			until false
			builder[i] = ']'
			i = i+1
			return
		end

		builder[i] = '{'
		i = i+1
		local oldi = i
		for k, v in pairs(o) do
			if type(k) ~= 'string' then
				error("non-string key")
			end
			f_string(k)
			builder[i] = ':'
			i = i+1
			dodecode(v)
			builder[i] = ','
			i = i+1
		end
		if i > oldi then
			i = i-1
		end
		builder[i] = '}'
		i = i+1
	end

	local dispatcher = {
		boolean = f_tostring,
		number = f_tostring,
		string = f_string,
		table = arraylen and f_table_arraylen or f_table,
		__index = function()
			error("invalid type value")
		end
	}
	setmetatable(dispatcher, dispatcher)

	function dodecode(v)
		if rawequal(v, nullv) then
			builder[i] = 'null'
			i = i+1
			return
		end
		dispatcher[type(v)](v)
	end

	-- exec
	dodecode(v)
	return table.concat(builder)
end

return {
	encode = encode
}
<commit_msg>shortcut gsub when encoding string<commit_after>local byte = string.byte
local format = string.format
local find = string.find
local gsub = string.gsub
local rawequal = rawequal
local tostring = tostring
local type = type
local inf = 1/0

local function encode(v, nullv)
	local i = 1
	local builder = {}
	local visited = {}

	local function f_tostring(v)
		builder[i] = tostring(v)
		i = i+1
	end

	local radixmark = find(tostring(0.5), '[^0-9]')
	local delimmark = find(tostring(123456789.123456789), '[^0-9' .. radixmark .. ']')
	if radixmark == '.' then
		radixmark == nil
	end

	local f_number = f_tostring
	if radixmark or and delimmark then
		if radixmark and find(radixmark, '%W') then
			radixmark = '%' .. radixmark
		end
		if delimmark and find(selimmark, '%W') then
			delimmark = '%' .. delimmark
		end
		f_number = function(n)
			local s = tostring(n)
			if delimmark then
				s = gsub(s, delimmark, '', )
			end
			if radixmark then
				s = gsub(s, radixmark, '.')
			end
			builder[i] = s
			i = i+1
		end
	end

	local dodecode

	local f_string_subst = {
		['"'] = '\\"',
		['\\'] = '\\\\',
		['\b'] = '\\b',
		['\f'] = '\\f',
		['\n'] = '\\n',
		['\r'] = '\\r',
		['\t'] = '\\t',
		__index = function(_, c)
			return format('\\u00%02X', tostring(byte(c)))
		end
	}
	setmetatable(f_string_subst, f_string_subst)

	local function f_string(s)
		-- use the cluttered pattern because lua 5.1 does not handle \0 in a pattern correctly
		local pat = '[^\32-\33\35-\91\93-\255]'
		builder[i] = '"'
		if find(s, pat) then
			s = gsub(s, pat, f_string_subst)
		end
		builder[i+1] = s
		builder[i+2] = '"'
		i = i+3
	end

	local function f_table(o)
		if visited[o] then
			error("loop detected")
		end
		visited[o] = true

		local alen = o[0]
		if type(alen) == 'number' then
			builder[i] = '['
			i = i+1
			for j = 1, alen do
				dodecode(o[j])
				builder[i] = ','
				i = i+1
			end
			if alen > 0 then
				i = i-1
			end
			builder[i] = ']'
			i = i+1
			return
		end

		local v = o[1]
		if v ~= nil then
			builder[i] = '['
			i = i+1
			local j = 2
			repeat
				dodecode(v)
				v = o[j]
				if v == nil then
					break
				end
				j = j+1
				builder[i] = ','
				i = i+1
			until false
			builder[i] = ']'
			i = i+1
			return
		end

		builder[i] = '{'
		i = i+1
		local oldi = i
		for k, v in pairs(o) do
			if type(k) ~= 'string' then
				error("non-string key")
			end
			f_string(k)
			builder[i] = ':'
			i = i+1
			dodecode(v)
			builder[i] = ','
			i = i+1
		end
		if i > oldi then
			i = i-1
		end
		builder[i] = '}'
		i = i+1
	end

	local dispatcher = {
		boolean = f_tostring,
		number = f_tostring,
		string = f_string,
		table = arraylen and f_table_arraylen or f_table,
		__index = function()
			error("invalid type value")
		end
	}
	setmetatable(dispatcher, dispatcher)

	function dodecode(v)
		if rawequal(v, nullv) then
			builder[i] = 'null'
			i = i+1
			return
		end
		dispatcher[type(v)](v)
	end

	-- exec
	dodecode(v)
	return table.concat(builder)
end

return {
	encode = encode
}

{'fraction_non_alphanumeric': 0.1576296793688098, 'fraction_numerical': 0.023837502098371665, 'mean_word_length': 2.521276595744681, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7099297', 'n_tokens_mistral': 2945, 'n_tokens_neox': 2478, 'n_words': 656}