<commit_before>--[[
	Licensed according to the included 'LICENSE' document
	Author: <NAME> <<EMAIL>>
]]
local lpeg = require("lpeg")
local util = require("json.decode.util")

local tonumber = tonumber
local string = string
local string_char = string.char
local floor = math.floor
local table_concat = table.concat

module("json.decode.strings")
local knownReplacements = {
	n = "\n",
	r = "\r",
	f = "\f",
	t = "\t",
	b = "\b",
	z = "\z",
	['\\'] = "\\",
	['/'] = "/",
	['"'] = '"'
}
-- NOTE: Technically incorrect, correct one will be incorporated eventually
local function nullDecodeUnicode(code1, code2)
	code1, code2 = tonumber(code1, 16), tonumber(code2, 16)
	return string.char(code1, code2)
end

-- according to the table at http://da.wikipedia.org/wiki/UTF-8
local function utf8DecodeUnicode(code1, code2)
	code1, code2 = tonumber(code1, 16), tonumber(code2, 16)
	if code1 == 0 and code2 < 0x80 then
		return string_char(code2)
	end
	if code1 < 0x08 then
		return string_char(
			0xC0 + code1 * 4 + floor(code2 / 64),
			0x80 + code2 % 64)
	end
	return string_char(
		0xE0 + floor(code1 / 16),
		0x80 + (code1 % 16) * 4 + floor(code2 / 64),
		0x80 + code2 % 64)
end

local doSimpleSub = lpeg.C(lpeg.S("rnfbt/\\z\"")) / knownReplacements
local doUniSub = (lpeg.P('u') * lpeg.C(util.hexpair) * lpeg.C(util.hexpair) + lpeg.P(false))
local doSub = doSimpleSub

local defaultOptions = {
	badChars = '',
	additionalEscapes = lpeg.C(1), -- any escape char not handled will be dumped as-is
	escapeCheck = false, -- no check on valid characters
	decodeUnicode = utf8DecodeUnicode,
	strict_quotes = false,
	postProcess = false  -- post-processing for strings after decoding, such as for UTF8 handling
}

default = nil -- Let the buildCapture optimization take place

strict = {
	badChars = '\r\n\f\b\t',
	additionalEscapes = false, -- no additional escapes
	escapeCheck = #lpeg.S('rnfbt/\\"\'u'), --only these chars are allowed to be escaped
	strict_quotes = true
}

local function buildCaptureString(quote, badChars, escapeMatch, postProcess)
	local captureString = lpeg.P(quote) * lpeg.Cs(((1 - lpeg.S("\\" .. badChars .. quote)) + (lpeg.P("\\") / "" * escapeMatch))^0)
	if postProcess then
		captureString = captureString / postProcess
	end
	captureString = captureString * lpeg.P(quote)
	return captureString
end

function buildMatch(options)
	options = options and util.merge({}, defaultOptions, options) or defaultOptions
	local quotes = { '"' }
	if not options.strict_quotes then
		quotes[#quotes + 1] = "'"
	end
	local badChars = options.badChars
	local escapeMatch = doSub
	escapeMatch = escapeMatch + doUniSub / options.decodeUnicode
	if options.additionalEscapes then
		escapeMatch = escapeMatch + options.additionalEscapes
	end
	if options.escapeCheck then
		escapeMatch = options.escapeCheck * escapeMatch
	end
	local captureString
	for i = 1, #quotes do
		local cap = buildCaptureString(quotes[i], badChars, escapeMatch, options.postProcess)
		if captureString == nil then
			captureString = cap
		else
			captureString = captureString + cap
		end<commit_msg>decoder: reorders replacements based on spec, adds 'x' escape<commit_after>--[[
	Licensed according to the included 'LICENSE' document
	Author: <NAME> <<EMAIL>>
]]
local lpeg = require("lpeg")
local util = require("json.decode.util")

local tonumber = tonumber
local string = string
local string_char = string.char
local floor = math.floor
local table_concat = table.concat

module("json.decode.strings")
local knownReplacements = {
	["'"] = "'",
	['"'] = '"',
	['\\'] = '\\',
	['/'] = '/',
	b = '\b',
	f = '\f',
	n = '\n',
	r = '\r',
	t = '\t',
	v = '\v',
	z = '\z'
}
-- NOTE: Technically incorrect, correct one will be incorporated eventually
local function nullDecodeUnicode(code1, code2)
	code1, code2 = tonumber(code1, 16), tonumber(code2, 16)
	return string.char(code1, code2)
end

-- according to the table at http://da.wikipedia.org/wiki/UTF-8
local function utf8DecodeUnicode(code1, code2)
	code1, code2 = tonumber(code1, 16), tonumber(code2, 16)
	if code1 == 0 and code2 < 0x80 then
		return string_char(code2)
	end
	if code1 < 0x08 then
		return string_char(
			0xC0 + code1 * 4 + floor(code2 / 64),
			0x80 + code2 % 64)
	end
	return string_char(
		0xE0 + floor(code1 / 16),
		0x80 + (code1 % 16) * 4 + floor(code2 / 64),
		0x80 + code2 % 64)
end

local function decodeX(code)
	code = tonumber(code, 16)
	return string_char(code)
end

local doSimpleSub = lpeg.C(lpeg.S("'\"\\/bfnrtvz")) / knownReplacements
local doUniSub = (lpeg.P('u') * lpeg.C(util.hexpair) * lpeg.C(util.hexpair) + lpeg.P(false))
local doXSub = (lpeg.P('x') * lpeg.C(util.hexpair))
local doSub = doSimpleSub

local defaultOptions = {
	badChars = '',
	additionalEscapes = lpeg.C(1), -- any escape char not handled will be dumped as-is
	escapeCheck = false, -- no check on valid characters
	decodeUnicode = utf8DecodeUnicode,
	strict_quotes = false,
	postProcess = false  -- post-processing for strings after decoding, such as for UTF8 handling
}

default = nil -- Let the buildCapture optimization take place

strict = {
	badChars = '\b\f\n\r\t\v',
	additionalEscapes = false, -- no additional escapes
	escapeCheck = #lpeg.S('bfnrtv/\\"u'), --only these chars are allowed to be escaped
	strict_quotes = true
}

local function buildCaptureString(quote, badChars, escapeMatch, postProcess)
	local captureString = lpeg.P(quote) * lpeg.Cs(((1 - lpeg.S("\\" .. badChars .. quote)) + (lpeg.P("\\") / "" * escapeMatch))^0)
	if postProcess then
		captureString = captureString / postProcess
	end
	captureString = captureString * lpeg.P(quote)
	return captureString
end

function buildMatch(options)
	options = options and util.merge({}, defaultOptions, options) or defaultOptions
	local quotes = { '"' }
	if not options.strict_quotes then
		quotes[#quotes + 1] = "'"
	end
	local badChars = options.badChars
	local escapeMatch = doSub
	escapeMatch = escapeMatch + doXSub / decodeX
	escapeMatch = escapeMatch + doUniSub / options.decodeUnicode
	if options.additionalEscapes then
		escapeMatch = escapeMatch + options.additionalEscapes
	end
	if options.escapeCheck then
		escapeMatch = options.escapeCheck * escapeMatch
	end
	local captureString
	for i = 1, #quotes do
		local cap = buildCaptureString(quotes[i], badChars, escapeMatch, options.postProcess)
		if captureString == nil then
			captureString = cap
		else
			captureString = captureString + cap
		end
{'fraction_non_alphanumeric': 0.13040762142745588, 'fraction_numerical': 0.02405122598781821, 'mean_word_length': 4.070467141726049, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5802895', 'n_tokens_mistral': 2319, 'n_tokens_neox': 2170, 'n_words': 624}