<commit_before>--[[
	Licensed according to the included 'LICENSE' document
	Author: <NAME> <<EMAIL>>
]]
local lpeg = require("lpeg")
local jsonutil = require("json.util")
local util = require("json.decode.util")

-- Container module for other JavaScript types (bool, null, undefined)
module("json.decode.others")

-- For null and undefined, use the util.null value to preserve null-ness
local booleanCapture =
	lpeg.P("true") * lpeg.Cc(true)
	+ lpeg.P("false") * lpeg.Cc(false)

local nullCapture = lpeg.P("null")
local undefinedCapture = lpeg.P("undefined")

local defaultOptions = {
	allowUndefined = true,
	null = jsonutil.null,
	undefined = jsonutil.undefined
}

default = nil -- Let the buildCapture optimization take place
strict = {
	allowUndefined = false
}

local function buildCapture(options)
	options = options and jsonutil.merge({}, defaultOptions, options) or defaultOptions
	local valueCapture = (
		booleanCapture
		+ nullCapture * lpeg.Cc(options.null)
	)
	if options.allowUndefined then
		valueCapture = valueCapture + undefinedCapture * lpeg.Cc(options.undefined)
	end
	return valueCapture
end

function load_types(options, global_options, grammar)
	local capture = buildCapture(options)
	util.append_grammar_item(grammar, "VALUE", capture)
end<commit_msg>decoder: adds support for 'simple' decoding to map null and undefined => nil<commit_after>--[[
	Licensed according to the included 'LICENSE' document
	Author: <NAME> <<EMAIL>>
]]
local lpeg = require("lpeg")
local jsonutil = require("json.util")
local util = require("json.decode.util")

-- Container module for other JavaScript types (bool, null, undefined)
module("json.decode.others")

-- For null and undefined, use the util.null value to preserve null-ness
local booleanCapture =
	lpeg.P("true") * lpeg.Cc(true)
	+ lpeg.P("false") * lpeg.Cc(false)

local nullCapture = lpeg.P("null")
local undefinedCapture = lpeg.P("undefined")

local defaultOptions = {
	allowUndefined = true,
	null = jsonutil.null,
	undefined = jsonutil.undefined
}

default = nil -- Let the buildCapture optimization take place
simple = {
	null = false,     -- Mapped to nil
	undefined = false -- Mapped to nil
}
strict = {
	allowUndefined = false
}

local function buildCapture(options)
	-- The 'or nil' clause allows false to map to a nil value since 'nil' cannot be merged
	options = options and jsonutil.merge({}, defaultOptions, options) or defaultOptions
	local valueCapture = (
		booleanCapture
		+ nullCapture * lpeg.Cc(options.null or nil)
	)
	if options.allowUndefined then
		valueCapture = valueCapture + undefinedCapture * lpeg.Cc(options.undefined or nil)
	end
	return valueCapture
end

function load_types(options, global_options, grammar)
	local capture = buildCapture(options)
	util.append_grammar_item(grammar, "VALUE", capture)
end
{'fraction_non_alphanumeric': 0.10964285714285714, 'fraction_numerical': 0.0, 'mean_word_length': 4.959574468085107, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 10, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4120598', 'n_tokens_mistral': 852, 'n_tokens_neox': 839, 'n_words': 263}