<commit_before>-- Work in progress -- Do not use this yet --

-- Set of void elements
-- whatwg.org/specs/web-apps/current-work/multipage/syntax.html#void-elements
local void = {
    area = true,
    base = true,
    br = true,
    col = true,
    embed = true,
    hr = true,
    img = true,
    input = true,
    keygen = true,
    link = true,
    menuitem = true,
    meta = true,
    param = true,
    source = true,
    track = true,
    wbr = true
}

local entity_map = {
    ["&"] = "&amp;",
    ["<"] = "&lt;",
    [">"] = "&gt;",
    ['"'] = "&quot;",
    ["'"] = "&#x27;",
    ["/"] = "&#x2F;"
}

local function escape(s)
    return string.gsub(s, "[&<>\"'/]", entity_map)
end

-- Generates a string of spaces for a given level of indentation and
-- memoizes it to avoid creating excess garbage.
local indent = setmetatable({[0] = "", [1] = "    "}, {
    __index = function(self, i)
        self[i] = string.rep(self[1], i)
        return self[i]
    end
})

local function new_rope()
    local methods = {
        append = function(self, str)
            self.n = self.n + 1
            self[self.n] = str
        end,
        appendf = function(self, fmt, ...)
            self.n = self.n + 1
            self[self.n] = string.format(fmt, ...)
        end,
        concat = function(self)
            return table.concat(self)
        end,
        __length = function(self)
            return self.n
        end
    }
    return setmetatable({n = 0}, {__index = methods})
end

local function to_html(node)
    local rope = new_rope()
    local level = 0

    local function serialize(node)
        if node.type == "element" then
            local length = #node
            -- Add start tag and attributes
            rope:appendf('%s<%s', indent[level], node.tag)
            for name, value in pairs(node.attr or {}) do
                rope:appendf(' %s="%s"', name, value:gsub('"', "&quot;"))
            end

            if length > 0 then -- recurse into child nodes
                rope:append(">\n")
                level = level + 1
                for i = 1, length do
                    serialize(node[i])
                end
                level = level - 1
                if not void[node.tag] then
                    rope:appendf("%s</%s>\n", indent[level], node.tag)
                end
            else
                rope:append(">")
                if not void[node.tag] then
                    rope:appendf("</%s>\n", node.tag)
                else
                    rope:append("\n")
                end
            end
        elseif node.type == "text" then
            rope:appendf('%s%s\n', indent[level], escape(node.text))
        elseif node.type == "comment" then
            rope:appendf('%s<!--%s-->\n', indent[level], node.text)
        elseif node.type == "document" then
            if node.has_doctype == true then
                rope:appendf("<!doctype %s>\n", node.name)
            end
            for i = 1, #node do
                serialize(node[i])
            end
        end
    end

    serialize(node)
    return rope:concat()
end

local function to_table(node)
    local rope = new_rope()
    local level = 0

    function rope:append_qpair(indent, name, value)
        local escval = value:gsub('[\n\t"]', {
            ["\n"] = "\\n",
            ["\t"] = "\\t",
            ['"'] = '\\"',
        })
        self:appendf('%s%s = "%s",\n', indent, name, escval)
    end

    -- TODO: This code is ugly as sin. Refactor it.
    local function serialize(node)
        if node.type == "element" then
            rope:appendf("%s{\n", indent[level])
            level = level + 1
            rope:append_qpair(indent[level], "type", "element")
            rope:append_qpair(indent[level], "tag", node.tag)
            -- TODO: Add attributes
            for i = 1, #node do serialize(node[i], i) end
            level = level - 1
            rope:appendf("%s},\n", indent[level])
        elseif node.text then
            local i1, i2 = indent[level], indent[level+1]
            rope:appendf("%s{\n", i1)
            rope:append_qpair(i2, "type", node.type)
            rope:append_qpair(i2, "text", node.text)
            rope:appendf("%s%s = %d,\n", i2, "line", node.line)
            rope:appendf("%s%s = %d,\n", i2, "column", node.column)
            rope:appendf("%s%s = %d,\n", i2, "offset", node.offset)
            rope:appendf("%s},\n", i1)
        elseif node.type == "document" then
            rope:append("{\n")
            level = level + 1
            local i = indent[level]
            rope:append_qpair(i, "type", "document")
            rope:appendf('%s%s = %s,\n', i, "has_doctype", node.has_doctype)
            rope:append_qpair(i, "name", node.name)
            rope:append_qpair(i, "system_identifier", node.system_identifier)
            rope:append_qpair(i, "public_identifier", node.public_identifier)
            rope:append_qpair(i, "quirks_mode", node.quirks_mode)
            for i = 1, #node do serialize(node[i]) end
            level = level - 1
            rope:append("}\n")
        end
    end

    serialize(node)
    return rope:concat()
end
<commit_msg>Include attributes in table serialization<commit_after>-- Work in progress -- Do not use this yet --

-- Set of void elements
-- whatwg.org/specs/web-apps/current-work/multipage/syntax.html#void-elements
local void = {
    area = true,
    base = true,
    br = true,
    col = true,
    embed = true,
    hr = true,
    img = true,
    input = true,
    keygen = true,
    link = true,
    menuitem = true,
    meta = true,
    param = true,
    source = true,
    track = true,
    wbr = true
}

local entity_map = {
    ["&"] = "&amp;",
    ["<"] = "&lt;",
    [">"] = "&gt;",
    ['"'] = "&quot;",
    ["'"] = "&#x27;",
    ["/"] = "&#x2F;"
}

local function escape(s)
    return string.gsub(s, "[&<>\"'/]", entity_map)
end

-- Generates a string of spaces for a given level of indentation and
-- memoizes it to avoid creating excess garbage.
local indent = setmetatable({[0] = "", [1] = "    "}, {
    __index = function(self, i)
        self[i] = string.rep(self[1], i)
        return self[i]
    end
})

local function new_rope()
    local methods = {
        append = function(self, str)
            self.n = self.n + 1
            self[self.n] = str
        end,
        appendf = function(self, fmt, ...)
            self.n = self.n + 1
            self[self.n] = string.format(fmt, ...)
        end,
        concat = function(self)
            return table.concat(self)
        end,
        __length = function(self)
            return self.n
        end
    }
    return setmetatable({n = 0}, {__index = methods})
end

local function to_html(node)
    local rope = new_rope()
    local level = 0

    local function serialize(node)
        if node.type == "element" then
            local length = #node
            -- Add start tag and attributes
            rope:appendf('%s<%s', indent[level], node.tag)
            for name, value in pairs(node.attr or {}) do
                rope:appendf(' %s="%s"', name, value:gsub('"', "&quot;"))
            end

            if length > 0 then -- recurse into child nodes
                rope:append(">\n")
                level = level + 1
                for i = 1, length do
                    serialize(node[i])
                end
                level = level - 1
                if not void[node.tag] then
                    rope:appendf("%s</%s>\n", indent[level], node.tag)
                end
            else
                rope:append(">")
                if not void[node.tag] then
                    rope:appendf("</%s>\n", node.tag)
                else
                    rope:append("\n")
                end
            end
        elseif node.type == "text" then
            rope:appendf('%s%s\n', indent[level], escape(node.text))
        elseif node.type == "comment" then
            rope:appendf('%s<!--%s-->\n', indent[level], node.text)
        elseif node.type == "document" then
            if node.has_doctype == true then
                rope:appendf("<!doctype %s>\n", node.name)
            end
            for i = 1, #node do
                serialize(node[i])
            end
        end
    end

    serialize(node)
    return rope:concat()
end

local function to_table(node)
    local rope = new_rope()
    local level = 0

    function rope:append_qpair(indent, name, value)
        local escval = value:gsub('[\n\t"]', {
            ["\n"] = "\\n",
            ["\t"] = "\\t",
            ['"'] = '\\"',
        })
        self:appendf('%s%s = "%s",\n', indent, name, escval)
    end

    -- TODO: This code is ugly as sin. Refactor it.
    local function serialize(node)
        if node.type == "element" then
            rope:appendf("%s{\n", indent[level])
            level = level + 1
            rope:append_qpair(indent[level], "type", "element")
            rope:append_qpair(indent[level], "tag", node.tag)
            if node.attr then -- add attributes
                local indent1, indent2 = indent[level], indent[level+1]
                rope:appendf("%sattr = {\n", indent1)
                for name, value in pairs(node.attr) do
                    rope:append_qpair(indent2, name, value)
                end
                rope:appendf("%s},\n", indent1)
            end
            for i = 1, #node do -- add children
                serialize(node[i], i)
            end
            level = level - 1
            rope:appendf("%s},\n", indent[level])
        elseif node.text then
            local i1, i2 = indent[level], indent[level+1]
            rope:appendf("%s{\n", i1)
            rope:append_qpair(i2, "type", node.type)
            rope:append_qpair(i2, "text", node.text)
            rope:appendf("%s%s = %d,\n", i2, "line", node.line)
            rope:appendf("%s%s = %d,\n", i2, "column", node.column)
            rope:appendf("%s%s = %d,\n", i2, "offset", node.offset)
            rope:appendf("%s},\n", i1)
        elseif node.type == "document" then
            rope:append("{\n")
            level = level + 1
            local i = indent[level]
            rope:append_qpair(i, "type", "document")
            rope:appendf('%s%s = %s,\n', i, "has_doctype", node.has_doctype)
            rope:append_qpair(i, "name", node.name)
            rope:append_qpair(i, "system_identifier", node.system_identifier)
            rope:append_qpair(i, "public_identifier", node.public_identifier)
            rope:append_qpair(i, "quirks_mode", node.quirks_mode)
            for i = 1, #node do serialize(node[i]) end
            level = level - 1
            rope:append("}\n")
        end
    end

    serialize(node)
    return rope:concat()
end

{'fraction_non_alphanumeric': 0.15015378879671917, 'fraction_numerical': 0.006524373194146705, 'mean_word_length': 1.6671638081034055, 'pattern_counts': {'":': 0, '<': 17, '<?xml version=': 0, '>': 21, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6231224', 'n_tokens_mistral': 3436, 'n_tokens_neox': 3394, 'n_words': 839}