<commit_before>-- Because dkjson is simply much much too slow we need to do this :(

local dcs_remote_export_cache = {}
local dcs_remote_buffer = {}
local dcs_remote_buffer_len = 0
local escape_strings = {}
escape_strings[string.byte("\b")] = "\\b"
escape_strings[string.byte("\f")] = "\\f"
escape_strings[string.byte("\n")] = "\\n"
escape_strings[string.byte("\r")] = "\\r"
escape_strings[string.byte("\t")] = "\\t"
escape_strings[string.byte("\"")] = "\\\""
escape_strings[string.byte("\\")] = "\\\\"
local cached_writers = {}

local write_value

local function append(s)
    dcs_remote_buffer[dcs_remote_buffer_len + 1] = s
    dcs_remote_buffer_len = dcs_remote_buffer_len + 1
end

local function needs_escape(str)
    for idx = 1, #str do
        local char = str:byte(idx)
        if escape_strings[char] then
            return true
        end
    end
    return false
end

local function append_escaped_string(str)

    append("\"")

    if not needs_escape(str) then
        append(str)
    else

        -- Assume DCS at least exports in utf8 format

        LoSetCommand(156)

        for idx = 1, #str do
            local byte = str:byte(idx)
            append(escape_strings[byte] or string.char(byte))
        end
    end

    append("\"")
end

local function append_number(data)
    append(tostring(data))
end

local function append_boolean(data)
    append(data and "true" or "false")
end

local function append_array(data, path)
    local elementPath = path .. ".element"
    local n = #data
    local knownWriter = cached_writers[elementPath]
    append("[")
    for i = 1,n  do
        if i > 1 then append(",") end
        write_value(data[i], elementPath, knownWriter)
    end
    append("]")
end<commit_msg>Removed excess LoSetCommand<commit_after>-- Because dkjson is simply much much too slow we need to do this :(

local dcs_remote_export_cache = {}
local dcs_remote_buffer = {}
local dcs_remote_buffer_len = 0
local escape_strings = {}
escape_strings[string.byte("\b")] = "\\b"
escape_strings[string.byte("\f")] = "\\f"
escape_strings[string.byte("\n")] = "\\n"
escape_strings[string.byte("\r")] = "\\r"
escape_strings[string.byte("\t")] = "\\t"
escape_strings[string.byte("\"")] = "\\\""
escape_strings[string.byte("\\")] = "\\\\"
local cached_writers = {}

local write_value

local function append(s)
    dcs_remote_buffer[dcs_remote_buffer_len + 1] = s
    dcs_remote_buffer_len = dcs_remote_buffer_len + 1
end

local function needs_escape(str)
    for idx = 1, #str do
        local char = str:byte(idx)
        if escape_strings[char] then
            return true
        end
    end
    return false
end

local function append_escaped_string(str)

    append("\"")

    if not needs_escape(str) then
        append(str)
    else

        -- Assume DCS at least exports in utf8 format

        for idx = 1, #str do
            local byte = str:byte(idx)
            append(escape_strings[byte] or string.char(byte))
        end
    end

    append("\"")
end

local function append_number(data)
    append(tostring(data))
end

local function append_boolean(data)
    append(data and "true" or "false")
end

local function append_array(data, path)
    local elementPath = path .. ".element"
    local n = #data
    local knownWriter = cached_writers[elementPath]
    append("[")
    for i = 1,n  do
        if i > 1 then append(",") end
        write_value(data[i], elementPath, knownWriter)
    end
    append("]")
end
{'fraction_non_alphanumeric': 0.12923790205737468, 'fraction_numerical': 0.0055056505360765, 'mean_word_length': 3.037426900584795, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5892918', 'n_tokens_mistral': 1147, 'n_tokens_neox': 1101, 'n_words': 298}