<commit_before>--------------------------------------------------------------------------------
-- luatexts/lua.lua: plain Lua implementation of luatexts
--------------------------------------------------------------------------------
-- Copyright (c) 2011-2012, luatexts authors
-- See license in the file named COPYRIGHT
--------------------------------------------------------------------------------

local assert, error, pairs, select, type
    = assert, error, pairs, select, type

local table_concat
    = table.concat

--------------------------------------------------------------------------------

local save
do
  local handlers = { }

  local handle_value = function(cat, v, visited, buf)
    local handler = handlers[type(v)]
    if handler == nil then
      return nil, "can't save `" .. type(v) .. "'"
    end
    return handler(cat, v, { }, buf)
  end

  handlers["nil"] = function(cat, v, visited, buf)
    return cat "-" "\n"
  end

  handlers["boolean"] = function(cat, v, visited, buf)
    return cat (v and "1" or "0") "\n"
  end

  handlers["number"] = function(cat, v, visited, buf)
    return cat "N" "\n" (("%.54g"):format(v)) "\n"
  end

  handlers["string"] = function(cat, v, visited, buf)
    return cat "S" "\n" (#v) "\n" (v) "\n"
  end

  handlers["table"] = function(cat, t, visited, buf)
    if visited[t] then
      -- TODO: This should be `return nil, err`, not `error()`!
      error("circular table reference detected")
    end
    visited[t] = true

    cat "T" "\n"

    local array_size = #t
    cat (array_size) "\n"

    local hash_size_pos = #buf + 1
    cat ("?") "\n"

    for i = 1, array_size do
      handle_value(cat, t[i], visited, buf)
    end

    local hash_size = 0
    for k, v in pairs(t) do
      if
        type(k) ~= "number" or
        k > array_size or k < 1 or -- integer key in hash part of the table
        k % 1 ~= 0 -- non-integer key
      then
        hash_size = hash_size + 1
        -- TODO: return nil, err on failure instead of asserting
        assert(handle_value(cat, k, visited, buf))
        assert(handle_value(cat, v, visited, buf))
      end
    end

    buf[hash_size_pos] = hash_size

    visited[t] = nil

    return cat
  end

  save = function(...)
    local nargs = select("#", ...)
    local buf = { }
    local function cat(s) buf[#buf + 1] = s; return cat end

    cat (nargs) "\n"

    for i = 1, nargs do
      handle_value(cat, select(i, ...), { }, buf)
    end

    return table_concat(buf)
  end
end

--------------------------------------------------------------------------------

return
{
  _VERSION = "luatexts-lua 0.1.3";
  _COPYRIGHT = "Copyright (C) 2011-2012, luatexts authors";
  _DESCRIPTION = "Trivial Lua human-readable binary-safe serialization library";
  --
  save = save;
  -- Sorry, no load() (yet). Patches are welcome.
}<commit_msg>lua: save_cat to ease large data streaming<commit_after>--------------------------------------------------------------------------------
-- luatexts/lua.lua: plain Lua implementation of luatexts
--------------------------------------------------------------------------------
-- Copyright (c) 2011-2012, luatexts authors
-- See license in the file named COPYRIGHT
--------------------------------------------------------------------------------

local assert, error, pairs, select, type
    = assert, error, pairs, select, type

local table_concat
    = table.concat

--------------------------------------------------------------------------------

local save, save_cat
do
  local handlers = { }

  local handle_value = function(cat, v, visited, buf)
    local handler = handlers[type(v)]
    if handler == nil then
      return nil, "can't save `" .. type(v) .. "'"
    end
    return handler(cat, v, { }, buf)
  end

  handlers["nil"] = function(cat, v, visited, buf)
    return cat "-" "\n"
  end

  handlers["boolean"] = function(cat, v, visited, buf)
    return cat (v and "1" or "0") "\n"
  end

  handlers["number"] = function(cat, v, visited, buf)
    return cat "N" "\n" (("%.54g"):format(v)) "\n"
  end

  handlers["string"] = function(cat, v, visited, buf)
    return cat "S" "\n" (#v) "\n" (v) "\n"
  end

  handlers["table"] = function(cat, t, visited, buf)
    if visited[t] then
      -- TODO: This should be `return nil, err`, not `error()`!
      error("circular table reference detected")
    end
    visited[t] = true

    cat "T" "\n"

    local array_size = #t
    cat (array_size) "\n"

    local hash_size_pos = buf and #buf + 1 or nil
    cat ("0") "\n" -- Would be left as 0 if save_cat is used

    for i = 1, array_size do
      handle_value(cat, t[i], visited, buf)
    end

    local hash_size = 0
    for k, v in pairs(t) do
      if
        type(k) ~= "number" or
        k > array_size or k < 1 or -- integer key in hash part of the table
        k % 1 ~= 0 -- non-integer key
      then
        hash_size = hash_size + 1
        -- TODO: return nil, err on failure instead of asserting
        assert(handle_value(cat, k, visited, buf))
        assert(handle_value(cat, v, visited, buf))
      end
    end

    if buf then
      buf[hash_size_pos] = hash_size
    end

    visited[t] = nil

    return cat
  end

  local impl = function(buf, cat, ...)
    local nargs = select("#", ...)

    cat (nargs) "\n"

    for i = 1, nargs do
      handle_value(cat, select(i, ...), { }, buf)
    end

    return cat
  end

  save_cat = function(cat, ...)
    return impl(nil, cat, ...)
  end

  save = function(...)
    local buf = { }
    local function cat(s) buf[#buf + 1] = s; return cat end

    impl(buf, cat, ...)

    return table_concat(buf)
  end
end

--------------------------------------------------------------------------------

return
{
  _VERSION = "luatexts-lua 0.1.3";
  _COPYRIGHT = "Copyright (C) 2011-2012, luatexts authors";
  _DESCRIPTION = "Trivial Lua human-readable binary-safe serialization library";
  --
  save = save;
  save_cat = save_cat;
  -- Sorry, no load() (yet). Patches are welcome.
}
{'fraction_non_alphanumeric': 0.2551701134089393, 'fraction_numerical': 0.011007338225483656, 'mean_word_length': 1.596103896103896, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2194009', 'n_tokens_mistral': 1897, 'n_tokens_neox': 1788, 'n_words': 602}