<commit_before>local cmark = require("cmark")
local yaml = require("yaml")
local Lust = require("lust")

local luacmark = {}

luacmark.version = "0.23"

luacmark.writers = {
  html = function(d, opts, _cols) return cmark.render_html(d, opts) end,
  man = cmark.render_man,
  xml = function(d, opts, _cols) return cmark.render_xml(d, opts) end,
  latex = cmark.render_latex,
  commonmark = cmark.render_commonmark
}


local defaultEnv = _ENV
table.insert(defaultEnv, cmark)

-- Run the specified filter, with source 'source' (a string
-- or function returning chunks) and name 'name', on the cmark
-- node 'doc', with output format 'to'.  The filter modifies
-- 'doc' destructively.  Returns true if successful, otherwise
-- false and an error message.
function luacmark.run_filter(source, name, doc, to)
  local result, msg = load(source, name, 't', defaultEnv)
  if result then
    result()(doc, to)
    return true
  else
    return false, msg
  end
end

--- Find a template and return its contents (or `false` if
-- not found). The template is sought first in the
-- working directory, then in `templates`, then in
-- `$HOME/.lunamark/templates`, then in the Windows
-- `APPDATA` directory.
function luacmark.find_template(name)
  local base, ext = name:match("([^%.]*)(.*)")
  if (not ext or ext == "") and format then ext = "." .. format end
  local fname = base .. ext
  local file = io.open(fname, "r")
  if not file then
    file = io.open("templates/" .. fname, "r")
  end
  if not file then
    local home = os.getenv("HOME")
    if home then
      file = io.open(home .. "/.luacmark/templates/" .. fname, "r")
    end
  end
  if not file then
    local appdata = os.getenv("APPDATA")
    if appdata then
      file = io.open(appdata .. "/luacmark/templates/" .. fname, "r")
    end
  end
  if file then
    return file:read("*all")
  else
    return false, "Could not find template '" .. fname .. "'"
  end
end

-- Render a metadata node in the target format.
local render_metadata = function(node, writer, options, columns)
  local firstblock = cmark.node_first_child(node)
  if cmark.node_get_type(firstblock) == cmark.NODE_PARAGRAPH and
     not cmark.node_next(firstblock) then
     -- render as inlines
     local ils = cmark.node_new(cmark.NODE_CUSTOM_INLINE)
     local b = cmark.node_first_child(firstblock)
     while b do
        local nextb = cmark.node_next(b)
        cmark.node_append_child(ils, b)
        b = nextb
     end
     local result = string.gsub(writer(ils, options, columns), "%s*$", "")
     cmark.node_free(ils)
     return result
  else -- render as blocks
     return writer(node, options, columns)
  end
end

-- Iterate over the metadata, converting to target format.
-- Returns a new table with rendered metadata.
function luacmark.render_metadata(table, writer, options, columns)
  local rendered = {}
  for i, v in pairs(table) do
    if type(v) == 'table' then
        rendered[i] = convert_metadata(v, writer, options, columns)
    else
        local vs = tostring(v)
        rendered[i] = render_metadata(
          cmark.parse_document(vs, string.len(vs), options),
          writer, options, columns)
    end
  end
  return rendered
end

-- Parses document with optional front YAML metadata; returns document,<commit_msg>Use parse_string not parse_document in luacmark.lua.<commit_after>local cmark = require("cmark")
local yaml = require("yaml")
local Lust = require("lust")

local luacmark = {}

luacmark.version = "0.23"

luacmark.writers = {
  html = function(d, opts, _cols) return cmark.render_html(d, opts) end,
  man = cmark.render_man,
  xml = function(d, opts, _cols) return cmark.render_xml(d, opts) end,
  latex = cmark.render_latex,
  commonmark = cmark.render_commonmark
}


local defaultEnv = _ENV
table.insert(defaultEnv, cmark)

-- Run the specified filter, with source 'source' (a string
-- or function returning chunks) and name 'name', on the cmark
-- node 'doc', with output format 'to'.  The filter modifies
-- 'doc' destructively.  Returns true if successful, otherwise
-- false and an error message.
function luacmark.run_filter(source, name, doc, to)
  local result, msg = load(source, name, 't', defaultEnv)
  if result then
    result()(doc, to)
    return true
  else
    return false, msg
  end
end

--- Find a template and return its contents (or `false` if
-- not found). The template is sought first in the
-- working directory, then in `templates`, then in
-- `$HOME/.lunamark/templates`, then in the Windows
-- `APPDATA` directory.
function luacmark.find_template(name)
  local base, ext = name:match("([^%.]*)(.*)")
  if (not ext or ext == "") and format then ext = "." .. format end
  local fname = base .. ext
  local file = io.open(fname, "r")
  if not file then
    file = io.open("templates/" .. fname, "r")
  end
  if not file then
    local home = os.getenv("HOME")
    if home then
      file = io.open(home .. "/.luacmark/templates/" .. fname, "r")
    end
  end
  if not file then
    local appdata = os.getenv("APPDATA")
    if appdata then
      file = io.open(appdata .. "/luacmark/templates/" .. fname, "r")
    end
  end
  if file then
    return file:read("*all")
  else
    return false, "Could not find template '" .. fname .. "'"
  end
end

-- Render a metadata node in the target format.
local render_metadata = function(node, writer, options, columns)
  local firstblock = cmark.node_first_child(node)
  if cmark.node_get_type(firstblock) == cmark.NODE_PARAGRAPH and
     not cmark.node_next(firstblock) then
     -- render as inlines
     local ils = cmark.node_new(cmark.NODE_CUSTOM_INLINE)
     local b = cmark.node_first_child(firstblock)
     while b do
        local nextb = cmark.node_next(b)
        cmark.node_append_child(ils, b)
        b = nextb
     end
     local result = string.gsub(writer(ils, options, columns), "%s*$", "")
     cmark.node_free(ils)
     return result
  else -- render as blocks
     return writer(node, options, columns)
  end
end

-- Iterate over the metadata, converting to target format.
-- Returns a new table with rendered metadata.
function luacmark.render_metadata(table, writer, options, columns)
  local rendered = {}
  for i, v in pairs(table) do
    if type(v) == 'table' then
        rendered[i] = convert_metadata(v, writer, options, columns)
    else
        local vs = tostring(v)
        rendered[i] = render_metadata(cmark.parse_string(vs, options),
                                      writer, options, columns)
    end
  end
  return rendered
end

-- Parses document with optional front YAML metadata; returns document,
{'fraction_non_alphanumeric': 0.11668185961713765, 'fraction_numerical': 0.0009115770282588879, 'mean_word_length': 3.142857142857143, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 4}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4685894', 'n_tokens_mistral': 2110, 'n_tokens_neox': 2094, 'n_words': 698}