<commit_before>-- HTML writer for lunamark

local gsub = string.gsub

local firstline = true

local formats = {}

local Html = {}

Html.options = { minimize   = false,
                 blanklines = true,
                 containers = false }

-- override string.format so that %n is a variable
-- newline (depending on the 'minimize' option).
-- formats are memoized after conversion.
local function format(fmt,...)
  local newfmt = formats[fmt]
  if not newfmt then
    newfmt = fmt
    local nl = "\n"
    if Html.options.minimize then nl = "" end
    newfmt = newfmt:gsub("\n",nl)
    local starts_with_nl = newfmt:byte(1) == 10
    if starts_with_nl and not Html.options.blanklines then
      newfmt  = newfmt:sub(2)
      starts_with_nl = false
    end
    formats[fmt] = newfmt
    -- don't memoize this change, just on first line
    if starts_with_nl and Html.options.firstline then
      newfmt = newfmt:sub(2)
      firstline = false
    end
  end
  return string.format(newfmt,...)
end

Html.format = format

Html.linebreak = "<br/>"

Html.space = " "

function Html.string(s)
  local escaped = {
   ["<" ] = "&lt;",
   [">" ] = "&gt;",
   ["&" ] = "&amp;",
   ["\"" ] = "&quot;",
   ["'" ] = "&#39;" }
  return s:gsub(".",escaped)
end

function Html.code(s)<commit_msg>Reduce use of gsub in format in html writer.<commit_after>-- HTML writer for lunamark

local gsub = string.gsub

local firstline = true

local formats = {}

local Html = {}

Html.options = { minimize   = false,
                 blanklines = true,
                 containers = false }

-- override string.format so that \n is a variable
-- newline (depending on the 'minimize' option).
-- formats are memoized after conversion.
local function format(fmt,...)
  local newfmt = formats[fmt]
  if not newfmt then
    newfmt = fmt
    if Html.options.minimize then
      newfmt = newfmt:gsub("\n","")
    end
    local starts_with_nl = newfmt:byte(1) == 10
    if starts_with_nl and not Html.options.blanklines then
      newfmt = newfmt:sub(2)
      starts_with_nl = false
    end
    formats[fmt] = newfmt
    -- don't memoize this change, just on first line
    if starts_with_nl and Html.options.firstline then
      newfmt = newfmt:sub(2)
      firstline = false
    end
  end
  return string.format(newfmt,...)
end

Html.format = format

Html.linebreak = "<br/>"

Html.space = " "

function Html.string(s)
  local escaped = {
   ["<" ] = "&lt;",
   [">" ] = "&gt;",
   ["&" ] = "&amp;",
   ["\"" ] = "&quot;",
   ["'" ] = "&#39;" }
  return s:gsub(".",escaped)
end

function Html.code(s)
{'fraction_non_alphanumeric': 0.14085603112840467, 'fraction_numerical': 0.005447470817120622, 'mean_word_length': 2.62112676056338, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6578777', 'n_tokens_mistral': 894, 'n_tokens_neox': 884, 'n_words': 245}