<commit_before>local bitshim = require("bitshim")
local utilities = {}

local epsilon = 1E-12

utilities.required = function (options, name, context, _type)
  if not options[name] then utilities.error(context.." needs a "..name.." parameter") end
  if _type then
    return utilities.cast(_type, options[name])
  end
  return options[name]
end

local function preferbool ()
  utilities.warn("Please use boolean values or strings such as 'true' and 'false' instead of 'yes' and 'no'.")
end

utilities.boolean = function (value, default)
  if value == false then return false end
  if value == true then return true end
  if value == "false" then return false end
  if value == "true" then return true end
  if value == "no" then preferbool(); return false end
  if value == "yes" then preferbool(); return true end
  return default
end

utilities.error = function(message, bug)
  utilities.warn(message, bug)
  io.stderr:flush()
  SILE.outputter:finish()
  os.exit(1)
end

utilities.warn = function(message, bug)
  io.stderr:write("\n! " .. message)
  if SILE.traceback or bug then
    io.stderr:write(" at:\n" .. SILE.traceStack:locationTrace())
    io.stderr:write(debug.traceback(nil, 2) or "\t! debug.traceback() did not identify code location")
  else
    io.stderr:write(" at " .. SILE.traceStack:locationHead())
  end
  io.stderr:write("\n")
end

utilities.debugging = function (category)
  return SILE.debugFlags.all or SILE.debugFlags[category]
end

utilities.feq = function (lhs, rhs) -- Float point equal
  lhs = SU.cast("number", lhs)
  rhs = SU.cast("number", rhs)
  local abs = math.abs
  return abs(lhs - rhs) <= epsilon * (abs(lhs) + abs(rhs))
end

utilities.gtoke = function (string, pattern)
  string = string and tostring(string) or ''
  pattern = pattern and tostring(pattern) or "%s+"
  local length = #string
  return coroutine.wrap(function()
    local index = 1
    repeat
      local first, last = string:find(pattern, index)
      if last then
        if index < first then coroutine.yield({ string = string:sub(index, first - 1) }) end
        coroutine.yield({ separator = string:sub(first, last) })
        index = last + 1
      else
        if index <= length then
          coroutine.yield({ string = string:sub(index) })
        end
        break
      end
    until index > length
  end)
end

utilities.deprecated = function (old, new, warnat, errorat, extra)
  -- SILE.version is defined *after* most of SILE loads. It’s available at
  -- runtime but not useful if we encounter deprecated code in core code. Users
  -- will never encounter this failure, but as a developer it’s hard to test a
  -- deprecation when core code refactoring is an all-or-nothing proposition.
  -- Hence we fake it ‘till we make it, all deprecations internally are warings.
  local _semver = SILE.version and SILE.version:match("v([0-9]*.[0-9]*.[0-9]*)") or warnat
  local msg = old .. "() was deprecated in SILE v" .. warnat .. ". Please use " .. new .. "() instead. " .. extra
  if errorat and _semver >= errorat then
    SU.error(msg)
  elseif warnat and _semver >= warnat then
    SU.warn(msg)
  end
end
<commit_msg>chore(utilities): Make deprecation notices more flexible<commit_after>local bitshim = require("bitshim")
local utilities = {}

local epsilon = 1E-12

utilities.required = function (options, name, context, _type)
  if not options[name] then utilities.error(context.." needs a "..name.." parameter") end
  if _type then
    return utilities.cast(_type, options[name])
  end
  return options[name]
end

local function preferbool ()
  utilities.warn("Please use boolean values or strings such as 'true' and 'false' instead of 'yes' and 'no'.")
end

utilities.boolean = function (value, default)
  if value == false then return false end
  if value == true then return true end
  if value == "false" then return false end
  if value == "true" then return true end
  if value == "no" then preferbool(); return false end
  if value == "yes" then preferbool(); return true end
  return default
end

utilities.error = function(message, bug)
  utilities.warn(message, bug)
  io.stderr:flush()
  SILE.outputter:finish()
  os.exit(1)
end

utilities.warn = function(message, bug)
  io.stderr:write("\n! " .. message)
  if SILE.traceback or bug then
    io.stderr:write(" at:\n" .. SILE.traceStack:locationTrace())
    io.stderr:write(debug.traceback(nil, 2) or "\t! debug.traceback() did not identify code location")
  else
    io.stderr:write(" at " .. SILE.traceStack:locationHead())
  end
  io.stderr:write("\n")
end

utilities.debugging = function (category)
  return SILE.debugFlags.all or SILE.debugFlags[category]
end

utilities.feq = function (lhs, rhs) -- Float point equal
  lhs = SU.cast("number", lhs)
  rhs = SU.cast("number", rhs)
  local abs = math.abs
  return abs(lhs - rhs) <= epsilon * (abs(lhs) + abs(rhs))
end

utilities.gtoke = function (string, pattern)
  string = string and tostring(string) or ''
  pattern = pattern and tostring(pattern) or "%s+"
  local length = #string
  return coroutine.wrap(function()
    local index = 1
    repeat
      local first, last = string:find(pattern, index)
      if last then
        if index < first then coroutine.yield({ string = string:sub(index, first - 1) }) end
        coroutine.yield({ separator = string:sub(first, last) })
        index = last + 1
      else
        if index <= length then
          coroutine.yield({ string = string:sub(index) })
        end
        break
      end
    until index > length
  end)
end

utilities.deprecated = function (old, new, warnat, errorat, extra)
  -- SILE.version is defined *after* most of SILE loads. It’s available at
  -- runtime but not useful if we encounter deprecated code in core code. Users
  -- will never encounter this failure, but as a developer it’s hard to test a
  -- deprecation when core code refactoring is an all-or-nothing proposition.
  -- Hence we fake it ‘till we make it, all deprecations internally are warings.
  local brackets = old:sub(1,1) == '\\' and "" or "()"
  local _semver = SILE.version and SILE.version:match("v([0-9]*.[0-9]*.[0-9]*)") or warnat
  local msg = (old .. brackets) .. " was deprecated in SILE v" .. warnat .. ". Please use " .. (new .. brackets) .. " instead. " .. (extra or "")
  if errorat and _semver >= errorat then
    SU.error(msg)
  elseif warnat and _semver >= warnat then
    SU.warn(msg)
  end
end

{'fraction_non_alphanumeric': 0.1223021582733813, 'fraction_numerical': 0.0046918986549890525, 'mean_word_length': 3.344429347826087, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4966372', 'n_tokens_mistral': 2034, 'n_tokens_neox': 1994, 'n_words': 701}