<commit_before>local fallbackQueue = std.object {

  init = function (self, text, fallbacks)
    self.q = {}
    self.fallbacks = fallbacks
    self.text = text
    self.q[1] = { start =1, stop = #text }
    self.q[#(self.q)+1] = { popFallbacks = true }
  end,

  pop = function (self)
    table.remove(self.fallbacks, 1)
    table.remove(self.q, 1)
    self.q[#(self.q)+1] = { popFallbacks = true }
  end,

  shift       = function (self) return table.remove(self.q, 1) end,

  continuing  = function (self) return #self.q > 0 and #self.fallbacks > 0 end,

  currentFont = function (self) return self.fallbacks[1] end,

  currentJob  = function (self) return self.q[1] end,

  lastJob     = function (self) return self.q[#(self.q)] end,

  currentText = function (self) return self.text:sub(self.q[1].start, self.q[1].stop) end,

  addJob = function (self, start, stop)
    self.q[#(self.q)+1] = { start = start, stop = stop }
  end

}

local fontlist = {}

SILE.registerCommand("font:clear-fallbacks", function ()
  fontlist = {}
end)

SILE.registerCommand("font:add-fallback", function (options, _)
  fontlist[#fontlist+1] = options
end)

SILE.shapers.harfbuzzWithFallback = SILE.shapers.harfbuzz {

  shapeToken = function (_, text, options)
    local items = {}
    local optionSet = { options }
    for i = 1, #fontlist do
      local moreOptions = pl.tablex.deepcopy(options)
      for k, v in pairs(fontlist[i]) do moreOptions[k] = v end
      optionSet[#optionSet+1] = moreOptions
    end
    local shapeQueue = fallbackQueue {}
    shapeQueue:init(text, optionSet)
    while shapeQueue:continuing() do
      SU.debug("fonts", "Queue: ".. shapeQueue.q)
      options = shapeQueue:currentFont()
      if not (options.family or options.filename) then return end
      SU.debug("fonts", shapeQueue:currentJob())
      if shapeQueue:currentJob().popFallbacks then shapeQueue:pop()
      else
        local chunk = shapeQueue:currentText()
        SU.debug("fonts", "Trying font '"..options.family.."' for '"..chunk.."'")
        local newItems = SILE.shapers.harfbuzz:shapeToken(chunk, options)
        local startOfNotdefRun = -1<commit_msg>refactor(packages): Replace std.object inheritance with pl.class()<commit_after>local fallbackQueue = pl.class({

    _init = function (self, text, fallbacks)
      self.q = {}
      self.fallbacks = fallbacks
      self.text = text
      self.q[1] = { start =1, stop = #text }
      self.q[#(self.q)+1] = { popFallbacks = true }
    end,

    pop = function (self)
      table.remove(self.fallbacks, 1)
      table.remove(self.q, 1)
      self.q[#(self.q)+1] = { popFallbacks = true }
    end,

    shift       = function (self) return table.remove(self.q, 1) end,

    continuing  = function (self) return #self.q > 0 and #self.fallbacks > 0 end,

    currentFont = function (self) return self.fallbacks[1] end,

    currentJob  = function (self) return self.q[1] end,

    lastJob     = function (self) return self.q[#(self.q)] end,

    currentText = function (self) return self.text:sub(self.q[1].start, self.q[1].stop) end,

    addJob = function (self, start, stop)
      self.q[#(self.q)+1] = { start = start, stop = stop }
    end

  })

local fontlist = {}

SILE.registerCommand("font:clear-fallbacks", function ()
  fontlist = {}
end)

SILE.registerCommand("font:add-fallback", function (options, _)
  fontlist[#fontlist+1] = options
end)

SILE.shapers.harfbuzzWithFallback = SILE.shapers.harfbuzz {

  shapeToken = function (_, text, options)
    local items = {}
    local optionSet = { options }
    for i = 1, #fontlist do
      local moreOptions = pl.tablex.deepcopy(options)
      for k, v in pairs(fontlist[i]) do moreOptions[k] = v end
      optionSet[#optionSet+1] = moreOptions
    end
    local shapeQueue = fallbackQueue(text, optionSet)
    while shapeQueue:continuing() do
      SU.debug("fonts", "Queue: ".. shapeQueue.q)
      options = shapeQueue:currentFont()
      if not (options.family or options.filename) then return end
      SU.debug("fonts", shapeQueue:currentJob())
      if shapeQueue:currentJob().popFallbacks then shapeQueue:pop()
      else
        local chunk = shapeQueue:currentText()
        SU.debug("fonts", "Trying font '"..options.family.."' for '"..chunk.."'")
        local newItems = SILE.shapers.harfbuzz:shapeToken(chunk, options)
        local startOfNotdefRun = -1
{'fraction_non_alphanumeric': 0.1401098901098901, 'fraction_numerical': 0.008241758241758242, 'mean_word_length': 3.180861244019139, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5255934', 'n_tokens_mistral': 1482, 'n_tokens_neox': 1444, 'n_words': 349}