<commit_before>local ot = SILE.require("core/opentype-parser")

SILE.shapers.harfbuzzWithColor = pl.class({
    _base = SILE.shapers.harfbuzz,

    shapeToken = function (self, str, options)
      if not options.family then return {} end
      local face = SILE.font.cache(options, SILE.shaper.getFace)
      local font = ot.parseFont(face)
      local items = self._base.shapeToken(self, str, options)
      if font.colr and font.cpal then
        local newItems = {}
        for i = 1, #items do
          local layers = font.colr[items[i].gid]
          if layers then
            for j = 1, #layers do
              local item = items[i]
              local layer = layers[j]
              local width = 0
              local text = ""
              if j == #layers then
                width = item.width
                text = item.text
              end
              -- XXX: handle multiple palette, add a font option?
              local color = font.cpal[1][layer.paletteIndex]
              local newItem = {
                gid = layer.gid,
                glyphAdvance = item.glyphAdvance,
                width = width,
                height= item.height,
                depth = item.depth,
                index = item.index,
                text = text,
                color = color,
              }
              newItems[#newItems+1] = newItem
            end
          else
            newItems[#newItems+1] = items[i]
          end
        end
        return newItems
      end
      return items
    end,

    createNnodes = function (self, token, options)
      local items, _ = self:shapeToken(token, options)
      if #items < 1 then return {} end
      local lang = options.language
      SILE.languageSupport.loadLanguage(lang)
      local nodeMaker = SILE.nodeMakers[lang] or SILE.nodeMakers.unicode
      local run = { [1] = { slice = {}, color = items[1].color, chunk = "" } }
      for i = 1, #items do
        if items[i].color ~= run[#run].color then
          run[#run+1] = { slice = {}, chunk = "", color = items[i].color }
          if i <#items then
            run[#run].color = items[i].color
          end<commit_msg>fix(packages): Correctly handle color fonts on TTB pages<commit_after>local ot = SILE.require("core/opentype-parser")

SILE.shapers.harfbuzzWithColor = pl.class({
    _base = SILE.shapers.harfbuzz,

    shapeToken = function (self, str, options)
      if not options.family then return {} end
      local face = SILE.font.cache(options, SILE.shaper.getFace)
      local font = ot.parseFont(face)
      local items = self._base.shapeToken(self, str, options)
      if font.colr and font.cpal then
        local newItems = {}
        for i = 1, #items do
          local layers = font.colr[items[i].gid]
          if layers then
            for j = 1, #layers do
              local item = items[i]
              local layer = layers[j]
              local width = 0
              local height = 0
              local text = ""
              if j == #layers then
                width = item.width
                height = item.height
                text = item.text
              end
              -- XXX: handle multiple palette, add a font option?
              local color = font.cpal[1][layer.paletteIndex]
              local newItem = {
                gid = layer.gid,
                glyphAdvance = item.glyphAdvance,
                width = width,
                height = height,
                depth = item.depth,
                index = item.index,
                text = text,
                color = color,
              }
              newItems[#newItems+1] = newItem
            end
          else
            newItems[#newItems+1] = items[i]
          end
        end
        return newItems
      end
      return items
    end,

    createNnodes = function (self, token, options)
      local items, _ = self:shapeToken(token, options)
      if #items < 1 then return {} end
      local lang = options.language
      SILE.languageSupport.loadLanguage(lang)
      local nodeMaker = SILE.nodeMakers[lang] or SILE.nodeMakers.unicode
      local run = { [1] = { slice = {}, color = items[1].color, chunk = "" } }
      for i = 1, #items do
        if items[i].color ~= run[#run].color then
          run[#run+1] = { slice = {}, chunk = "", color = items[i].color }
          if i <#items then
            run[#run].color = items[i].color
          end
{'fraction_non_alphanumeric': 0.10056497175141244, 'fraction_numerical': 0.00519774011299435, 'mean_word_length': 1.5378440366972477, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4977396', 'n_tokens_mistral': 1281, 'n_tokens_neox': 1233, 'n_words': 347}