<commit_before>local icu = require("justenoughicu")

require("char-def")
local chardata = characters.data

SILE.nodeMakers.base = std.object {
  makeToken = function (self)
    if #self.contents>0 then
      coroutine.yield(SILE.shaper:formNnode(self.contents, self.token, self.options))
      SU.debug("tokenizer", "Token: "..self.token)
      self.contents = {} ; self.token = "" ; self.lastnode = "nnode"
    end
  end,
  addToken = function (self, char, item)
    self.token = self.token .. char
    self.contents[#self.contents+1] = item
  end,
  makeGlue = function (self, item)
    if SILE.settings.get("typesetter.obeyspaces") or self.lastnode ~= "glue" then
      SU.debug("tokenizer", "Space node")
      coroutine.yield(SILE.shaper:makeSpaceNode(self.options, item))
    end
    self.lastnode = "glue"
    self.lasttype = "sp"
  end,
  makePenalty = function (self, p)
    if self.lastnode ~= "penalty" and self.lastnode ~= "glue" then
      coroutine.yield( SILE.nodefactory.newPenalty({ penalty = p or 0 }) )
    end
    self.lastnode = "penalty"
  end,
  init = function (self)
    self.contents = {}
    self.token = ""
    self.lastnode = false
    self.lasttype = false
  end,
  iterator = function (_, _)
    SU.error("Abstract function nodemaker:iterator called", true)
  end,
  charData = function (_, char)
    local cp = SU.codepoint(char)
    if not chardata[cp] then return {} end
    return chardata[cp]
  end,
  isPunctuation = function (self, char)
    return self.isPunctuationType[self:charData(char).category]
  end,
  isSpace = function (self, char)
    return self.isSpaceType[self:charData(char).linebreak]
  end,
  isBreaking = function (self, char)
    return self.isBreakingType[self:charData(char).linebreak]
  end
}

SILE.nodeMakers.unicode = SILE.nodeMakers.base {
  isWordType = { cm = true },
  isSpaceType = { sp = true },
  isBreakingType = { ba = true, zw = true },
  isPunctuationType = { po = true },
  dealWith = function (self, item)
    local char = item.text
    local cp = SU.codepoint(char)
    local thistype = chardata[cp] and chardata[cp].linebreak
    if self:isSpace(item.text) then
      self:makeToken()
      self:makeGlue(item)
    elseif self:isBreaking(item.text) then
      self:makeToken()
      self:makePenalty(0)
    elseif self.lasttype and (thistype and thistype ~= self.lasttype and not self.isWordType[thistype]) then
      self:addToken(char, item)
    else
      self:letterspace()
      self:addToken(char, item)
    end
    if not self.isWordType[thistype] then self.lasttype = chardata[cp] and chardata[cp].linebreak end
    self.lasttype = thistype
  end,
  handleInitialGlue = function (self, items)
    local i = 1
    while i <= #items do
      local item = items[i]
      if self:isSpace(item.text) then self:makeGlue(item) else break end
      i = i + 1
    end
    return i, items
  end,
  letterspace = function (self)
    if not SILE.settings.get("document.letterspaceglue") then return end
    if self.token then self:makeToken() end
    if self.lastnode and self.lastnode ~= "glue" then
      local w = SILE.settings.get("document.letterspaceglue").width
      SU.debug("tokenizer", "Letter space glue: "..w)
      coroutine.yield(SILE.nodefactory.newKern({ width = w }))
      self.lastnode = "glue"
      self.lasttype = "sp"
    end
  end,
  isICUBreakHere = function (_, chunks, item)
    return chunks[1] and (item.index >= chunks[1].index)
  end,
  handleICUBreak = function (self, chunks, item)
    -- The ICU library has told us there is a breakpoint at
    -- this index. We need to...
    local bp = chunks[1]
    -- ... remove this breakpoint (and any out of order ones)
    -- from the ICU breakpoints array so that chunks[1] is<commit_msg>fix(languages): Remove superfluous line<commit_after>local icu = require("justenoughicu")

require("char-def")
local chardata = characters.data

SILE.nodeMakers.base = std.object {
  makeToken = function (self)
    if #self.contents>0 then
      coroutine.yield(SILE.shaper:formNnode(self.contents, self.token, self.options))
      SU.debug("tokenizer", "Token: "..self.token)
      self.contents = {} ; self.token = "" ; self.lastnode = "nnode"
    end
  end,
  addToken = function (self, char, item)
    self.token = self.token .. char
    self.contents[#self.contents+1] = item
  end,
  makeGlue = function (self, item)
    if SILE.settings.get("typesetter.obeyspaces") or self.lastnode ~= "glue" then
      SU.debug("tokenizer", "Space node")
      coroutine.yield(SILE.shaper:makeSpaceNode(self.options, item))
    end
    self.lastnode = "glue"
    self.lasttype = "sp"
  end,
  makePenalty = function (self, p)
    if self.lastnode ~= "penalty" and self.lastnode ~= "glue" then
      coroutine.yield( SILE.nodefactory.newPenalty({ penalty = p or 0 }) )
    end
    self.lastnode = "penalty"
  end,
  init = function (self)
    self.contents = {}
    self.token = ""
    self.lastnode = false
    self.lasttype = false
  end,
  iterator = function (_, _)
    SU.error("Abstract function nodemaker:iterator called", true)
  end,
  charData = function (_, char)
    local cp = SU.codepoint(char)
    if not chardata[cp] then return {} end
    return chardata[cp]
  end,
  isPunctuation = function (self, char)
    return self.isPunctuationType[self:charData(char).category]
  end,
  isSpace = function (self, char)
    return self.isSpaceType[self:charData(char).linebreak]
  end,
  isBreaking = function (self, char)
    return self.isBreakingType[self:charData(char).linebreak]
  end
}

SILE.nodeMakers.unicode = SILE.nodeMakers.base {
  isWordType = { cm = true },
  isSpaceType = { sp = true },
  isBreakingType = { ba = true, zw = true },
  isPunctuationType = { po = true },
  dealWith = function (self, item)
    local char = item.text
    local cp = SU.codepoint(char)
    local thistype = chardata[cp] and chardata[cp].linebreak
    if self:isSpace(item.text) then
      self:makeToken()
      self:makeGlue(item)
    elseif self:isBreaking(item.text) then
      self:makeToken()
      self:makePenalty(0)
    elseif self.lasttype and (thistype and thistype ~= self.lasttype and not self.isWordType[thistype]) then
      self:addToken(char, item)
    else
      self:letterspace()
      self:addToken(char, item)
    end
    self.lasttype = thistype
  end,
  handleInitialGlue = function (self, items)
    local i = 1
    while i <= #items do
      local item = items[i]
      if self:isSpace(item.text) then self:makeGlue(item) else break end
      i = i + 1
    end
    return i, items
  end,
  letterspace = function (self)
    if not SILE.settings.get("document.letterspaceglue") then return end
    if self.token then self:makeToken() end
    if self.lastnode and self.lastnode ~= "glue" then
      local w = SILE.settings.get("document.letterspaceglue").width
      SU.debug("tokenizer", "Letter space glue: "..w)
      coroutine.yield(SILE.nodefactory.newKern({ width = w }))
      self.lastnode = "glue"
      self.lasttype = "sp"
    end
  end,
  isICUBreakHere = function (_, chunks, item)
    return chunks[1] and (item.index >= chunks[1].index)
  end,
  handleICUBreak = function (self, chunks, item)
    -- The ICU library has told us there is a breakpoint at
    -- this index. We need to...
    local bp = chunks[1]
    -- ... remove this breakpoint (and any out of order ones)
    -- from the ICU breakpoints array so that chunks[1] is
{'fraction_non_alphanumeric': 0.12169097784981091, 'fraction_numerical': 0.002701242571582928, 'mean_word_length': 3.1484593837535013, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1739234', 'n_tokens_mistral': 2518, 'n_tokens_neox': 2501, 'n_words': 639}