<commit_before>-- This is the default typesetter. You are, of course, welcome to create your own.
local awful_bad = 1073741823
local inf_bad = 10000
local eject_penalty = -inf_bad
local deplorable = 100000

SILE.settings.declare({
  name = "typesetter.widowpenalty", 
  type = "integer",
  default = 150,
  help = "Penalty to be applied to widow lines (at the start of a paragraph)"
})

SILE.settings.declare({
  name = "typesetter.orphanpenalty",
  type = "integer",
  default = 150,
  help = "Penalty to be applied to orphan lines (at the end of a paragraph)"
})

SILE.defaultTypesetter = std.object {
  -- Setup functions
  init = function(self, frame)
    self:initState();
    self.stateQueue = {};
    self:initFrame(frame);
    return self
  end,
  initFrame = function(self, f)
    self.frame = f;
    self.state.cursorX = self.frame:left(); -- XXX for bidi
    self.state.cursorY = self.frame:top();
    self.state.frameTotals = { height= 0 };
  end,
  initState = function(self)
    self.state = {
      nodes = {},
      outputQueue = {},
      lastBadness = awful_bad,
      frameTotals = { height = 0 },
      frameLines = {}
    };
  end,
  pushState = function(self)
    table.insert(self.stateQueue, self.state);
    self.initState();
  end,
  popState = function(self)
    self.state = table.remove(self.stateQueue);
  end,

  -- Boxy stuff
  pushHbox = function (self, spec) table.insert(self.state.nodes, SILE.nodefactory.newHbox(spec)); end,
  pushGlue = function (self, spec) return table.insert(self.state.nodes, SILE.nodefactory.newGlue(spec)); end,
  pushPenalty = function (self, spec) return table.insert(self.state.nodes, SILE.nodefactory.newPenalty(spec)); end,
  pushVbox = function (self, spec) local v = SILE.nodefactory.newVbox(spec); table.insert(self.state.outputQueue,v); return v; end,
  pushVglue = function (self, spec) return table.insert(self.state.outputQueue, SILE.nodefactory.newVglue(spec)); end,
  pushVpenalty = function (self, spec) return table.insert(self.state.outputQueue, SILE.nodefactory.newPenalty(spec)); end,

  parSepPattern = "\n\n+",

  -- Actual typesetting functions
  typeset = function (self, text)
    for t in SU.gtoke(text,self.parSepPattern) do
      if (t.separator) then self:leaveHmode();
      else self:setpar(t.string)
      end
    end
  end,

  -- Takes string, writes onto self.state.nodes
  setpar = function (self, t)
    t = string.gsub(t,"\n", " ");
    --t = string.gsub(t,"^%s+", "");
    if (#self.state.nodes == 0) then
      self:pushHbox({ width = SILE.length.new({length = 0}), value = {glyph = 0} });
      SILE.documentState.documentClass.newPar(self); -- XXX ?
    end
    for token in SU.gtoke(t, "-") do
      local t2= token.separator and token.separator or token.string
      local newNodes = SILE.shaper.shape(t2)
      for i=1,#newNodes do
          self.state.nodes[#(self.state.nodes)+1] = newNodes[i]
      end
    end
  end,

  -- Empties self.state.nodes, breaks into lines, puts lines into vbox, adds vbox to
  -- outputqueue, calls pageBuilder
  boxUpNodes = function (self, nl)
    while (#nl > 0 and (nl[#nl]:isPenalty() or nl[#nl]:isGlue())) do
      table.remove(nl);
    end

    while (#nl >0 and nl[1]:isPenalty()) do table.remove(nl,1) end
    self:pushGlue({ width = SILE.length.new({ length = 0, stretch =  10000 })});
    self:pushPenalty({ flagged= 1, penalty= -inf_bad });
    local breaks = SILE.linebreak:doBreak( nl, self.frame:width() );
    if (#breaks == 0) then
      SILE.SU.error("Couldn't break :(")
    end
    local lines = self:breakpointsToLines(breaks);
    local vboxes = {}
    local previousVbox = nil
    for index=1, #lines do
      local l = lines[index]
      local v = SILE.nodefactory.newVbox({ nodes = l.nodes, ratio = l.ratio });
      local pageBreakPenalty = 0
      if (#lines > 1 and index == 1) then
        pageBreakPenalty = SILE.settings.get("typesetter.widowpenalty")
      elseif (#lines > 1 and index == (#lines-1)) then
        pageBreakPenalty = SILE.settings.get("typesetter.orphanpenalty")
      end
      vboxes[#vboxes+1] = self:leadingFor(v, previousVbox)
      vboxes[#vboxes+1] = v
      previousVbox = v
      if pageBreakPenalty > 0 then<commit_msg>Various fixes.<commit_after>-- This is the default typesetter. You are, of course, welcome to create your own.
local awful_bad = 1073741823
local inf_bad = 10000
local eject_penalty = -inf_bad
local deplorable = 100000

SILE.settings.declare({
  name = "typesetter.widowpenalty", 
  type = "integer",
  default = 150,
  help = "Penalty to be applied to widow lines (at the start of a paragraph)"
})

SILE.settings.declare({
  name = "typesetter.orphanpenalty",
  type = "integer",
  default = 150,
  help = "Penalty to be applied to orphan lines (at the end of a paragraph)"
})

SILE.settings.declare({
  name = "typesetter.parfillskip",
  type = "Glue",
  default = SILE.nodefactory.newGlue("0pt plus 10000pt"),
  help = "Glue added at the end of a paragraph"
})

SILE.defaultTypesetter = std.object {
  -- Setup functions
  init = function(self, frame)
    self:initState();
    self.stateQueue = {};
    self:initFrame(frame);
    return self
  end,
  initFrame = function(self, f)
    self.frame = f;
    self.state.cursorX = self.frame:left(); -- XXX for bidi
    self.state.cursorY = self.frame:top();
    self.state.frameTotals = { height= 0 };
  end,
  initState = function(self)
    self.state = {
      nodes = {},
      outputQueue = {},
      lastBadness = awful_bad,
      frameTotals = { height = 0 },
      frameLines = {}
    };
    self:initline()
  end,
  pushState = function(self)
    table.insert(self.stateQueue, self.state);
    self:initState();
  end,
  popState = function(self)
    self.state = table.remove(self.stateQueue);
  end,

  -- Boxy stuff
  pushHbox = function (self, spec) table.insert(self.state.nodes, SILE.nodefactory.newHbox(spec)); end,
  pushGlue = function (self, spec) return table.insert(self.state.nodes, SILE.nodefactory.newGlue(spec)); end,
  pushPenalty = function (self, spec) return table.insert(self.state.nodes, SILE.nodefactory.newPenalty(spec)); end,
  pushVbox = function (self, spec) local v = SILE.nodefactory.newVbox(spec); table.insert(self.state.outputQueue,v); return v; end,
  pushVglue = function (self, spec) return table.insert(self.state.outputQueue, SILE.nodefactory.newVglue(spec)); end,
  pushVpenalty = function (self, spec) return table.insert(self.state.outputQueue, SILE.nodefactory.newPenalty(spec)); end,

  parSepPattern = "\n\n+",

  -- Actual typesetting functions
  typeset = function (self, text)
    for t in SU.gtoke(text,self.parSepPattern) do
      if (t.separator) then self:leaveHmode();
      else self:setpar(t.string)
      end
    end
  end,

  initline = function (self)
    if (#self.state.nodes == 0) then
      self:pushHbox({ width = SILE.length.new({length = 0}), value = {glyph = 0} });
    end
  end,

  -- Takes string, writes onto self.state.nodes
  setpar = function (self, t)
    t = string.gsub(t,"\n", " ");
    --t = string.gsub(t,"^%s+", "");
    if (#self.state.nodes == 0) then
      self:initline()
      SILE.documentState.documentClass.newPar(self); -- XXX ?
    end
    for token in SU.gtoke(t, "-") do
      local t2= token.separator and token.separator or token.string
      local newNodes = SILE.shaper.shape(t2)
      for i=1,#newNodes do
          self.state.nodes[#(self.state.nodes)+1] = newNodes[i]
      end
    end
  end,

  -- Empties self.state.nodes, breaks into lines, puts lines into vbox, adds vbox to
  -- outputqueue, calls pageBuilder
  boxUpNodes = function (self, nl, suppressFinalGlue)
    -- Question: If final discardables are discardable, how does "\hss foo \hss" work?
    --while (#nl > 0 and (nl[#nl]:isPenalty() or nl[#nl]:isGlue())) do
    --  table.remove(nl);
    --end

    while (#nl >0 and nl[1]:isPenalty()) do table.remove(nl,1) end
    self:pushGlue(SILE.settings.get("typesetter.parfillskip"));
    self:pushPenalty({ flagged= 1, penalty= -inf_bad });
    SU.debug("typesetter", "Boxed up "..nl);
    local breaks = SILE.linebreak:doBreak( nl, self.frame:width() );
    if (#breaks == 0) then
      SILE.SU.error("Couldn't break :(")
    end
    local lines = self:breakpointsToLines(breaks);
    local vboxes = {}
    local previousVbox = nil
    for index=1, #lines do
      local l = lines[index]
      local v = SILE.nodefactory.newVbox({ nodes = l.nodes, ratio = l.ratio });
      local pageBreakPenalty = 0
      if (#lines > 1 and index == 1) then
        pageBreakPenalty = SILE.settings.get("typesetter.widowpenalty")
      elseif (#lines > 1 and index == (#lines-1)) then
        pageBreakPenalty = SILE.settings.get("typesetter.orphanpenalty")
      end
      vboxes[#vboxes+1] = self:leadingFor(v, previousVbox)
      vboxes[#vboxes+1] = v
      previousVbox = v
      if pageBreakPenalty > 0 then
{'fraction_non_alphanumeric': 0.1383350230933874, 'fraction_numerical': 0.012954827081221133, 'mean_word_length': 3.2316491897044806, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6136731', 'n_tokens_mistral': 3171, 'n_tokens_neox': 2996, 'n_words': 764}