<commit_before>SILE.languageSupport = {
  languages = {},
  loadLanguage = function(language)
    if SILE.languageSupport.languages[language] then return end
    if SILE.hyphenator.languages[language] then return end
    if not(language) or language == "" then language = "en" end
    ok, fail = pcall(function () SILE.require("languages/"..language.."-compiled") end)
    if ok then return end
    ok, fail = pcall(function () SILE.require("languages/"..language) end)
  end,
  compile = function(language)
    local ser = require("serpent")
    -- Ensure things are loaded
    SILE.languageSupport.loadLanguage(language)
    SILE.hyphenate({ SILE.nodefactory.newNnode({language=language, text=""}) })
    local f = io.open("languages/"..language.."-compiled.lua", "w")
    f:write("_hyphenators."..language.."="..ser.line(_hyphenators[language]).."\n")
    f:write("SILE.hyphenator.languages."..language.."="..ser.line(SILE.hyphenator.languages[language]).."\n")
    f:close()
  end
}

SILE.registerCommand("language", function (o,c)
  local lang = SU.required(o, "main", "language setting")
  SILE.languageSupport.loadLanguage(lang)
  SILE.settings.set("document.language", lang)
end)

require("languages/unicode")

-- The following languages neither have hyphenation nor specific
-- language support at present. This code is here to suppress warnings.
SILE.hyphenator.languages.ar = {patterns={}}<commit_msg>Let us know when languages fail to compile.<commit_after>SILE.languageSupport = {
  languages = {},
  loadLanguage = function(language)
    if SILE.languageSupport.languages[language] then return end
    if SILE.hyphenator.languages[language] then return end
    if not(language) or language == "" then language = "en" end
    ok, fail = pcall(function () SILE.require("languages/"..language.."-compiled") end)
    if ok then return end
    ok, fail = pcall(function () SILE.require("languages/"..language) end)
    if fail then
      SU.error("Error loading language "..language..": "..fail)
    end
  end,
  compile = function(language)
    local ser = require("serpent")
    -- Ensure things are loaded
    SILE.languageSupport.loadLanguage(language)
    SILE.hyphenate({ SILE.nodefactory.newNnode({language=language, text=""}) })
    local f = io.open("languages/"..language.."-compiled.lua", "w")
    f:write("_hyphenators."..language.."="..ser.line(_hyphenators[language]).."\n")
    f:write("SILE.hyphenator.languages."..language.."="..ser.line(SILE.hyphenator.languages[language]).."\n")
    f:close()
  end
}

SILE.registerCommand("language", function (o,c)
  local lang = SU.required(o, "main", "language setting")
  SILE.languageSupport.loadLanguage(lang)
  SILE.settings.set("document.language", lang)
end)

require("languages/unicode")

-- The following languages neither have hyphenation nor specific
-- language support at present. This code is here to suppress warnings.
SILE.hyphenator.languages.ar = {patterns={}}
{'fraction_non_alphanumeric': 0.15528372409106353, 'fraction_numerical': 0.0, 'mean_word_length': 4.192239858906525, 'pattern_counts': {'":': 1, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2802742', 'n_tokens_mistral': 930, 'n_tokens_neox': 907, 'n_words': 203}