<commit_before>local CanvasContext = require("document/canvascontext")

local FontList = {
    fontdir = "./fonts",
    fontlist = {},
}

--[[
These non-LGC Kindle system fonts fail CRe's moronic header check.
--]]
local kindle_fonts_blacklist = {
    ["DiwanMuna-Bold.ttf"] = true,
    ["DiwanMuna-Regular.ttf"] = true,
    ["HYGothicBold.ttf"] = true,
    ["HYGothicMedium.ttf"] = true,
    ["HYMyeongJoBold.ttf"] = true,
    ["HYMyeongJoMedium.ttf"] = true,
    ["KindleBlackboxBoldItalic.ttf"] = true,
    ["KindleBlackboxBold.ttf"] = true,
    ["KindleBlackboxItalic.ttf"] = true,
    ["KindleBlackboxRegular.ttf"] = true,
    ["Kindle_MonospacedSymbol.ttf"] = true,
    ["Kindle_Symbol.ttf"] = true,
    ["MTChineseSurrogates.ttf"] = true,
    ["MYingHeiTBold.ttf"] = true,
    ["MYingHeiTMedium.ttf"] = true,
    ["NotoNaskhArabicUI-Bold.ttf"] = true,
    ["NotoNaskhArabicUI-Regular.ttf"] = true,
    ["NotoNaskh-Bold.ttf"] = true,
    ["NotoNaskh-Regular.ttf"] = true,
    ["NotoSansBengali-Regular.ttf"] = true,
    ["NotoSansDevanagari-Regular.ttf"] = true,
    ["NotoSansGujarati-Regular.ttf"] = true,
    ["NotoSansKannada-Regular.ttf"] = true,
    ["NotoSansMalayalam-Regular.ttf"] = true,
    ["NotoSansTamil-Regular.ttf"] = true,
    ["NotoSansTelugu-Regular.ttf"] = true,
    ["SakkalKitab-Bold.ttf"] = true,
    ["SakkalKitab-Regular.ttf"] = true,
    ["SongTBold.ttf"] = true,
    ["SongTMedium.ttf"] = true,
    ["STHeitiBold.ttf"] = true,
    ["STHeitiMedium.ttf"] = true,
    ["STSongBold.ttf"] = true,
    ["STSongMedium.ttf"] = true,
    ["TBGothicBold_213.ttf"] = true,
    ["TBGothicMed_213.ttf"] = true,
    ["TBMinchoBold_213.ttf"] = true,
    ["TBMinchoMedium_213.ttf"] = true,
    ["STKaiMedium.ttf"] = true,
    ["Caecilia_LT_67_Cond_Medium.ttf"] = true,
    ["Caecilia_LT_68_Cond_Medium_Italic.ttf"] = true,
    ["Caecilia_LT_77_Cond_Bold.ttf"] = true,
    ["Caecilia_LT_78_Cond_Bold_Italic.ttf"] = true,
    ["Helvetica_LT_65_Medium.ttf"] = true,
    ["Helvetica_LT_66_Medium_Italic.ttf"] = true,
    ["Helvetica_LT_75_Bold.ttf"] = true,
    ["Helvetica_LT_76_Bold_Italic.ttf"] = true,
}

local function isInFontsBlacklist(f)
    -- write test for this
    return CanvasContext.isKindle() and kindle_fonts_blacklist[f]
end

local function getExternalFontDir()
    if CanvasContext.isAndroid() or CanvasContext.isDesktop() then
        return require("frontend/ui/elements/font_settings"):getPath()
    else
        return os.getenv("EXT_FONT_DIR")
    end
end

local function _readList(target, dir)
    -- lfs.dir non-existent directory will give an error, weird!
    local ok, iter, dir_obj = pcall(lfs.dir, dir)
    if not ok then return end<commit_msg>Blacklist a few Amazon Ember styles to get the main ones to behave. (#5951)<commit_after>local CanvasContext = require("document/canvascontext")

local FontList = {
    fontdir = "./fonts",
    fontlist = {},
}

--[[
These non-LGC Kindle system fonts fail CRe's moronic header check.
Also applies to a number of LGC fonts that have different family names for different styles...
(Those are actually "fixed" via FontConfig in the stock system).
--]]
local kindle_fonts_blacklist = {
    ["DiwanMuna-Bold.ttf"] = true,
    ["DiwanMuna-Regular.ttf"] = true,
    ["HYGothicBold.ttf"] = true,
    ["HYGothicMedium.ttf"] = true,
    ["HYMyeongJoBold.ttf"] = true,
    ["HYMyeongJoMedium.ttf"] = true,
    ["KindleBlackboxBoldItalic.ttf"] = true,
    ["KindleBlackboxBold.ttf"] = true,
    ["KindleBlackboxItalic.ttf"] = true,
    ["KindleBlackboxRegular.ttf"] = true,
    ["Kindle_MonospacedSymbol.ttf"] = true,
    ["Kindle_Symbol.ttf"] = true,
    ["MTChineseSurrogates.ttf"] = true,
    ["MYingHeiTBold.ttf"] = true,
    ["MYingHeiTMedium.ttf"] = true,
    ["NotoNaskhArabicUI-Bold.ttf"] = true,
    ["NotoNaskhArabicUI-Regular.ttf"] = true,
    ["NotoNaskh-Bold.ttf"] = true,
    ["NotoNaskh-Regular.ttf"] = true,
    ["NotoSansBengali-Regular.ttf"] = true,
    ["NotoSansDevanagari-Regular.ttf"] = true,
    ["NotoSansGujarati-Regular.ttf"] = true,
    ["NotoSansKannada-Regular.ttf"] = true,
    ["NotoSansMalayalam-Regular.ttf"] = true,
    ["NotoSansTamil-Regular.ttf"] = true,
    ["NotoSansTelugu-Regular.ttf"] = true,
    ["SakkalKitab-Bold.ttf"] = true,
    ["SakkalKitab-Regular.ttf"] = true,
    ["SongTBold.ttf"] = true,
    ["SongTMedium.ttf"] = true,
    ["STHeitiBold.ttf"] = true,
    ["STHeitiMedium.ttf"] = true,
    ["STSongBold.ttf"] = true,
    ["STSongMedium.ttf"] = true,
    ["TBGothicBold_213.ttf"] = true,
    ["TBGothicMed_213.ttf"] = true,
    ["TBMinchoBold_213.ttf"] = true,
    ["TBMinchoMedium_213.ttf"] = true,
    ["STKaiMedium.ttf"] = true,
    ["Amazon-Ember-Bold.ttf"] = false,
    ["Amazon-Ember-BoldItalic.ttf"] = false,
    ["Amazon-Ember-Heavy.ttf"] = true,
    ["Amazon-Ember-HeavyItalic.ttf"] = true,
    ["Amazon-Ember-Medium.ttf"] = true,
    ["Amazon-Ember-MediumItalic.ttf"] = true,
    ["Amazon-Ember-Regular.ttf"] = false,
    ["Amazon-Ember-RegularItalic.ttf"] = false,
    ["AmazonEmberBold-Bold.ttf"] = true,
    ["AmazonEmberBold-BoldItalic.ttf"] = true,
    ["AmazonEmberBold-Italic.ttf"] = true,
    ["AmazonEmberBold-Regular.ttf"] = true,
    ["Caecilia_LT_65_Medium.ttf"] = false,
    ["Caecilia_LT_66_Medium_Italic.ttf"] = false,
    ["Caecilia_LT_75_Bold.ttf"] = false,
    ["Caecilia_LT_76_Bold_Italic.ttf"] = false,
    ["Caecilia_LT_67_Cond_Medium.ttf"] = true,
    ["Caecilia_LT_68_Cond_Medium_Italic.ttf"] = true,
    ["Caecilia_LT_77_Cond_Bold.ttf"] = true,
    ["Caecilia_LT_78_Cond_Bold_Italic.ttf"] = true,
    ["Futura-Bold.ttf"] = true,
    ["Futura-BoldOblique.ttf"] = true,
    ["Helvetica_LT_65_Medium.ttf"] = true,
    ["Helvetica_LT_66_Medium_Italic.ttf"] = true,
    ["Helvetica_LT_75_Bold.ttf"] = true,
    ["Helvetica_LT_76_Bold_Italic.ttf"] = true,
}

local function isInFontsBlacklist(f)
    -- write test for this
    return CanvasContext.isKindle() and kindle_fonts_blacklist[f]
end

local function getExternalFontDir()
    if CanvasContext.isAndroid() or CanvasContext.isDesktop() then
        return require("frontend/ui/elements/font_settings"):getPath()
    else
        return os.getenv("EXT_FONT_DIR")
    end
end

local function _readList(target, dir)
    -- lfs.dir non-existent directory will give an error, weird!
    local ok, iter, dir_obj = pcall(lfs.dir, dir)
    if not ok then return end
{'fraction_non_alphanumeric': 0.15949764521193094, 'fraction_numerical': 0.010675039246467817, 'mean_word_length': 3.786626596543952, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1419395', 'n_tokens_mistral': 2725, 'n_tokens_neox': 2548, 'n_words': 404}