<commit_before>local Blitbuffer = require("ffi/blitbuffer")
local CreOptions = require("ui/data/creoptions")
local DataStorage = require("datastorage")
local Document = require("document/document")
local Font = require("ui/font")
local Geom = require("ui/geometry")
local Screen = require("device").screen
local ffi = require("ffi")
local lfs = require("libs/libkoreader-lfs")
local logger = require("logger")

-- engine can be initialized only once, on first document opened
local engine_initialized = false

local CreDocument = Document:new{
    -- this is defined in kpvcrlib/crengine/crengine/include/lvdocview.h
    SCROLL_VIEW_MODE = 0,
    PAGE_VIEW_MODE = 1,

    _document = false,
    _loaded = false,

    line_space_percent = 100,
    default_font = G_reader_settings:readSetting("cre_font") or "Noto Serif",
    header_font = G_reader_settings:readSetting("header_font") or "Noto Sans",
    fallback_font = G_reader_settings:readSetting("fallback_font") or "Noto Sans CJK SC",
    default_css = "./data/cr3.css",
    options = CreOptions,
    provider = "crengine",
    provider_name = "Cool Reader Engine",
}

-- NuPogodi, 20.05.12: inspect the zipfile content
function CreDocument:zipContentExt(fname)
    local std_out = io.popen("unzip ".."-qql \""..fname.."\"")
    if std_out then
        for line in std_out:lines() do
            local size, ext = string.match(line, "%s+(%d+)%s+.+%.([^.]+)")
            -- return the extention
            if size and ext then return string.lower(ext) end
        end
    end
end

function CreDocument:cacheInit()
    -- remove legacy cr3cache directory
    if lfs.attributes("./cr3cache", "mode") == "directory" then
        os.execute("rm -r ./cr3cache")
    end
    cre.initCache(DataStorage:getDataDir() .. "/cache/cr3cache", 1024*1024*32,
        G_reader_settings:nilOrTrue("cre_compress_cached_data"),
        G_reader_settings:readSetting("cre_storage_size_factor"))
end
<commit_msg>Set default 'cre_storage_size_factor' to 20 (#3732)<commit_after>local Blitbuffer = require("ffi/blitbuffer")
local CreOptions = require("ui/data/creoptions")
local DataStorage = require("datastorage")
local Document = require("document/document")
local Font = require("ui/font")
local Geom = require("ui/geometry")
local Screen = require("device").screen
local ffi = require("ffi")
local lfs = require("libs/libkoreader-lfs")
local logger = require("logger")

-- engine can be initialized only once, on first document opened
local engine_initialized = false

local CreDocument = Document:new{
    -- this is defined in kpvcrlib/crengine/crengine/include/lvdocview.h
    SCROLL_VIEW_MODE = 0,
    PAGE_VIEW_MODE = 1,

    _document = false,
    _loaded = false,

    line_space_percent = 100,
    default_font = G_reader_settings:readSetting("cre_font") or "Noto Serif",
    header_font = G_reader_settings:readSetting("header_font") or "Noto Sans",
    fallback_font = G_reader_settings:readSetting("fallback_font") or "Noto Sans CJK SC",
    default_css = "./data/cr3.css",
    options = CreOptions,
    provider = "crengine",
    provider_name = "Cool Reader Engine",
}

-- NuPogodi, 20.05.12: inspect the zipfile content
function CreDocument:zipContentExt(fname)
    local std_out = io.popen("unzip ".."-qql \""..fname.."\"")
    if std_out then
        for line in std_out:lines() do
            local size, ext = string.match(line, "%s+(%d+)%s+.+%.([^.]+)")
            -- return the extention
            if size and ext then return string.lower(ext) end
        end
    end
end

function CreDocument:cacheInit()
    -- remove legacy cr3cache directory
    if lfs.attributes("./cr3cache", "mode") == "directory" then
        os.execute("rm -r ./cr3cache")
    end
    -- crengine various in-memory caches max-sizes are rather small
    -- (2.5 / 4.5 / 1.5 / 1 MB), and we can avoid some bugs if we
    -- increase them. Let's multiply them by 20 (each cache would
    -- grow only when needed, depending on book characteristics).
    -- People who would get out of memory crashes with big books on
    -- older devices can decrease that with setting:
    --   "cre_storage_size_factor"=1    (or 2, or 5)
    local default_cre_storage_size_factor = 20
    cre.initCache(DataStorage:getDataDir() .. "/cache/cr3cache", 1024*1024*32,
        G_reader_settings:nilOrTrue("cre_compress_cached_data"),
        G_reader_settings:readSetting("cre_storage_size_factor") or default_cre_storage_size_factor)
end

{'fraction_non_alphanumeric': 0.12258643915581499, 'fraction_numerical': 0.016165244723843737, 'mean_word_length': 3.6894736842105265, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7299399', 'n_tokens_mistral': 1467, 'n_tokens_neox': 1390, 'n_words': 371}