<commit_before>local CreOptions = require("ui/data/creoptions")
local Document = require("document/document")
local Configurable = require("configurable")
local Blitbuffer = require("ffi/blitbuffer")
local lfs = require("libs/libkoreader-lfs")
local Geom = require("ui/geometry")
local Device = require("device")
local Screen = require("device").screen
local Font = require("ui/font")
local DEBUG = require("dbg")
local ffi = require("ffi")

local CreDocument = Document:new{
    -- this is defined in kpvcrlib/crengine/crengine/include/lvdocview.h
    SCROLL_VIEW_MODE = 0,
    PAGE_VIEW_MODE = 1,

    _document = false,
    _loaded = false,
    engine_initilized = false,

    line_space_percent = 100,
    default_font = G_reader_settings:readSetting("cre_font") or "Noto Serif",
    header_font = G_reader_settings:readSetting("header_font") or "Noto Sans",
    fallback_font = G_reader_settings:readSetting("fallback_font") or "Droid Sans Fallback H",
    default_css = "./data/cr3.css",
    options = CreOptions,
}

-- NuPogodi, 20.05.12: inspect the zipfile content
function CreDocument:zipContentExt(fname)
    local std_out = io.popen("unzip ".."-qql \""..fname.."\"")
    if std_out then
        for line in std_out:lines() do
            local size, ext = string.match(line, "%s+(%d+)%s+.+%.([^.]+)")
            -- return the extention
            if size and ext then return string.lower(ext) end
        end
    end
end

function CreDocument:cacheInit()
    -- remove legacy cr3cache directory
    if lfs.attributes("./cr3cache", "mode") == "directory" then
        os.execute("rm -r ./cr3cache")
    end
    cre.initCache("./cache/cr3cache", 1024*1024*32)
end

function CreDocument:engineInit()
    if not engine_initilized then
        -- initialize cache
        self:cacheInit()

        -- initialize hyph dictionaries
        cre.initHyphDict("./data/hyph/")

        -- we need to initialize the CRE font list
        local fonts = Font:getFontList()
        for _k, _v in ipairs(fonts) do
            if _v:sub(1, 4) ~= "urw/" then
                local ok, err = pcall(cre.registerFont, Font.fontdir..'/'.._v)
                if not ok then
                    DEBUG(err)
                end
            end
        end

        engine_initilized = true
    end
end

function CreDocument:init()
    require "libs/libkoreader-cre"
    self:engineInit()
    self.configurable:loadDefaults(self.options)

    local file_type = string.lower(string.match(self.file, ".+%.([^.]+)"))
    if file_type == "zip" then
        -- NuPogodi, 20.05.12: read the content of zip-file
        -- and return extention of the 1st file
        file_type = self:zipContentExt(self.file) or "unknown"
    end
    -- these two format use the same css file
    if file_type == "html" then
        file_type = "htm"
    end
    -- if native css-file doesn't exist, one needs to use default cr3.css
    if not io.open("./data/"..file_type..".css") then
        file_type = "cr3"
    end
    self.default_css = "./data/"..file_type..".css"

    -- @TODO check the default view_mode to a global user configurable
    -- variable  22.12 2012 (houqp)
    local ok
    ok, self._document = pcall(cre.newDocView,
        Screen:getWidth(), Screen:getHeight(), self.PAGE_VIEW_MODE
    )
    if not ok then
        error(self._document)  -- will contain error message
    end

    -- adjust font sizes according to screen dpi
    self._document:adjustFontSizes(Screen:getDPI())

    -- set fallback font face
    self._document:setStringProperty("crengine.font.fallback.face", self.fallback_font)

    self.is_open = true
    self.info.has_pages = false
    self:_readMetadata()
    self.info.configurable = true
end

function CreDocument:loadDocument()
    if not self._loaded then
        self._document:loadDocument(self.file)
        self._loaded = true
    end
end

function CreDocument:render()
    -- load document before rendering
    self:loadDocument()
    -- set visible page count in landscape
    if math.max(Screen:getWidth(), Screen:getHeight()) / Screen:getDPI()
        < DCREREADER_TWO_PAGE_THRESHOLD then
        self:setVisiblePageCount(1)
    end
    self._document:renderDocument()
    if not self.info.has_pages then
        self.info.doc_height = self._document:getFullHeight()
    end
end

function CreDocument:close()
    Document.close(self)
end

function CreDocument:getPageCount()
    return self._document:getPages()
end

function CreDocument:getCoverPageImage()
    -- don't need to render document in order to get cover image
    self:loadDocument()
    local data, size = self._document:getCoverPageImageData()
    if data and size then
        local Mupdf = require("ffi/mupdf")
        local ok, image = pcall(Mupdf.renderImage, data, size)
        if ok then
            ffi.C.free(data)
            return image
        end
    end
end

function CreDocument:getWordFromPosition(pos)
    local word_box = self._document:getWordFromPosition(pos.x, pos.y)
    DEBUG("CreDocument: get word box", word_box)
    local text_range = self._document:getTextFromPositions(pos.x, pos.y, pos.x, pos.y)
    DEBUG("CreDocument: get text range", text_range)
    local wordbox = {
        word = text_range.text == "" and word_box.word or text_range.text,
        page = self._document:getCurrentPage(),
    }
    if word_box.word then
        wordbox.sbox = Geom:new{
            x = word_box.x0, y = word_box.y0,
            w = word_box.x1 - word_box.x0,
            h = word_box.y1 - word_box.y0,
        }
    else
        -- dummy word box
        wordbox.sbox = Geom:new{
            x = pos.x, y = pos.y,
            w = 20, h = 20,
        }
    end
    return wordbox
end

function CreDocument:getTextFromPositions(pos0, pos1)
    local text_range = self._document:getTextFromPositions(pos0.x, pos0.y, pos1.x, pos1.y)
    DEBUG("CreDocument: get text range", text_range)
    local line_boxes = self:getScreenBoxesFromPositions(text_range.pos0, text_range.pos1)
    return {
        text = text_range.text,
        pos0 = text_range.pos0,
        pos1 = text_range.pos1,
        --sboxes = line_boxes,     -- boxes on screen
    }
end

function CreDocument:getScreenBoxesFromPositions(pos0, pos1)
    local line_boxes = {}
    if pos0 and pos1 then
        local word_boxes = self._document:getWordBoxesFromPositions(pos0, pos1)
        --DEBUG("word boxes", word_boxes)
        for i = 1, #word_boxes do
            local line_box = word_boxes[i]
            table.insert(line_boxes, Geom:new{
                x = line_box.x0, y = line_box.y0,
                w = line_box.x1 - line_box.x0,
                h = line_box.y1 - line_box.y0,
            })
        end<commit_msg>fix #968 by check the nullity of text range<commit_after>local CreOptions = require("ui/data/creoptions")
local Document = require("document/document")
local Configurable = require("configurable")
local Blitbuffer = require("ffi/blitbuffer")
local lfs = require("libs/libkoreader-lfs")
local Geom = require("ui/geometry")
local Device = require("device")
local Screen = require("device").screen
local Font = require("ui/font")
local DEBUG = require("dbg")
local ffi = require("ffi")

local CreDocument = Document:new{
    -- this is defined in kpvcrlib/crengine/crengine/include/lvdocview.h
    SCROLL_VIEW_MODE = 0,
    PAGE_VIEW_MODE = 1,

    _document = false,
    _loaded = false,
    engine_initilized = false,

    line_space_percent = 100,
    default_font = G_reader_settings:readSetting("cre_font") or "Noto Serif",
    header_font = G_reader_settings:readSetting("header_font") or "Noto Sans",
    fallback_font = G_reader_settings:readSetting("fallback_font") or "Droid Sans Fallback H",
    default_css = "./data/cr3.css",
    options = CreOptions,
}

-- NuPogodi, 20.05.12: inspect the zipfile content
function CreDocument:zipContentExt(fname)
    local std_out = io.popen("unzip ".."-qql \""..fname.."\"")
    if std_out then
        for line in std_out:lines() do
            local size, ext = string.match(line, "%s+(%d+)%s+.+%.([^.]+)")
            -- return the extention
            if size and ext then return string.lower(ext) end
        end
    end
end

function CreDocument:cacheInit()
    -- remove legacy cr3cache directory
    if lfs.attributes("./cr3cache", "mode") == "directory" then
        os.execute("rm -r ./cr3cache")
    end
    cre.initCache("./cache/cr3cache", 1024*1024*32)
end

function CreDocument:engineInit()
    if not engine_initilized then
        -- initialize cache
        self:cacheInit()

        -- initialize hyph dictionaries
        cre.initHyphDict("./data/hyph/")

        -- we need to initialize the CRE font list
        local fonts = Font:getFontList()
        for _k, _v in ipairs(fonts) do
            if _v:sub(1, 4) ~= "urw/" then
                local ok, err = pcall(cre.registerFont, Font.fontdir..'/'.._v)
                if not ok then
                    DEBUG(err)
                end
            end
        end

        engine_initilized = true
    end
end

function CreDocument:init()
    require "libs/libkoreader-cre"
    self:engineInit()
    self.configurable:loadDefaults(self.options)

    local file_type = string.lower(string.match(self.file, ".+%.([^.]+)"))
    if file_type == "zip" then
        -- NuPogodi, 20.05.12: read the content of zip-file
        -- and return extention of the 1st file
        file_type = self:zipContentExt(self.file) or "unknown"
    end
    -- these two format use the same css file
    if file_type == "html" then
        file_type = "htm"
    end
    -- if native css-file doesn't exist, one needs to use default cr3.css
    if not io.open("./data/"..file_type..".css") then
        file_type = "cr3"
    end
    self.default_css = "./data/"..file_type..".css"

    -- @TODO check the default view_mode to a global user configurable
    -- variable  22.12 2012 (houqp)
    local ok
    ok, self._document = pcall(cre.newDocView,
        Screen:getWidth(), Screen:getHeight(), self.PAGE_VIEW_MODE
    )
    if not ok then
        error(self._document)  -- will contain error message
    end

    -- adjust font sizes according to screen dpi
    self._document:adjustFontSizes(Screen:getDPI())

    -- set fallback font face
    self._document:setStringProperty("crengine.font.fallback.face", self.fallback_font)

    self.is_open = true
    self.info.has_pages = false
    self:_readMetadata()
    self.info.configurable = true
end

function CreDocument:loadDocument()
    if not self._loaded then
        self._document:loadDocument(self.file)
        self._loaded = true
    end
end

function CreDocument:render()
    -- load document before rendering
    self:loadDocument()
    -- set visible page count in landscape
    if math.max(Screen:getWidth(), Screen:getHeight()) / Screen:getDPI()
        < DCREREADER_TWO_PAGE_THRESHOLD then
        self:setVisiblePageCount(1)
    end
    self._document:renderDocument()
    if not self.info.has_pages then
        self.info.doc_height = self._document:getFullHeight()
    end
end

function CreDocument:close()
    Document.close(self)
end

function CreDocument:getPageCount()
    return self._document:getPages()
end

function CreDocument:getCoverPageImage()
    -- don't need to render document in order to get cover image
    self:loadDocument()
    local data, size = self._document:getCoverPageImageData()
    if data and size then
        local Mupdf = require("ffi/mupdf")
        local ok, image = pcall(Mupdf.renderImage, data, size)
        if ok then
            ffi.C.free(data)
            return image
        end
    end
end

function CreDocument:getWordFromPosition(pos)
    local word_box = self._document:getWordFromPosition(pos.x, pos.y)
    DEBUG("CreDocument: get word box", word_box)
    local text_range = self._document:getTextFromPositions(pos.x, pos.y, pos.x, pos.y)
    DEBUG("CreDocument: get text range", text_range)
    local wordbox = {
        word = text_range.text == "" and word_box.word or text_range.text,
        page = self._document:getCurrentPage(),
    }
    if word_box.word then
        wordbox.sbox = Geom:new{
            x = word_box.x0, y = word_box.y0,
            w = word_box.x1 - word_box.x0,
            h = word_box.y1 - word_box.y0,
        }
    else
        -- dummy word box
        wordbox.sbox = Geom:new{
            x = pos.x, y = pos.y,
            w = 20, h = 20,
        }
    end
    return wordbox
end

function CreDocument:getTextFromPositions(pos0, pos1)
    local text_range = self._document:getTextFromPositions(pos0.x, pos0.y, pos1.x, pos1.y)
    DEBUG("CreDocument: get text range", text_range)
    if text_range then
        local line_boxes = self:getScreenBoxesFromPositions(text_range.pos0, text_range.pos1)
        return {
            text = text_range.text,
            pos0 = text_range.pos0,
            pos1 = text_range.pos1,
            --sboxes = line_boxes,     -- boxes on screen
        }
    end
end

function CreDocument:getScreenBoxesFromPositions(pos0, pos1)
    local line_boxes = {}
    if pos0 and pos1 then
        local word_boxes = self._document:getWordBoxesFromPositions(pos0, pos1)
        --DEBUG("word boxes", word_boxes)
        for i = 1, #word_boxes do
            local line_box = word_boxes[i]
            table.insert(line_boxes, Geom:new{
                x = line_box.x0, y = line_box.y0,
                w = line_box.x1 - line_box.x0,
                h = line_box.y1 - line_box.y0,
            })
        end
{'fraction_non_alphanumeric': 0.10896817743490839, 'fraction_numerical': 0.012239448112157852, 'mean_word_length': 2.569499602859412, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3931457', 'n_tokens_mistral': 4377, 'n_tokens_neox': 4235, 'n_words': 1071}