<commit_before>require "dbg"
require "cache"
require "ui/geometry"
require "ui/device"
require "ui/reader/readerconfig"

KoptInterface = {
	tessocr_data = "data",
	ocr_lang = "eng",
	ocr_type = 0, -- default, for more accuracy use 3
}

ContextCacheItem = CacheItem:new{}

function ContextCacheItem:onFree()
	if self.kctx.free then
		DEBUG("free koptcontext", self.kctx)
		self.kctx:free()
	end
end

function KoptInterface:waitForContext(kc)
	-- if koptcontext is being processed in background thread
	-- the isPreCache will return 1.
	while kc and kc:isPreCache() == 1 do
		DEBUG("waiting for background rendering")
		util.usleep(100000)
	end
	return kc
end

--[[
get reflow context
--]]
function KoptInterface:createContext(doc, pageno, bbox)
	-- Now koptcontext keeps track of its dst bitmap reflowed by libk2pdfopt.
	-- So there is no need to check background context when creating new context.
	local kc = KOPTContext.new()
	local screen_size = Screen:getSize()
	kc:setTrim(doc.configurable.trim_page)
	kc:setWrap(doc.configurable.text_wrap)
	kc:setIndent(doc.configurable.detect_indent)
	kc:setRotate(doc.configurable.screen_rotation)
	kc:setColumns(doc.configurable.max_columns)
	kc:setDeviceDim(screen_size.w, screen_size.h)
	kc:setDeviceDPI(doc.screen_dpi)
	kc:setStraighten(doc.configurable.auto_straighten)
	kc:setJustification(doc.configurable.justification)
	kc:setZoom(doc.configurable.font_size)
	kc:setMargin(doc.configurable.page_margin)
	kc:setQuality(doc.configurable.quality)
	kc:setContrast(doc.configurable.contrast)
	kc:setDefectSize(doc.configurable.defect_size)
	kc:setLineSpacing(doc.configurable.line_spacing)
	kc:setWordSpacing(doc.configurable.word_spacing)
	kc:setBBox(bbox.x0, bbox.y0, bbox.x1, bbox.y1)
	if Dbg.is_on then kc:setDebug() end
	return kc
end

function KoptInterface:getContextHash(doc, pageno, bbox)
	local screen_size = Screen:getSize()
	local screen_size_hash = screen_size.w.."|"..screen_size.h
	local bbox_hash = bbox.x0.."|"..bbox.y0.."|"..bbox.x1.."|"..bbox.y1
	return doc.file.."|"..pageno.."|"..doc.configurable:hash("|").."|"..bbox_hash.."|"..screen_size_hash
end

function KoptInterface:getAutoBBox(doc, pageno)
	local bbox = {
		x0 = 0, y0 = 0,
		x1 = 0, y1 = 0,
	}
	local context_hash = self:getContextHash(doc, pageno, bbox)
	local hash = "autobbox|"..context_hash
	local cached = Cache:check(hash)
	if not cached then
		local page = doc._document:openPage(pageno)
		local kc = self:createContext(doc, pageno, bbox)
		bbox.x0, bbox.y0, bbox.x1, bbox.y1 = page:getAutoBBox(kc)
		DEBUG("Auto detected bbox", bbox)
		page:close()
		Cache:insert(hash, CacheItem:new{ autobbox = bbox })
		return bbox
	else
		return cached.autobbox
	end
end

function KoptInterface:getPageText(doc, pageno)
	local bbox = doc:getPageBBox(pageno)
	local context_hash = self:getContextHash(doc, pageno, bbox)
	local hash = "pgtext|"..context_hash
	local cached = Cache:check(hash)
	if not cached then
		local kctx_hash = "kctx|"..context_hash
		local cached = Cache:check(kctx_hash)
		if cached then
			local kc = self:waitForContext(cached.kctx)
			local fullwidth, fullheight = kc:getPageDim()
			local text = kc:getWordBoxes(0, 0, fullwidth, fullheight)
			Cache:insert(hash, CacheItem:new{ pgtext = text })
			return text
		end
	else
		return cached.pgtext
	end
end

function KoptInterface:getOCRWord(doc, pageno, rect)
	local bbox = doc:getPageBBox(pageno)
	local context_hash = self:getContextHash(doc, pageno, bbox)
	local hash = "ocrword|"..context_hash..rect.x..rect.y..rect.w..rect.h
	local cached = Cache:check(hash)
	if not cached then
		local kctx_hash = "kctx|"..context_hash
		local cached = Cache:check(kctx_hash)
		if cached then
			local kc = self:waitForContext(cached.kctx)
			local fullwidth, fullheight = kc:getPageDim()
			local ok, word = pcall(
				kc.getOCRWord, kc,
				self.tessocr_data,
				self.ocr_lang,
				self.ocr_type, 
				rect.x, rect.y,
				rect.w, rect.h)
			Cache:insert(hash, CacheItem:new{ ocrword = word })
			return word
		end
	else
		return cached.ocrword
	end
end

--[[
get cached koptcontext for centain page. if context doesn't exist in cache make<commit_msg>try to drop cache before OCR<commit_after>require "dbg"
require "cache"
require "ui/geometry"
require "ui/device"
require "ui/reader/readerconfig"

KoptInterface = {
	tessocr_data = "data",
	ocr_lang = "eng",
	ocr_type = 0, -- default, for more accuracy use 3
}

ContextCacheItem = CacheItem:new{}

function ContextCacheItem:onFree()
	if self.kctx.free then
		DEBUG("free koptcontext", self.kctx)
		self.kctx:free()
	end
end

function KoptInterface:waitForContext(kc)
	-- if koptcontext is being processed in background thread
	-- the isPreCache will return 1.
	while kc and kc:isPreCache() == 1 do
		DEBUG("waiting for background rendering")
		util.usleep(100000)
	end
	return kc
end

--[[
get reflow context
--]]
function KoptInterface:createContext(doc, pageno, bbox)
	-- Now koptcontext keeps track of its dst bitmap reflowed by libk2pdfopt.
	-- So there is no need to check background context when creating new context.
	local kc = KOPTContext.new()
	local screen_size = Screen:getSize()
	kc:setTrim(doc.configurable.trim_page)
	kc:setWrap(doc.configurable.text_wrap)
	kc:setIndent(doc.configurable.detect_indent)
	kc:setRotate(doc.configurable.screen_rotation)
	kc:setColumns(doc.configurable.max_columns)
	kc:setDeviceDim(screen_size.w, screen_size.h)
	kc:setDeviceDPI(doc.screen_dpi)
	kc:setStraighten(doc.configurable.auto_straighten)
	kc:setJustification(doc.configurable.justification)
	kc:setZoom(doc.configurable.font_size)
	kc:setMargin(doc.configurable.page_margin)
	kc:setQuality(doc.configurable.quality)
	kc:setContrast(doc.configurable.contrast)
	kc:setDefectSize(doc.configurable.defect_size)
	kc:setLineSpacing(doc.configurable.line_spacing)
	kc:setWordSpacing(doc.configurable.word_spacing)
	kc:setBBox(bbox.x0, bbox.y0, bbox.x1, bbox.y1)
	if Dbg.is_on then kc:setDebug() end
	return kc
end

function KoptInterface:getContextHash(doc, pageno, bbox)
	local screen_size = Screen:getSize()
	local screen_size_hash = screen_size.w.."|"..screen_size.h
	local bbox_hash = bbox.x0.."|"..bbox.y0.."|"..bbox.x1.."|"..bbox.y1
	return doc.file.."|"..pageno.."|"..doc.configurable:hash("|").."|"..bbox_hash.."|"..screen_size_hash
end

function KoptInterface:getAutoBBox(doc, pageno)
	local bbox = {
		x0 = 0, y0 = 0,
		x1 = 0, y1 = 0,
	}
	local context_hash = self:getContextHash(doc, pageno, bbox)
	local hash = "autobbox|"..context_hash
	local cached = Cache:check(hash)
	if not cached then
		local page = doc._document:openPage(pageno)
		local kc = self:createContext(doc, pageno, bbox)
		bbox.x0, bbox.y0, bbox.x1, bbox.y1 = page:getAutoBBox(kc)
		DEBUG("Auto detected bbox", bbox)
		page:close()
		Cache:insert(hash, CacheItem:new{ autobbox = bbox })
		return bbox
	else
		return cached.autobbox
	end
end

function KoptInterface:getPageText(doc, pageno)
	local bbox = doc:getPageBBox(pageno)
	local context_hash = self:getContextHash(doc, pageno, bbox)
	local hash = "pgtext|"..context_hash
	local cached = Cache:check(hash)
	if not cached then
		local kctx_hash = "kctx|"..context_hash
		local cached = Cache:check(kctx_hash)
		if cached then
			local kc = self:waitForContext(cached.kctx)
			local fullwidth, fullheight = kc:getPageDim()
			local text = kc:getWordBoxes(0, 0, fullwidth, fullheight)
			Cache:insert(hash, CacheItem:new{ pgtext = text })
			return text
		end
	else
		return cached.pgtext
	end
end

function KoptInterface:getOCRWord(doc, pageno, rect)
	local bbox = doc:getPageBBox(pageno)
	local context_hash = self:getContextHash(doc, pageno, bbox)
	local hash = "ocrword|"..context_hash..rect.x..rect.y..rect.w..rect.h
	local cached = Cache:check(hash)
	if not cached then
		local kctx_hash = "kctx|"..context_hash
		local cached = Cache:check(kctx_hash)
		if cached then
			local kc = self:waitForContext(cached.kctx)
			local fullwidth, fullheight = kc:getPageDim()
			--os.execute("echo 3 > /proc/sys/vm/drop_caches")
			local ok, word = pcall(
				kc.getOCRWord, kc,
				self.tessocr_data,
				self.ocr_lang,
				self.ocr_type, 
				rect.x, rect.y,
				rect.w, rect.h)
			Cache:insert(hash, CacheItem:new{ ocrword = word })
			return word
		end
	else
		return cached.ocrword
	end
end

--[[
get cached koptcontext for centain page. if context doesn't exist in cache make
{'fraction_non_alphanumeric': 0.1202971483345315, 'fraction_numerical': 0.008027797747423916, 'mean_word_length': 4.788488210818308, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 4, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5813035', 'n_tokens_mistral': 3308, 'n_tokens_neox': 3130, 'n_words': 679}