<commit_before>local MultiInputDialog = require("ui/widget/multiinputdialog")
local ButtonDialog = require("ui/widget/buttondialog")
local InfoMessage = require("ui/widget/infomessage")
local PathChooser = require("ui/widget/pathchooser")
local lfs = require("libs/libkoreader-lfs")
local OPDSParser = require("ui/opdsparser")
local NetworkMgr = require("ui/networkmgr")
local UIManager = require("ui/uimanager")
local CacheItem = require("cacheitem")
local Menu = require("ui/widget/menu")
local Screen = require("ui/screen")
local Device = require("ui/device")
local url = require('socket.url')
local util = require("ffi/util")
local Cache = require("cache")
local DEBUG = require("dbg")
local _ = require("gettext")
local ffi = require("ffi")

local socket = require('socket')
local http = require('socket.http')
local https = require('ssl.https')
local ltn12 = require('ltn12')

local CatalogCacheItem = CacheItem:new{
    size = 1024,  -- fixed size for catalog item
}

-- cache catalog parsed from feed xml
local CatalogCache = Cache:new{
    max_memsize = 20*1024, -- keep only 20 cache items
    current_memsize = 0,
    cache = {},
    cache_order = {},
}

local OPDSBrowser = Menu:extend{
    opds_servers = {},
    calibre_name = _("Local calibre catalog"),

    catalog_type = "application/atom%+xml",
    search_type = "application/opensearchdescription%+xml",
    acquisition_rel = "^http://opds%-spec%.org/acquisition",
    thumbnail_rel = "http://opds-spec.org/image/thumbnail",

    formats = {
        ["application/epub+zip"] = "EPUB",
        ["application/fb2+zip"] = "FB2",
        ["application/pdf"] = "PDF",
        ["text/plain"] = "TXT",
        ["application/x-mobipocket-ebook"] = "MOBI",
        ["application/x-mobi8-ebook"] = "AZW3",
    },

    width = Screen:getWidth(),
    height = Screen:getHeight(),
    no_title = false,
    parent = nil,
}

function OPDSBrowser:init()
    self.item_table = self:genItemTableFromRoot(self.opds_servers)
    Menu.init(self) -- call parent's init()
end

function OPDSBrowser:addServerFromInput(fields)
    DEBUG("input catalog", fields)
    local servers = G_reader_settings:readSetting("opds_servers") or {}
    table.insert(servers, {
        title = fields[1],
        url = fields[2],
    })
    G_reader_settings:saveSetting("opds_servers", servers)
    self:init()
end

function OPDSBrowser:editCalibreFromInput(fields)
    DEBUG("input calibre server", fields)
    local calibre = G_reader_settings:readSetting("calibre_opds") or {}
    if fields[1] then
        calibre.host = fields[1]
    end
    if tonumber(fields[2]) then
        calibre.port = fields[2]
    end
    G_reader_settings:saveSetting("calibre_opds", calibre)
    self:init()
end

function OPDSBrowser:addNewCatalog()
    self.add_server_dialog = MultiInputDialog:new{
        title = _("Add OPDS catalog"),
        fields = {
            {
                text = "",
                hint = _("Catalog Name"),
            },
            {
                text = "",
                hint = _("Catalog URL"),
            },
        },
        buttons = {
            {
                {
                    text = _("Cancel"),
                    callback = function()
                        self.add_server_dialog:onClose()
                        UIManager:close(self.add_server_dialog)
                    end
                },
                {
                    text = _("Add"),
                    callback = function()
                        self.add_server_dialog:onClose()
                        UIManager:close(self.add_server_dialog)
                        self:addServerFromInput(MultiInputDialog:getFields())
                    end
                },
            },
        },
        width = Screen:getWidth() * 0.95,
        height = Screen:getHeight() * 0.2,
    }
    self.add_server_dialog:onShowKeyboard()
    UIManager:show(self.add_server_dialog)
end

function OPDSBrowser:editCalibreServer()
    local calibre = G_reader_settings:readSetting("calibre_opds") or {}
    self.add_server_dialog = MultiInputDialog:new{
        title = _("Edit local calibre host and port"),
        fields = {
            {
                -- TODO: get IP address of current device
                text = calibre.host or "192.168.1.1",
                hint = _("Calibre host"),
            },
            {
                text = calibre.port and tostring(calibre.port) or "8080",
                hint = _("Calibre port"),
            },
        },
        buttons = {
            {
                {
                    text = _("Cancel"),
                    callback = function()
                        self.add_server_dialog:onClose()
                        UIManager:close(self.add_server_dialog)
                    end
                },
                {
                    text = _("Apply"),
                    callback = function()
                        self.add_server_dialog:onClose()
                        UIManager:close(self.add_server_dialog)
                        self:editCalibreFromInput(MultiInputDialog:getFields())
                    end
                },
            },
        },
        width = Screen:getWidth() * 0.95,
        height = Screen:getHeight() * 0.2,
    }
    self.add_server_dialog:onShowKeyboard()
    UIManager:show(self.add_server_dialog)
end

function OPDSBrowser:genItemTableFromRoot()
    local item_table = {}
    for i, server in ipairs(self.opds_servers) do
        table.insert(item_table, {
            text = server.title,
            content = server.subtitle,
            url = server.url,
            baseurl = server.baseurl,
        })
    end
    local added_servers = G_reader_settings:readSetting("opds_servers") or {}
    for i, server in ipairs(added_servers) do
        table.insert(item_table, {
            text = server.title,
            content = server.subtitle,
            url = server.url,
            baseurl = server.baseurl,
            deletable = true,
            editable = true,
        })
    end
    local calibre_opds = G_reader_settings:readSetting("calibre_opds") or {}
    local calibre_callback = nil
    if not calibre_opds.host or not calibre_opds.port then
        table.insert(item_table, {
            text = self.calibre_name,
            callback = function()
                self:editCalibreServer()
            end,
            deletable = false,
        })
    else
        table.insert(item_table, {
            text = self.calibre_name,
            url = string.format("http://%s:%d/opds",
                calibre_opds.host, calibre_opds.port),
            editable = true,
            deletable = false,
        })
    end
    table.insert(item_table, {
        text = _("Add new OPDS catalog"),
        callback = function()
            self:addNewCatalog()
        end,
    })
    return item_table
end

function OPDSBrowser:fetchFeed(feed_url)
    local headers, request, sink = {}, {}, {}
    local parsed = url.parse(feed_url)
    request['url'] = feed_url
    request['method'] = 'GET'
    request['sink'] = ltn12.sink.table(sink)
    DEBUG("request", request)
    http.TIMEOUT, https.TIMEOUT = 10, 10
    local httpRequest = parsed.scheme == 'http' and http.request or https.request
    local code, headers, status = socket.skip(1, httpRequest(request))

    -- raise error message when network is unavailable
    if headers == nil then
        error(code)
    end

    local xml = table.concat(sink)
    if xml ~= "" then
        --DEBUG("xml", xml)
        return xml
    end
end

function OPDSBrowser:parseFeed(feed_url)
    local feed = nil
    local hash = "opds|catalog|" .. feed_url
    local cache = CatalogCache:check(hash)
    if cache then
        feed = cache.feed
    else
        DEBUG("cache", hash)
        feed = self:fetchFeed(feed_url)
        if feed then
            local cache = CatalogCacheItem:new{
                feed = feed
            }
            CatalogCache:insert(hash, cache)
        end
    end
    if feed then
        return OPDSParser:parse(feed)
    end
end

function OPDSBrowser:getCatalog(feed_url)
    local ok, catalog = pcall(self.parseFeed, self, feed_url)
    -- prompt users to turn on Wifi if network is unreachable
    if not ok and catalog and catalog:find("Network is unreachable") then
        NetworkMgr:promptWifiOn()
        return
    elseif not ok and catalog then
        DEBUG("cannot get catalog info from", feed_url, catalog)
        UIManager:show(InfoMessage:new{
            text = _("Cannot get catalog info from ") .. feed_url,
        })
        return
    end

    if ok and catalog then
        DEBUG("catalog", catalog)
        return catalog
    end
end

function OPDSBrowser:genItemTableFromURL(item_url, base_url)
    local item_table = {}
    local catalog = self:getCatalog(item_url or base_url)
    if catalog then
        local feed = catalog.feed or catalog
        local function build_href(href)
            if href:match("^http") then
                return href
            elseif href:match("^//") then
                local parsed = url.parse(item_url or base_url)
                if parsed and parsed.scheme then
                    return parsed.scheme .. ":" .. href
                else
                    return "http:" .. href
                end
            elseif base_url then
                return base_url .. "/" .. href
            elseif item_url then
                local parsed = url.parse(item_url)
                -- update item url with href parts(mostly path and query)
                for k, v in pairs(url.parse(href) or {}) do
                    if k == "path" then
                        v = "/" .. v
                    end
                    parsed[k] = v
                end
                return url.build(parsed)
            end
        end
        local hrefs = {}<commit_msg>get rid of query filed of baseurl when building href otherwise there will be "?offset=90" query in downloading href for calibre opds server which doesn't accept the url<commit_after>local MultiInputDialog = require("ui/widget/multiinputdialog")
local ButtonDialog = require("ui/widget/buttondialog")
local InfoMessage = require("ui/widget/infomessage")
local PathChooser = require("ui/widget/pathchooser")
local lfs = require("libs/libkoreader-lfs")
local OPDSParser = require("ui/opdsparser")
local NetworkMgr = require("ui/networkmgr")
local UIManager = require("ui/uimanager")
local CacheItem = require("cacheitem")
local Menu = require("ui/widget/menu")
local Screen = require("ui/screen")
local Device = require("ui/device")
local url = require('socket.url')
local util = require("ffi/util")
local Cache = require("cache")
local DEBUG = require("dbg")
local _ = require("gettext")
local ffi = require("ffi")

local socket = require('socket')
local http = require('socket.http')
local https = require('ssl.https')
local ltn12 = require('ltn12')

local CatalogCacheItem = CacheItem:new{
    size = 1024,  -- fixed size for catalog item
}

-- cache catalog parsed from feed xml
local CatalogCache = Cache:new{
    max_memsize = 20*1024, -- keep only 20 cache items
    current_memsize = 0,
    cache = {},
    cache_order = {},
}

local OPDSBrowser = Menu:extend{
    opds_servers = {},
    calibre_name = _("Local calibre catalog"),

    catalog_type = "application/atom%+xml",
    search_type = "application/opensearchdescription%+xml",
    acquisition_rel = "^http://opds%-spec%.org/acquisition",
    thumbnail_rel = "http://opds-spec.org/image/thumbnail",

    formats = {
        ["application/epub+zip"] = "EPUB",
        ["application/fb2+zip"] = "FB2",
        ["application/pdf"] = "PDF",
        ["text/plain"] = "TXT",
        ["application/x-mobipocket-ebook"] = "MOBI",
        ["application/x-mobi8-ebook"] = "AZW3",
    },

    width = Screen:getWidth(),
    height = Screen:getHeight(),
    no_title = false,
    parent = nil,
}

function OPDSBrowser:init()
    self.item_table = self:genItemTableFromRoot(self.opds_servers)
    Menu.init(self) -- call parent's init()
end

function OPDSBrowser:addServerFromInput(fields)
    DEBUG("input catalog", fields)
    local servers = G_reader_settings:readSetting("opds_servers") or {}
    table.insert(servers, {
        title = fields[1],
        url = fields[2],
    })
    G_reader_settings:saveSetting("opds_servers", servers)
    self:init()
end

function OPDSBrowser:editCalibreFromInput(fields)
    DEBUG("input calibre server", fields)
    local calibre = G_reader_settings:readSetting("calibre_opds") or {}
    if fields[1] then
        calibre.host = fields[1]
    end
    if tonumber(fields[2]) then
        calibre.port = fields[2]
    end
    G_reader_settings:saveSetting("calibre_opds", calibre)
    self:init()
end

function OPDSBrowser:addNewCatalog()
    self.add_server_dialog = MultiInputDialog:new{
        title = _("Add OPDS catalog"),
        fields = {
            {
                text = "",
                hint = _("Catalog Name"),
            },
            {
                text = "",
                hint = _("Catalog URL"),
            },
        },
        buttons = {
            {
                {
                    text = _("Cancel"),
                    callback = function()
                        self.add_server_dialog:onClose()
                        UIManager:close(self.add_server_dialog)
                    end
                },
                {
                    text = _("Add"),
                    callback = function()
                        self.add_server_dialog:onClose()
                        UIManager:close(self.add_server_dialog)
                        self:addServerFromInput(MultiInputDialog:getFields())
                    end
                },
            },
        },
        width = Screen:getWidth() * 0.95,
        height = Screen:getHeight() * 0.2,
    }
    self.add_server_dialog:onShowKeyboard()
    UIManager:show(self.add_server_dialog)
end

function OPDSBrowser:editCalibreServer()
    local calibre = G_reader_settings:readSetting("calibre_opds") or {}
    self.add_server_dialog = MultiInputDialog:new{
        title = _("Edit local calibre host and port"),
        fields = {
            {
                -- TODO: get IP address of current device
                text = calibre.host or "192.168.1.1",
                hint = _("Calibre host"),
            },
            {
                text = calibre.port and tostring(calibre.port) or "8080",
                hint = _("Calibre port"),
            },
        },
        buttons = {
            {
                {
                    text = _("Cancel"),
                    callback = function()
                        self.add_server_dialog:onClose()
                        UIManager:close(self.add_server_dialog)
                    end
                },
                {
                    text = _("Apply"),
                    callback = function()
                        self.add_server_dialog:onClose()
                        UIManager:close(self.add_server_dialog)
                        self:editCalibreFromInput(MultiInputDialog:getFields())
                    end
                },
            },
        },
        width = Screen:getWidth() * 0.95,
        height = Screen:getHeight() * 0.2,
    }
    self.add_server_dialog:onShowKeyboard()
    UIManager:show(self.add_server_dialog)
end

function OPDSBrowser:genItemTableFromRoot()
    local item_table = {}
    for i, server in ipairs(self.opds_servers) do
        table.insert(item_table, {
            text = server.title,
            content = server.subtitle,
            url = server.url,
            baseurl = server.baseurl,
        })
    end
    local added_servers = G_reader_settings:readSetting("opds_servers") or {}
    for i, server in ipairs(added_servers) do
        table.insert(item_table, {
            text = server.title,
            content = server.subtitle,
            url = server.url,
            baseurl = server.baseurl,
            deletable = true,
            editable = true,
        })
    end
    local calibre_opds = G_reader_settings:readSetting("calibre_opds") or {}
    local calibre_callback = nil
    if not calibre_opds.host or not calibre_opds.port then
        table.insert(item_table, {
            text = self.calibre_name,
            callback = function()
                self:editCalibreServer()
            end,
            deletable = false,
        })
    else
        table.insert(item_table, {
            text = self.calibre_name,
            url = string.format("http://%s:%d/opds",
                calibre_opds.host, calibre_opds.port),
            editable = true,
            deletable = false,
        })
    end
    table.insert(item_table, {
        text = _("Add new OPDS catalog"),
        callback = function()
            self:addNewCatalog()
        end,
    })
    return item_table
end

function OPDSBrowser:fetchFeed(feed_url)
    local headers, request, sink = {}, {}, {}
    local parsed = url.parse(feed_url)
    request['url'] = feed_url
    request['method'] = 'GET'
    request['sink'] = ltn12.sink.table(sink)
    DEBUG("request", request)
    http.TIMEOUT, https.TIMEOUT = 10, 10
    local httpRequest = parsed.scheme == 'http' and http.request or https.request
    local code, headers, status = socket.skip(1, httpRequest(request))

    -- raise error message when network is unavailable
    if headers == nil then
        error(code)
    end

    local xml = table.concat(sink)
    if xml ~= "" then
        --DEBUG("xml", xml)
        return xml
    end
end

function OPDSBrowser:parseFeed(feed_url)
    local feed = nil
    local hash = "opds|catalog|" .. feed_url
    local cache = CatalogCache:check(hash)
    if cache then
        feed = cache.feed
    else
        DEBUG("cache", hash)
        feed = self:fetchFeed(feed_url)
        if feed then
            local cache = CatalogCacheItem:new{
                feed = feed
            }
            CatalogCache:insert(hash, cache)
        end
    end
    if feed then
        return OPDSParser:parse(feed)
    end
end

function OPDSBrowser:getCatalog(feed_url)
    local ok, catalog = pcall(self.parseFeed, self, feed_url)
    -- prompt users to turn on Wifi if network is unreachable
    if not ok and catalog and catalog:find("Network is unreachable") then
        NetworkMgr:promptWifiOn()
        return
    elseif not ok and catalog then
        DEBUG("cannot get catalog info from", feed_url, catalog)
        UIManager:show(InfoMessage:new{
            text = _("Cannot get catalog info from ") .. feed_url,
        })
        return
    end

    if ok and catalog then
        DEBUG("catalog", catalog)
        return catalog
    end
end

function OPDSBrowser:genItemTableFromURL(item_url, base_url)
    local item_table = {}
    local catalog = self:getCatalog(item_url or base_url)
    if catalog then
        local feed = catalog.feed or catalog
        local function build_href(href)
            if href:match("^http") then
                return href
            elseif href:match("^//") then
                local parsed = url.parse(item_url or base_url)
                if parsed and parsed.scheme then
                    return parsed.scheme .. ":" .. href
                else
                    return "http:" .. href
                end
            elseif base_url then
                return base_url .. "/" .. href
            elseif item_url then
                local parsed = url.parse(item_url)
                -- get rid of query field of base url
                parsed.query = nil
                -- update item url with href parts(mostly path and query)
                for k, v in pairs(url.parse(href) or {}) do
                    if k == "path" then
                        v = "/" .. v
                    end
                    parsed[k] = v
                end
                return url.build(parsed)
            end
        end
        local hrefs = {}
{'fraction_non_alphanumeric': 0.1049444500050045, 'fraction_numerical': 0.005705134621159043, 'mean_word_length': 1.8839659402511184, 'pattern_counts': {'":': 2, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 16}, 'pii_count': 2, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6065240', 'n_tokens_mistral': 5787, 'n_tokens_neox': 5686, 'n_words': 1319}