<commit_before>local BD = require("ui/bidi")
local Device = require("device")
local DocSettings = require("docsettings")
local DocumentRegistry = require("document/documentregistry")
local OpenWithDialog = require("ui/widget/openwithdialog")
local ConfirmBox = require("ui/widget/confirmbox")
local Font = require("ui/font")
local Menu = require("ui/widget/menu")
local UIManager = require("ui/uimanager")
local ffi = require("ffi")
local lfs = require("libs/libkoreader-lfs")
local ffiUtil = require("ffi/util")
local T = ffiUtil.template
local C = ffi.C
local _ = require("gettext")
local N_ = _.ngettext
local Screen = Device.screen
local util = require("util")
local getFileNameSuffix = util.getFileNameSuffix
local getFriendlySize = util.getFriendlySize

ffi.cdef[[
int strcoll (const char *str1, const char *str2);
]]

-- string sort function respecting LC_COLLATE
local function strcoll(str1, str2)
    return C.strcoll(str1, str2) < 0
end

local function kobostrcoll(str1, str2)
    return str1 < str2
end

local FileChooser = Menu:extend{
    cface = Font:getFace("smallinfofont"),
    no_title = true,
    path = lfs.currentdir(),
    show_path = true,
    parent = nil,
    show_hidden = nil,
    exclude_dirs = {"%.sdr$"},
    collate = "strcoll", -- or collate = "access",
    reverse_collate = false,
    path_items = {}, -- store last browsed location(item index) for each path
    perpage = G_reader_settings:readSetting("items_per_page"),
    goto_letter = true,
}

function FileChooser:init()
    self.width = Screen:getWidth()
    -- common dir filter
    self.dir_filter = function(dirname)
        for _, pattern in ipairs(self.exclude_dirs) do
            if dirname:match(pattern) then return false end
        end
        return true
    end
    self.list = function(path, dirs, files, count_only)
        -- lfs.dir directory without permission will give error
        local ok, iter, dir_obj = pcall(lfs.dir, path)
        if ok then
            for f in iter, dir_obj do
                if count_only then
                    if (not self.show_hidden and not util.stringStartsWith(f, "."))
                        or (self.show_hidden and f ~= "." and f ~= ".." and self.dir_filter(f) and not util.stringStartsWith(f, "._"))
                    then
                        table.insert(dirs, true)
                    end
                elseif self.show_hidden or not string.match(f, "^%.[^.]") then
                    local filename = path.."/"..f
                    local attributes = lfs.attributes(filename)
                    if attributes ~= nil then
                        if attributes.mode == "directory" and f ~= "." and f ~= ".." then
                            if self.dir_filter(filename) then<commit_msg>file count: Fixes #5843 (#5858)<commit_after>local BD = require("ui/bidi")
local Device = require("device")
local DocSettings = require("docsettings")
local DocumentRegistry = require("document/documentregistry")
local OpenWithDialog = require("ui/widget/openwithdialog")
local ConfirmBox = require("ui/widget/confirmbox")
local Font = require("ui/font")
local Menu = require("ui/widget/menu")
local UIManager = require("ui/uimanager")
local ffi = require("ffi")
local lfs = require("libs/libkoreader-lfs")
local ffiUtil = require("ffi/util")
local T = ffiUtil.template
local C = ffi.C
local _ = require("gettext")
local N_ = _.ngettext
local Screen = Device.screen
local util = require("util")
local getFileNameSuffix = util.getFileNameSuffix
local getFriendlySize = util.getFriendlySize

ffi.cdef[[
int strcoll (const char *str1, const char *str2);
]]

-- string sort function respecting LC_COLLATE
local function strcoll(str1, str2)
    return C.strcoll(str1, str2) < 0
end

local function kobostrcoll(str1, str2)
    return str1 < str2
end

local FileChooser = Menu:extend{
    cface = Font:getFace("smallinfofont"),
    no_title = true,
    path = lfs.currentdir(),
    show_path = true,
    parent = nil,
    show_hidden = nil,
    exclude_dirs = {"%.sdr$"},
    collate = "strcoll", -- or collate = "access",
    reverse_collate = false,
    path_items = {}, -- store last browsed location(item index) for each path
    perpage = G_reader_settings:readSetting("items_per_page"),
    goto_letter = true,
}

function FileChooser:init()
    self.width = Screen:getWidth()
    -- common dir filter
    self.dir_filter = function(dirname)
        for _, pattern in ipairs(self.exclude_dirs) do
            if dirname:match(pattern) then return false end
        end
        return true
    end
    self.list = function(path, dirs, files, count_only)
        -- lfs.dir directory without permission will give error
        local ok, iter, dir_obj = pcall(lfs.dir, path)
        if ok then
            for f in iter, dir_obj do
                if count_only then
                    if self.dir_filter(f) and ((not self.show_hidden and not util.stringStartsWith(f, "."))
                        or (self.show_hidden and f ~= "." and f ~= ".." and not util.stringStartsWith(f, "._")))
                    then
                        table.insert(dirs, true)
                    end
                elseif self.show_hidden or not string.match(f, "^%.[^.]") then
                    local filename = path.."/"..f
                    local attributes = lfs.attributes(filename)
                    if attributes ~= nil then
                        if attributes.mode == "directory" and f ~= "." and f ~= ".." then
                            if self.dir_filter(filename) then
{'fraction_non_alphanumeric': 0.11314265025343953, 'fraction_numerical': 0.005430847212165098, 'mean_word_length': 2.6087524493794905, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1192231', 'n_tokens_mistral': 1668, 'n_tokens_neox': 1629, 'n_words': 461}