<commit_before>do
    local Entity, parent = torch.class('dokx.Entity')

    --[[ Abstract Base Class for items extracted from lua source.

    Fields `_package`, `_file` and `_lineNo` keep track of where the item was extracted from.

    --]]
    function Entity:__init(package, file, lineNo)
        assert(package)
        assert(file)
        assert(lineNo)
        self._package = package
        self._file = file
        self._lineNo = lineNo
    end

    -- Return the name of the package from which this documentation entity was extracted
    function Entity:package()
        return self._package
    end

    -- Return the name of the source file from which this documentation entity was extracted
    function Entity:file()
        return self._file
    end

    -- Return the (last) line number of this documentation entity in the source file
    function Entity:lineNo()
        return self._lineNo
    end
end

--[[ Information about a comment string, as extracted from lua source ]]
do
    local Comment, parent = torch.class("dokx.Comment", "dokx.Entity")
    function Comment:__init(text, ...)
        parent.__init(self, ...)
        self._text = dokx._normalizeComment(text)
    end
    -- Return a new dokx.Comment by concatenating this with another comment
    function Comment:combine(other)
        return dokx.Comment(self._text .. other._text, self._package, self._file, self._lineNo)
    end
    -- Return a string representation of this Comment entity
    function Comment:str()
        return "{Comment: " .. self._text .. "}"
    end
    -- Return this comment's text
    function Comment:text()
        return self._text
    end
end

do
    local File, parent = torch.class("dokx.File", "dokx.Entity")
    function File:__init(text, ...)
        parent.__init(self, ...)
        self._text = dokx._normalizeComment(text)
    end
    -- Return a string representation of this File entity
    function File:str()
        return "{File: " .. self._text .. "}"
    end
    -- Return this file's docString
    function File:text()
        return self._text
    end
end

do
    --[[ Information about a Function, as extracted from lua source ]]
    local Function, parent = torch.class("dokx.Function", "dokx.Entity")
    function Function:__init(name, args, ...)
        parent.__init(self, ...)
        assert(name)
        self._method = false
        self._local = false
        local hasClass = false
        local pos = name:find(":")
        if pos then
            hasClass = true
            self._method = true
        else
            pos = name:find("%.")
            if pos and name:sub(1, pos-1) ~= self:package() then
                hasClass = true
            end
        end
        if hasClass then
            self._className = name:sub(1, pos-1)
            self._name = name:sub(pos+1, -1)
        else
            self._className = false

            pos = name:find("%.")
            if pos and name:sub(1, pos-1) == self:package() then
                self._name = name:sub(pos+1, -1)
            else
                self._name = name
            end
        end
        self._args = args
    end
    -- Set whether this is a local function
    function Function:setLocal(localVal) self._local = localVal end
    -- Return whether this is a local function
    function Function:isLocal() return self._local end
    -- Return whether this is a private function
    function Function:isPrivate() return stringx.startswith(self._name, "_") end
    -- Return a string describing the args consumed by this function
    function Function:args() return self._args end
    -- Return the name of this function
    function Function:name() return self._name end
    -- Return the name of the class to which this function belongs, or false if it's not a method at all
    function Function:class() return self._className end
    function Function:nameWithClass()
        local name = self._name
        if self._className then
            if name == '__init' then
                name = self._className
            else
                if self._method then
                    name = self._className .. ":" .. name
                else
                    name = self._className .. "." .. name
                end<commit_msg>Ensure __init and other methods are not considered private<commit_after>-- We keep a whitelist of methods that begin with an underscore but which
-- nevertheless should be included in generated documentation
local _metamethods = {
        __init = true,
        __index = true,
        __newindex = true,
        __mode = true,
        __call = true,
        __metatable = true,
        __tostring = true,
        __len = true,
        __gc = true,
        __unm = true,
        __add = true,
        __sub = true,
        __mul = true,
        __div = true,
        __mod = true,
        __div = true,
        __pow = true,
        __concat = true,
        __eq = true,
        __lt = true,
        __le = true
    }

do
    local Entity, parent = torch.class('dokx.Entity')

    --[[ Abstract Base Class for items extracted from lua source.

    Fields `_package`, `_file` and `_lineNo` keep track of where the item was extracted from.

    --]]
    function Entity:__init(package, file, lineNo)
        assert(package)
        assert(file)
        assert(lineNo)
        self._package = package
        self._file = file
        self._lineNo = lineNo
    end

    -- Return the name of the package from which this documentation entity was extracted
    function Entity:package()
        return self._package
    end

    -- Return the name of the source file from which this documentation entity was extracted
    function Entity:file()
        return self._file
    end

    -- Return the (last) line number of this documentation entity in the source file
    function Entity:lineNo()
        return self._lineNo
    end
end

--[[ Information about a comment string, as extracted from lua source ]]
do
    local Comment, parent = torch.class("dokx.Comment", "dokx.Entity")
    function Comment:__init(text, ...)
        parent.__init(self, ...)
        self._text = dokx._normalizeComment(text)
    end
    -- Return a new dokx.Comment by concatenating this with another comment
    function Comment:combine(other)
        return dokx.Comment(self._text .. other._text, self._package, self._file, self._lineNo)
    end
    -- Return a string representation of this Comment entity
    function Comment:str()
        return "{Comment: " .. self._text .. "}"
    end
    -- Return this comment's text
    function Comment:text()
        return self._text
    end
end

do
    local File, parent = torch.class("dokx.File", "dokx.Entity")
    function File:__init(text, ...)
        parent.__init(self, ...)
        self._text = dokx._normalizeComment(text)
    end
    -- Return a string representation of this File entity
    function File:str()
        return "{File: " .. self._text .. "}"
    end
    -- Return this file's docString
    function File:text()
        return self._text
    end
end

do
    --[[ Information about a Function, as extracted from lua source ]]
    local Function, parent = torch.class("dokx.Function", "dokx.Entity")
    function Function:__init(name, args, ...)
        parent.__init(self, ...)
        assert(name)
        self._method = false
        self._local = false
        local hasClass = false
        local pos = name:find(":")
        if pos then
            hasClass = true
            self._method = true
        else
            pos = name:find("%.")
            if pos and name:sub(1, pos-1) ~= self:package() then
                hasClass = true
            end
        end
        if hasClass then
            self._className = name:sub(1, pos-1)
            self._name = name:sub(pos+1, -1)
        else
            self._className = false

            pos = name:find("%.")
            if pos and name:sub(1, pos-1) == self:package() then
                self._name = name:sub(pos+1, -1)
            else
                self._name = name
            end
        end
        self._args = args
    end
    -- Set whether this is a local function
    function Function:setLocal(localVal) self._local = localVal end
    -- Return whether this is a local function
    function Function:isLocal() return self._local end
    -- Return whether this is a private function
    function Function:isPrivate()
        return stringx.startswith(self._name, "_") and not _metamethods[self._name]
    end
    -- Return a string describing the args consumed by this function
    function Function:args() return self._args end
    -- Return the name of this function
    function Function:name() return self._name end
    -- Return the name of the class to which this function belongs, or false if it's not a method at all
    function Function:class() return self._className end
    function Function:nameWithClass()
        local name = self._name
        if self._className then
            if name == '__init' then
                name = self._className
            else
                if self._method then
                    name = self._className .. ":" .. name
                else
                    name = self._className .. "." .. name
                end
{'fraction_non_alphanumeric': 0.09053631164346607, 'fraction_numerical': 0.0021581957483543758, 'mean_word_length': 1.9724182168056446, 'pattern_counts': {'":': 4, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6112273', 'n_tokens_mistral': 2474, 'n_tokens_neox': 2421, 'n_words': 865}