<commit_before>--
-- A script to automatically generate exported function registration
-- code and documentation for those from C source.
-- 
-- The script can also parse documentation comments from Lua code.
-- 


-- Helper functions {{{

function errorf(fmt, ...)
    error(string.format(fmt, unpack(arg)), 2)
end

function fprintf(h, fmt, ...)
    h:write(string.format(fmt, unpack(arg)))
end

function trim(str)
    return string.gsub(str, "^[%s\n]*(.-)[%s\n]*$", "%1")
end

--}}}


-- Some conversion tables {{{

desc2ct={
    ["v"]="void",
    ["i"]="int",
    ["d"]="double",
    ["b"]="bool",
    ["t"]="ExtlTab",
    ["f"]="ExtlFn",
    ["o"]="WObj*",
    ["s"]="char*",
    ["S"]="const char*",
}

ct2desc={
    ["uint"] = "i",
}

for d, t in desc2ct do
    ct2desc[t]=d
end

desc2human={
    ["v"]="void",
    ["i"]="integer",
    ["d"]="double",
    ["b"]="bool",
    ["t"]="table",
    ["f"]="function",
    ["o"]="object",
    ["s"]="string",
    ["S"]="string",
}

--}}}


-- Parser {{{

local fns={}
local classes={}
local chnds={}

function add_chnd(fnt)
    local odesc=string.gsub(fnt.odesc, "S", "s")
    local idesc=string.gsub(fnt.idesc, "S", "s")
    local str="l2chnd_" .. odesc .. "_" .. idesc .. "_"
    
    for i, t in ipairs(fnt.itypes) do
        str=str .. "_" .. t
    end
    
    chnds[str]={odesc=odesc, idesc=idesc, itypes=fnt.itypes}
    fnt.chnd=str
end

function parse_type(t)
    local desc, otype, varname="?", "", ""
    
    -- Remove whitespace at start end end of the string and compress elsewhere.
    t=string.gsub(trim(t), "[%s\n]+", " ")
    -- Remove spaces around asterisks.
    t=string.gsub(t, " *%* *", "*")
    -- Add space after asterisks.
    t=string.gsub(t, "%*", "* ")
    
    -- Check for const
    local is_const=false
    local s, e=string.find(t, "^const +")
    if s then
        is_const=true
        t=string.sub(t, e+1)
    end
    
    -- Find variable name part
    tn=t
    s, e=string.find(tn, " ")
    if s then
        varname=string.sub(tn, e+1)
        tn=string.sub(tn, 1, s-1)
        assert(not string.find(varname, " "))
    end
    
    -- Try to check for supported types
    if ct2desc[tn] then
        desc=ct2desc[tn]
    end
    
    if desc=="o" or desc=="?" then
        s, e=string.find(tn, "^W[%w_]*%*$")
        if s then
            desc="o"
            otype=string.sub(tn, s, e-1)
            classes[otype]=true
        else
            errorf("Error parsing type from \"%s\"", t)
        end
    end
    
    return desc, otype, varname
end

function parse(d)
    local doc=nil
    -- Handle /*EXTL_DOC ... */
    local function do_doc(s)
        --s=string.gsub(s, "/%*EXTL_DOC(.-)%*/", "%1")
        local st=string.len("/*EXTL_DOC")
        local en, _=string.find(s, "%*/")
        if not en then
            errorf("Could not find end of comment in \"%s...\"",
                   string.sub(s, 1, 50))
        end
        
        s=string.sub(s, st+1, en-1)
        s=string.gsub(s, "\n[%s]*%*", "\n")<commit_msg>trunk: changeset 627<commit_after>--
-- A script to automatically generate exported function registration
-- code and documentation for those from C source.
-- 
-- The script can also parse documentation comments from Lua code.
-- 


-- Helper functions {{{

function errorf(fmt, ...)
    error(string.format(fmt, unpack(arg)), 2)
end

function fprintf(h, fmt, ...)
    h:write(string.format(fmt, unpack(arg)))
end

function trim(str)
    return string.gsub(str, "^[%s\n]*(.-)[%s\n]*$", "%1")
end

--}}}


-- Some conversion tables {{{

desc2ct={
    ["v"]="void",
    ["i"]="int",
    ["d"]="double",
    ["b"]="bool",
    ["t"]="ExtlTab",
    ["f"]="ExtlFn",
    ["o"]="WObj*",
    ["s"]="char*",
    ["S"]="const char*",
}

ct2desc={
    ["uint"] = "i",
}

for d, t in desc2ct do
    ct2desc[t]=d
end

desc2human={
    ["v"]="void",
    ["i"]="integer",
    ["d"]="double",
    ["b"]="bool",
    ["t"]="table",
    ["f"]="function",
    ["o"]="object",
    ["s"]="string",
    ["S"]="string",
}

--}}}


-- Parser {{{

local fns={}
local classes={}
local chnds={}

function add_chnd(fnt)
    local odesc=string.gsub(fnt.odesc, "S", "s")
    local idesc=string.gsub(fnt.idesc, "S", "s")
    local str="l2chnd_" .. odesc .. "_" .. idesc .. "_"
    
    for i, t in ipairs(fnt.itypes) do
        str=str .. "_" .. t
    end
    
    chnds[str]={odesc=odesc, idesc=idesc, itypes=fnt.itypes}
    fnt.chnd=str
end

function parse_type(t)
    local desc, otype, varname="?", "", ""
    
    -- Remove whitespace at start end end of the string and compress elsewhere.
    t=string.gsub(trim(t), "[%s\n]+", " ")
    -- Remove spaces around asterisks.
    t=string.gsub(t, " *%* *", "*")
    -- Add space after asterisks.
    t=string.gsub(t, "%*", "* ")
    
    -- Check for const
    local is_const=""
    local s, e=string.find(t, "^const +")
    if s then
        is_const="const "
        t=string.sub(t, e+1)
    end
    
    -- Find variable name part
    tn=t
    s, e=string.find(tn, " ")
    if s then
        varname=string.sub(tn, e+1)
        tn=string.sub(tn, 1, s-1)
        assert(not string.find(varname, " "))
    end
    
    -- Try to check for supported types
    desc = ct2desc[is_const .. tn]
    
    if not desc or desc=="o" then
        s, e=string.find(tn, "^W[%w_]*%*$")
        if s then
            desc="o"
            otype=string.sub(tn, s, e-1)
            classes[otype]=true
        else
            errorf("Error parsing type from \"%s\"", t)
        end
    end
    
    return desc, otype, varname
end

function parse(d)
    local doc=nil
    -- Handle /*EXTL_DOC ... */
    local function do_doc(s)
        --s=string.gsub(s, "/%*EXTL_DOC(.-)%*/", "%1")
        local st=string.len("/*EXTL_DOC")
        local en, _=string.find(s, "%*/")
        if not en then
            errorf("Could not find end of comment in \"%s...\"",
                   string.sub(s, 1, 50))
        end
        
        s=string.sub(s, st+1, en-1)
        s=string.gsub(s, "\n[%s]*%*", "\n")
{'fraction_non_alphanumeric': 0.20541798238324746, 'fraction_numerical': 0.007312614259597806, 'mean_word_length': 2.1824431517715492, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2522033', 'n_tokens_mistral': 2335, 'n_tokens_neox': 2275, 'n_words': 524}