<commit_before>local args = { ... }

--[[
local t={
	[1001]={
		["ID"] = 1001,
		["Description"] = "ȼ",
	},
	[1004]={
		["ID"] = 1004,
		["Description"] = "ò׿סгࣹ׷",
	},
}
]]
function work(filepath)
    print("pack:", filepath)
    local keys = {}

    local file = io.open(filepath, "rb")
    local str = file:read("*all")
    file:close()
    
    if str:find("setmetatable") then return end
    
    local isParse = false
    string.gsub(str, "t=%{.-%[%d-%]=%{(.-)%}%,%s", function(val)
        if not isParse then 
            isParse = true
            
            string.gsub(val, '%[(%b"")%]', function(key)                
                keys[#keys + 1] = string.sub(key, 2, #key - 1)
            end)
            
        end
    end)

    local ret, stat = dofile(filepath)
 
    local indexs = {}
    local indexContents = "local indexs = { "
    for k, v in pairs(keys) do 
        indexs[v] = k
        indexContents = indexContents .. string.format([[ ["%s"] = %d, ]], v, k)
    end
    indexContents = indexContents .. ' }'
    
    local content = string.format([[
%s
local mt = {  
    __index = function(t, key)
        local index = indexs[key]
        if index then 
            return t[index]
        end
    end
}

local t = {]], indexContents)
    
    for k, v in pairs(ret) do 
        local line = string.format("[%d] = { ", v[keys[1]])
        for _, key in ipairs(keys) do 
            local val = v[key]
            local tp = type(val)
            if tp == 'string' then 
                line = string.format("%s\"%s\", ", line, val)
            elseif tp == 'bool' then 
                line = string.format("%s%s, ", line, val and "true" or "false")
            elseif tp == 'number' then 
                line = string.format("%s%d, ", line, val)
            end
        end
        line = line .. '},'
        
        content = string.format('%s\n\t%s', content, line)
    end
    
    content = content .. '\n}\nsetmetatable(t, mt)\nreturn t'
 
    local file = io.open(filepath .. '', "wb")
    file:write(content)
    file:close()   
end

for k, v in ipairs(args) do 
    work(v)
end<commit_msg>add tools for retools<commit_after>local args = { ... }

--[[
local t={
	[1001]={
		["ID"] = 1001,
		["Description"] = "ȼ",
	},
	[1004]={
		["ID"] = 1004,
		["Description"] = "ò׿סгࣹ׷",
	},
}
]]
function work(filepath)
    print("pack:", filepath)
    local keys = {}

    local file = io.open(filepath, "rb")
    local str = file:read("*all")
    file:close()
    
    if str:find("setmetatable") then return end
    
    local isParse = false
    string.gsub(str, "t=%{.-%[%d-%]=%{(.-)%}%,%s", function(val)
        if not isParse then 
            isParse = true
            
            string.gsub(val, '%[(%b"")%]', function(key)                
                keys[#keys + 1] = string.sub(key, 2, #key - 1)
            end)
            
        end
    end)

    local ret, stat = dofile(filepath)
 
    local indexs = {}
    local indexContents = "local indexs = { "
    for k, v in pairs(keys) do 
        indexs[v] = k
        indexContents = indexContents .. string.format([[ ["%s"] = %d, ]], v, k)
    end
    indexContents = indexContents .. ' }'
    
    local content = string.format([[
local setmetatable = setmetatable
%s
local mt = {  
    __index = function(t, key)
        local index = indexs[key]
        if index then 
            return t[index]
        end
    end
}

local t = {]], indexContents)
    
    for k, v in pairs(ret) do 
        local line = string.format("[%d] = setmetatable({ ", v[keys[1]])
        for _, key in ipairs(keys) do 
            local val = v[key]
            local tp = type(val)
            if tp == 'string' then 
                line = string.format([==[%s[=[%s]=], ]==], line, string.gsub(val, "\n", "\\n"))
            elseif tp == 'bool' then 
                line = string.format([[%s%s, ]], line, val and "true" or "false")
            elseif tp == 'number' then 
                line = string.format([[%s%d, ]], line, val)
            end
        end
        line = line .. '}, mt),'
        
        content = string.format('%s\n\t%s', content, line)
    end
    
    content = content .. '\n}\nreturn t'
 
    local file = io.open(filepath .. '', "wb")
    file:write(content)
    file:close()   
end

for k, v in ipairs(args) do 
    work(v)
end
{'fraction_non_alphanumeric': 0.17115251897860592, 'fraction_numerical': 0.009201748332183115, 'mean_word_length': 1.7466835123183828, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3505054', 'n_tokens_mistral': 1541, 'n_tokens_neox': 1479, 'n_words': 378}