<commit_before>local util = {}

function util.get_file_contents(filename)
    local f, err = io.open(filename, "r")
    if not f then
        return false, err
    end
    local s = f:read("a")
    f:close()
    if not s then
        return false, "unable to open file " .. filename
    else
        return s
    end
end

function util.set_file_contents(filename, contents)
    local f, err = io.open(filename, "w")
    if not f then
        return false, err
    end
    f:write(contents)
    f:close()
    return true
end

-- Barebones string-based template function for generating C/Lua code.
-- Replaces $VAR placeholders in the `code` template by the corresponding
-- strings in the `substs` table.
function util.render(code, substs)
    return (string.gsub(code, "$([%w_]+)", function(k)
        local v = substs[k]
        if not v then
            error("Internal compiler error: missing template variable " .. k .. "\n" .. code)
        end
        return v
    end))
end

function util.make_visitor(functions)
    return function(node, ...)
        assert(type(node) == "table")
        assert(node._tag)
        local f = assert(functions[node._tag])
        return f(node, ...)
    end
end

--
-- Functional
--

function util.curry(f, a)
    return function(...)
        return f(a, ...)
    end
end

function util.any(f, l)
    for _, elem in ipairs(l) do
        if f(elem) then
            return true
        end
    end
    return false
end

return util
<commit_msg>Add utility to_set function<commit_after>local util = {}

function util.get_file_contents(filename)
    local f, err = io.open(filename, "r")
    if not f then
        return false, err
    end
    local s = f:read("a")
    f:close()
    if not s then
        return false, "unable to open file " .. filename
    else
        return s
    end
end

function util.set_file_contents(filename, contents)
    local f, err = io.open(filename, "w")
    if not f then
        return false, err
    end
    f:write(contents)
    f:close()
    return true
end

-- Barebones string-based template function for generating C/Lua code.
-- Replaces $VAR placeholders in the `code` template by the corresponding
-- strings in the `substs` table.
function util.render(code, substs)
    return (string.gsub(code, "$([%w_]+)", function(k)
        local v = substs[k]
        if not v then
            error("Internal compiler error: missing template variable " .. k .. "\n" .. code)
        end
        return v
    end))
end

function util.make_visitor(functions)
    return function(node, ...)
        assert(type(node) == "table")
        assert(node._tag)
        local f = assert(functions[node._tag])
        return f(node, ...)
    end
end

function util.to_set(array)
    local set = {}
    for _, v in ipairs(array) do
        set[v] = true
    end
    return set
end

--
-- Functional
--

function util.curry(f, a)
    return function(...)
        return f(a, ...)
    end
end

function util.any(f, l)
    for _, elem in ipairs(l) do
        if f(elem) then
            return true
        end
    end
    return false
end

return util

{'fraction_non_alphanumeric': 0.1112540192926045, 'fraction_numerical': 0.0, 'mean_word_length': 2.1519756838905777, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4704476', 'n_tokens_mistral': 980, 'n_tokens_neox': 969, 'n_words': 324}