<commit_before>local have_ffi, ffi = pcall(require, "ffi")
local want_ffi = os.getenv "LGUMBO_USE_FFI"
local gumbo

if have_ffi == true then
    if want_ffi == "1" then
        gumbo = require "gumbo.ffi"
    elseif want_ffi == "0" then
        gumbo = require "cgumbo"
    else -- use default
        if jit then -- prefer FFI for LuaJIT
            gumbo = require "gumbo.ffi"
        else -- prefer C module over (slow) luaffi
            gumbo = require "cgumbo"
        end
    end
else
    if want_ffi == "1" then
        error "Explicitly requested FFI module but FFI not available"
    else
        gumbo = require "cgumbo"
    end
end

function gumbo.parse_file(filename, tab_stop)
    local file, err = io.open(filename)
    if file then
        local text = file:read("*a")
        file:close()
        return gumbo.parse(text, tab_stop)
    else
        return nil, err
    end
end

return gumbo
<commit_msg>Handle read errors properly in gumbo.parse_file() function<commit_after>local have_ffi, ffi = pcall(require, "ffi")
local want_ffi = os.getenv "LGUMBO_USE_FFI"
local gumbo

if have_ffi == true then
    if want_ffi == "1" then
        gumbo = require "gumbo.ffi"
    elseif want_ffi == "0" then
        gumbo = require "cgumbo"
    else -- use default
        if jit then -- prefer FFI for LuaJIT
            gumbo = require "gumbo.ffi"
        else -- prefer C module over (slow) luaffi
            gumbo = require "cgumbo"
        end
    end
else
    if want_ffi == "1" then
        error "Explicitly requested FFI module but FFI not available"
    else
        gumbo = require "cgumbo"
    end
end

function gumbo.parse_file(filename, tab_stop)
    local file, openerr = io.open(filename)
    if file then
        local text, readerr = file:read("*a")
        file:close()
        if text then
            return gumbo.parse(text, tab_stop)
        else
            return nil, readerr
        end
    else
        return nil, openerr
    end
end

return gumbo

{'fraction_non_alphanumeric': 0.08215725806451613, 'fraction_numerical': 0.0030241935483870967, 'mean_word_length': 2.1558028616852147, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1623445', 'n_tokens_mistral': 683, 'n_tokens_neox': 651, 'n_words': 213}