<commit_before>require "helpers"
require "mulelib"
require "tc_store"
local c = require "column_db"
local l = require "lightning_mdb"
require "httpd"
local posix_exists,posix = pcall(require,'posix')

pcall(require, "profiler")

local function strip_slash(path_)
  return string.match(path_,"^(.-)/?$")
end

-- this should be in helpers but actually collides with helpers disabling posix for lunit's sake
function first_files(path_,pattern_,max_)
  local num = 0
  return coroutine.wrap(
    function()
      for f in posix.files(path_) do
        if num==max_ then return end
        if string.match(f,pattern_) then
          num = num+1
          coroutine.yield(path_.."/"..f)
        end
      end
    end)
end

local function guess_db(db_path_,readonly_)
  logd("guess_db",db_path_)
  -- strip a trailing / if it exists
  db_path_ = strip_slash(db_path_)<commit_msg>protect mule form missing lightning<commit_after>require "helpers"
require "mulelib"
require "tc_store"
local c = require "column_db"
local _,l = pcall(require,"lightning_mdb")
require "httpd"
local posix_exists,posix = pcall(require,'posix')

pcall(require, "profiler")

local function strip_slash(path_)
  return string.match(path_,"^(.-)/?$")
end

-- this should be in helpers but actually collides with helpers disabling posix for lunit's sake
function first_files(path_,pattern_,max_)
  local num = 0
  return coroutine.wrap(
    function()
      for f in posix.files(path_) do
        if num==max_ then return end
        if string.match(f,pattern_) then
          num = num+1
          coroutine.yield(path_.."/"..f)
        end
      end
    end)
end

local function guess_db(db_path_,readonly_)
  logd("guess_db",db_path_)
  -- strip a trailing / if it exists
  db_path_ = strip_slash(db_path_)
{'fraction_non_alphanumeric': 0.10704225352112676, 'fraction_numerical': 0.0022535211267605635, 'mean_word_length': 3.6736842105263157, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '28661', 'n_tokens_mistral': 597, 'n_tokens_neox': 589, 'n_words': 166}