<commit_before>local utils = {}

local dir_sep = package.config:sub(1,1)

utils.is_windows = dir_sep == "\\"

utils.has_lfs, utils.lfs = pcall(require, "lfs")

if utils.has_lfs then
   -- Returns whether path points to a directory. 
   function utils.is_dir(path)
      return lfs.attributes(path, "mode") == "directory"
   end

   -- Returns whether path points to a file. 
   function utils.is_file(path)
      return lfs.attributes(path, "mode") == "file"
   end

   -- Returns list of all files in directory matching pattern. 
   function utils.extract_files(dir_path, pattern)
      local res = {}

      local function scan(dir_path)
         for path in lfs.dir(dir_path) do
            if path ~= "." and path ~= ".." then
               local full_path = dir_path .. dir_sep .. path

               if utils.is_dir(full_path) then
                  scan(full_path)
               elseif path:match(pattern) and utils.is_file(full_path) then
                  table.insert(res, full_path)
               end
            end
         end
      end

      scan(dir_path)
      table.sort(res)
      return res
   end

   -- Returns modification time for a file. 
   function utils.mtime(path)
      return lfs.attributes(path, "modification")
   end
else
   -- No luafilesystem. Effectively disable recursive directory checking.
   -- Using something like os.execute("ls") may be possible but is a hack.

   function utils.is_dir(_)
      return false
   end

   function utils.is_file(path)
      local fh = io.open(path)

      if fh then
         fh:close()
         return true
      else
         return false
      end
   end

   function utils.extract_files(_, _)
      return {}<commit_msg>Use lfs as utils.lfs, not global lfs<commit_after>local utils = {}

local dir_sep = package.config:sub(1,1)

utils.is_windows = dir_sep == "\\"

utils.has_lfs, utils.lfs = pcall(require, "lfs")

if utils.has_lfs then
   -- Returns whether path points to a directory. 
   function utils.is_dir(path)
      return utils.lfs.attributes(path, "mode") == "directory"
   end

   -- Returns whether path points to a file. 
   function utils.is_file(path)
      return utils.lfs.attributes(path, "mode") == "file"
   end

   -- Returns list of all files in directory matching pattern. 
   function utils.extract_files(dir_path, pattern)
      local res = {}

      local function scan(dir_path)
         for path in utils.lfs.dir(dir_path) do
            if path ~= "." and path ~= ".." then
               local full_path = dir_path .. dir_sep .. path

               if utils.is_dir(full_path) then
                  scan(full_path)
               elseif path:match(pattern) and utils.is_file(full_path) then
                  table.insert(res, full_path)
               end
            end
         end
      end

      scan(dir_path)
      table.sort(res)
      return res
   end

   -- Returns modification time for a file. 
   function utils.mtime(path)
      return utils.lfs.attributes(path, "modification")
   end
else
   -- No luafilesystem. Effectively disable recursive directory checking.
   -- Using something like os.execute("ls") may be possible but is a hack.

   function utils.is_dir(_)
      return false
   end

   function utils.is_file(path)
      local fh = io.open(path)

      if fh then
         fh:close()
         return true
      else
         return false
      end
   end

   function utils.extract_files(_, _)
      return {}
{'fraction_non_alphanumeric': 0.09204052098408104, 'fraction_numerical': 0.0011577424023154848, 'mean_word_length': 2.0802139037433154, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6502586', 'n_tokens_mistral': 1057, 'n_tokens_neox': 1049, 'n_words': 304}