<commit_before>require 'fileutils'

Signal.trap('INT') do
  $stdout.puts "throwing in the towel..."
  exit
end

module Broom

  VERSION = '0.3.0'
  
  class Directory

    attr_reader :path, :pattern, :failure_dir, :success_dir

    def initialize(path, options={})
      @path        = File.expand_path(path)
      @pattern     = options[:pattern] || "*"
      @failure_dir = options[:failure_dir] || File.join(@path, "_failure")
      @success_dir = options[:success_dir] || File.join(@path, "_success")
      [@path, @failure_dir, @success_dir].each do |dir|
        FileUtils.mkdir(dir) unless File.exist?(dir)
      end
    end
    
    def entries
      files = {}
      Dir.glob(File.join(@path, @pattern)).each { |file|
        next if file == @success_dir || file == @failure_dir
        files[file] = File.mtime(file)
      }
      files
    end
        
  end
  
  extend self
  
  def sweep(path, options={}, &block)
    dir = Directory.new(path, options)
    loop do
      cache = dir.entries
      sleep options[:sleep] || 1
      
      cache.each do |file, modified_at|
        next if modified_at != File.mtime(file)
        begin
          yield(file)
        rescue Object => boom
          FileUtils.mv(file, dir.failure_dir)
        else
          FileUtils.mv(file, dir.success_dir)
        end
      end
    end
  end
  
end<commit_msg>Added error reporting<commit_after>require 'fileutils'

Signal.trap('INT') do
  $stdout.puts "throwing in the towel..."
  exit
end

module Broom

  VERSION = '0.3.1'
  
  class Directory

    attr_reader :path, :pattern, :failure_dir, :success_dir

    def initialize(path, options={})
      @path        = File.expand_path(path)
      @pattern     = options[:pattern] || "*"
      @failure_dir = options[:failure_dir] || File.join(@path, "_failure")
      @success_dir = options[:success_dir] || File.join(@path, "_success")
      [@path, @failure_dir, @success_dir].each do |dir|
        FileUtils.mkdir(dir) unless File.exist?(dir)
      end
    end
    
    def entries
      files = {}
      Dir.glob(File.join(@path, @pattern)).each { |file|
        next if file == @success_dir || file == @failure_dir
        files[file] = File.mtime(file)
      }
      files
    end
        
  end
  
  extend self
  
  def sweep(path, options={}, &block)
    dir = Directory.new(path, options)
    loop do
      cache = dir.entries
      sleep options[:sleep] || 1
      
      cache.each do |file, modified_at|
        next if modified_at != File.mtime(file)
        begin
          yield(file)
        rescue Object => boom
          puts "#{Time.now} : #{boom.class}: #{boom.message}"
          puts boom.backtrace.join("\n")
          FileUtils.mv(file, dir.failure_dir)
        else
          FileUtils.mv(file, dir.success_dir)
        end
      end
    end
  end
  
end
{'fraction_non_alphanumeric': 0.12760692824319547, 'fraction_numerical': 0.002827854365500177, 'mean_word_length': 2.2013574660633486, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2166102', 'n_tokens_mistral': 919, 'n_tokens_neox': 906, 'n_words': 213}