<commit_before>require "album_processor"

class AlbumSetProcessor
  attr_reader :set_path, :ignore_file

  def initialize(set_path, ignore_file: '.album_set_ignore')
    @set_path = File.expand_path set_path
    @ignore_file = ignore_file
  end

  def process
    each_entry do |entry, full_path|
      ObjectSpace.garbage_collect # seems bad, hopefully there is a better way
      processor = AlbumProcessor.new(full_path)
      processor.process_images
    end
  end

  def each_entry
    Dir.foreach set_path do |entry|
      full_path = File.join(set_path, entry)
      next unless valid_album?(entry, full_path)
      puts "processing #{full_path}"
      yield entry, full_path
    end
  end

  def syncronize_photos
    each_entry do |entry, full_path|
      AlbumProcessor.new(full_path).insert_photos
    end
  end

  def valid_album?(name, full_path)
    return false unless File.directory?(full_path)
    return false if name =~ /\A\./
    return is_ignored?(name)
  end

  def is_ignored?(name)
    ignores.each do |ignore|
      return false if name == ignore.strip
    end
    true
  end

  def ignores
    @ignores ||= load_ignores
  end

  def load_ignores
    ignore_path = File.join(set_path, ignore_file)
    if File.exists? ignore_path
      IO.readlines(ignore_path)
    else
      []
    end
  end
end<commit_msg>Clean up ignore logic<commit_after>require "album_processor"

class AlbumSetProcessor
  attr_reader :set_path, :ignore_file

  def initialize(set_path, ignore_file: '.album_set_ignore')
    @set_path = File.expand_path set_path
    @ignore_file = ignore_file
  end

  def process
    each_entry do |entry, full_path|
      ObjectSpace.garbage_collect # seems bad, hopefully there is a better way
      processor = AlbumProcessor.new(full_path)
      processor.process_images
    end
  end

  def each_entry
    Dir.foreach set_path do |entry|
      full_path = File.join(set_path, entry)
      next unless valid_album?(entry, full_path)
      puts "processing #{full_path}"
      yield entry, full_path
    end
  end

  def syncronize_photos
    each_entry do |entry, full_path|
      AlbumProcessor.new(full_path).insert_photos
    end
  end

  def valid_album?(name, full_path)
    return false unless File.directory?(full_path)
    return false if name =~ /\A\./
    return false if ignored?(name)
    return true
  end

  def ignored?(name)
    return ignores.include?(name)
  end

  def ignores
    @ignores ||= load_ignores.map(&:strip)
  end

  def load_ignores
    ignore_path = File.join(set_path, ignore_file)
    if File.exists? ignore_path
      IO.readlines(ignore_path)
    else
      []
    end
  end
end
{'fraction_non_alphanumeric': 0.07576328684508105, 'fraction_numerical': 0.0, 'mean_word_length': 3.1275272161741836, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4120465', 'n_tokens_mistral': 868, 'n_tokens_neox': 855, 'n_words': 224}