<commit_before>require "erb"
require "pp"
require "fileutils"
require "wraith/wraith"

class Wraith::GalleryGenerator
  attr_reader :wraith

  MATCH_FILENAME = /(\S+)_(\S+)\.\S+/

  TEMPLATE_LOCATION = File.expand_path("gallery_template/gallery_template.erb", File.dirname(__FILE__))
  TEMPLATE_BY_DOMAIN_LOCATION = File.expand_path("gallery_template/gallery_template.erb", File.dirname(__FILE__))

  def initialize(config, multi)
    @wraith = Wraith::Wraith.new(config)
    @location = wraith.directory
    @mutli = multi
    @folder_manager = Wraith::FolderManager.new(config)
  end

  def parse_directories(dirname)
    @dirs = {}
    categories = Dir.foreach(dirname).select do |category|
      if [".", "..", "thumbnails"].include? category
        false
      elsif File.directory? "#{dirname}/#{category}"
        true
      else
        false
      end
    end
    match(categories, dirname)
  end

  def match(categories, dirname)
    categories.each do |category|
      @dirs[category] = {}

      Dir.foreach("#{dirname}/#{category}") do |filename|
        match = MATCH_FILENAME.match(filename)
        unless match.nil?
          matcher(match, filename, dirname, category)
        end
      end
    end
    @folder_manager.tidy_shots_folder(@dirs)
    @failed_shots = @folder_manager.threshold_rate(@dirs)
    sorting_dirs(@dirs)
  end

  def matcher(match, filename, dirname, category)
    @size = match[1].split('_')[0]
    @group = get_group_from_match match
    @filepath = category + "/" + filename
    @thumbnail = "thumbnails/#{category}/#{filename}"

    if @dirs[category][@size].nil?
      @dirs[category][@size] = { :variants => [] }
    end<commit_msg>fixed build<commit_after>require "erb"
require "pp"
require "fileutils"
require "wraith/wraith"

class Wraith::GalleryGenerator
  attr_reader :wraith

  MATCH_FILENAME = /(\S+)_(\S+)\.\S+/

  TEMPLATE_LOCATION = File.expand_path("gallery_template/gallery_template.erb", File.dirname(__FILE__))
  TEMPLATE_BY_DOMAIN_LOCATION = File.expand_path("gallery_template/gallery_template.erb", File.dirname(__FILE__))

  def initialize(config, multi)
    @wraith = Wraith::Wraith.new(config)
    @location = wraith.directory
    @mutli = multi
    @folder_manager = Wraith::FolderManager.new(config)
  end

  def parse_directories(dirname)
    @dirs = {}
    categories = Dir.foreach(dirname).select do |category|
      if [".", "..", "thumbnails"].include? category
        false
      elsif File.directory? "#{dirname}/#{category}"
        true
      else
        false
      end
    end
    match(categories, dirname)
  end

  def match(categories, dirname)
    categories.each do |category|
      @dirs[category] = {}

      Dir.foreach("#{dirname}/#{category}") do |filename|
        match = MATCH_FILENAME.match(filename)
        unless match.nil?
          matcher(match, filename, dirname, category)
        end
      end
    end
    @folder_manager.tidy_shots_folder(@dirs)
    @failed_shots = @folder_manager.threshold_rate(@dirs)
    sorting_dirs(@dirs)
  end

  def matcher(match, filename, dirname, category)
    @size = match[1].to_i
    @group = get_group_from_match match
    @filepath = category + "/" + filename
    @thumbnail = "thumbnails/#{category}/#{filename}"

    if @dirs[category][@size].nil?
      @dirs[category][@size] = { :variants => [] }
    end
{'fraction_non_alphanumeric': 0.13504631012847326, 'fraction_numerical': 0.00089632506722438, 'mean_word_length': 3.405263157894737, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6500237', 'n_tokens_mistral': 1138, 'n_tokens_neox': 1099, 'n_words': 200}