<commit_before>require 'image_processing/config'
require 'image_processing/ftp_adaptor'
require 'image_processing/file_manager'

module ImageProcessing
  class ImageFile
    attr_reader :name, :slice_number, :version_number
    
    def initialize(filename)
      raise "#{filename} doesn't seem to be an image file." unless filename =~ /\.bmp$/
      @name = filename
      m = @name.match(/(\d+)(\(\d+\))?.bmp$/)
      @slice_number = m[1].to_i
      @version_number = m[2].to_i if m[2]
    end                                    
    
  end
  
  class Picker
    def initialize(argv)
      @config = Config.new(argv)
      @ftp = FtpAdaptor.new(@config)
      @file_manager = FileManager.new(@config, @ftp)
      pick_files
    end
    
    def files
      @files.map {|f| f.name }
    end
    
  private  
    def pick_files      
      # select one every @config.downsample_ratio images
      available_files = @file_manager.remote_originals.sort.map {|f| ImageFile.new(f) }
      puts available_files.count
      puts available_files.first.name
      puts available_files.last.name
      slice_range = (available_files.first.slice_number)..(available_files.last.slice_number)
      @files = []
      
      slice_range.display
      
      slice_range.step(@config.downsample_ratio).each do |target_slice_number|
        # pick list of candidates
        candidates = available_files.take_while{ |file| file.slice_number < target_slice_number + @config.downsample_ratio }
        
        # raise an exception if there aren't any candidates
        raise "There aren't any candidates" if candidates.empty?
        
        # decide on an image to move and append its file name to the list
        @files << pick_image(candidates, target_slice_number)
        
        # next image must be above previous index and previous image
        cutoff = [@files.last.slice_number, target_slice_number].max
        available_files = available_files.drop_while {|file| file.slice_number <= cutoff }
      end
    end

    def pick_image(candidates, target_slice_number)
      # sort list of candidates in order of decreasing preference<commit_msg>Changed Regexp in ImageFile and file names on server to match.<commit_after>require 'image_processing/config'
require 'image_processing/ftp_adaptor'
require 'image_processing/file_manager'

module ImageProcessing
  class ImageFile
    attr_reader :name, :slice_number, :version_number
    
    REGEXP = /(\d+)([a-z])?.bmp$/
    
    def initialize(filename)
      @name = filename
      raise "Cannot extract slice number and version from #{filename}." unless m = @name.match(REGEXP)
      @slice_number = m[1].to_i
      @version_number = m[2].to_i if m[2]
    end
    
  end
  
  class Picker
    def initialize(argv)
      @config = Config.new(argv)
      @ftp = FtpAdaptor.new(@config)
      @file_manager = FileManager.new(@config, @ftp)
      pick_files
    end
    
    def files
      @files.map {|f| f.name }
    end
    
  private  
    def pick_files
      # select one every @config.downsample_ratio images
      available_files = @file_manager.remote_originals.sort.map {|f| ImageFile.new(f) }
      puts available_files.count
      puts available_files.first.name
      puts available_files.last.name
      slice_range = (available_files.first.slice_number)..(available_files.last.slice_number)
      @files = []
      
      slice_range.display
      
      slice_range.step(@config.downsample_ratio).each do |target_slice_number|
        # pick list of candidates
        candidates = available_files.take_while{ |file| file.slice_number < target_slice_number + @config.downsample_ratio }
        
        # raise an exception if there aren't any candidates
        raise "There aren't any candidates" if candidates.empty?
        
        # decide on an image to move and append its file name to the list
        @files << pick_image(candidates, target_slice_number)
        
        # next image must be above previous index and previous image
        cutoff = [@files.last.slice_number, target_slice_number].max
        available_files = available_files.drop_while {|file| file.slice_number <= cutoff }
      end
    end

    def pick_image(candidates, target_slice_number)
      # sort list of candidates in order of decreasing preference
{'fraction_non_alphanumeric': 0.08579400139502441, 'fraction_numerical': 0.0013950244129272262, 'mean_word_length': 2.724675324675325, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5823316', 'n_tokens_mistral': 1277, 'n_tokens_neox': 1252, 'n_words': 326}