<commit_before>module S3ImageOptimizer::Bucket
  class ImageDownloader < Performer
    attr_accessor :downloaded_images

    def initialize(images: [], client:, bucket:, options: {})
      super(bucket, options)
      @s3_client = client
      @images = images
      @downloaded_images = []
      @optimize_opts = @options[:optimize]
      download_images()
    end

    def download_images
      puts "\nDownloading..."
      @images.select {|i| is_image?(i) && !is_already_optimized?(i) }.each do |i|
        if File.exist?(File.join(@options[:tmp_download_path], i))
          puts "Already downloaded #{i}"
          @downloaded_images << File.join(@options[:tmp_download_path], i)) { |f| f }
          next
        end
        s3_object = @s3_client.get_object({:key => i, :bucket => @bucket.name})
        @downloaded_images << create_file(s3_object.body, i)
      end
    end

    private

    def is_image?(image)
      ['jpg','jpeg','gif','png'].include?(image.split('.').last)
    end

    def is_already_optimized?(image)
      return false unless @optimize_opts[:rename][:append]
      image.split('.')[-2][-@optimize_opts[:rename][:append].length..-1] == @optimize_opts[:rename][:append]
    end

    def create_file(s3_object, key)<commit_msg>fix typo<commit_after>module S3ImageOptimizer::Bucket
  class ImageDownloader < Performer
    attr_accessor :downloaded_images

    def initialize(images: [], client:, bucket:, options: {})
      super(bucket, options)
      @s3_client = client
      @images = images
      @downloaded_images = []
      @optimize_opts = @options[:optimize]
      download_images()
    end

    def download_images
      puts "\nDownloading..."
      @images.select {|i| is_image?(i) && !is_already_optimized?(i) }.each do |i|
        if File.exist?(File.join(@options[:tmp_download_path], i))
          puts "Already downloaded #{i}"
          @downloaded_images << File.open(File.join(@options[:tmp_download_path], i)) { |f| f }
          next
        end
        s3_object = @s3_client.get_object({:key => i, :bucket => @bucket.name})
        @downloaded_images << create_file(s3_object.body, i)
      end
    end

    private

    def is_image?(image)
      ['jpg','jpeg','gif','png'].include?(image.split('.').last)
    end

    def is_already_optimized?(image)
      return false unless @optimize_opts[:rename][:append]
      image.split('.')[-2][-@optimize_opts[:rename][:append].length..-1] == @optimize_opts[:rename][:append]
    end

    def create_file(s3_object, key)
{'fraction_non_alphanumeric': 0.15244144501786422, 'fraction_numerical': 0.006351726875744343, 'mean_word_length': 3.165289256198347, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 22, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7384870', 'n_tokens_mistral': 825, 'n_tokens_neox': 827, 'n_words': 150}