<commit_before># encoding: utf-8
# (c) 2011 <NAME> (<EMAIL>)

require "jpegtran"
require "optipng"
require "jpegoptim"

require "mini_magick"
require "unix/whereis"
require "options-hash" 

module ImageOptimizer
    
    ##
    # Optimizes given file or folder.
    #
    # If array given, go recursively deep to it. Yields paths to 
    # optimized files.
    #
    # @param [String, Array] item path or paths to files or directories
    # @param [Hash] options options for optimizing
    # @param [Proc] block block for giving back the results
    # @option options [Boolean] :strip indicates it should strip metadata from JPEG files
    # @option options [Integer] :level indicates level of PNG optimizations
    #
    
    def self.optimize(item, options = { }, &block)
        if item.kind_of? Array
            item.each do |i|
                self::optimize(i, options, &block)
            end
        else
            if File.file? item
                self::optimize_file(item, options, &block)
            elsif File.directory? item
                self::optimize_directory(item, options, &block)
            end
        end
    end
    
    ##
    # Optimizes given folder. 
    # Yields paths to optimized files.
    #
    # @param [String] item path to directory
    # @param [Hash] options options for optimizing
    # @param [Proc] block block for giving back the results
    # @option options [Boolean] :strip indicates it should strip metadata from JPEG files
    # @option options [Integer] :level indicates level of PNG optimizations
    #
    
    def self.optimize_directory(item, options = { }, &block)
        Dir.open(item) do |dir|
            dir.each do |i|
                if i[0].chr != ?.
                    self::optimize(item + "/" + i, options, &block)
                end
            end
        end
    end
    
    ##
    # Optimizes given file.
    # Yields paths to optimized files.
    #
    # @param [String] item path to file
    # @param [Hash] options options for optimizing
    # @param [Proc] block block for giving back the results
    # @option options [Boolean] :strip indicates it should strip metadata from JPEG files
    # @option options [Integer] :level indicates level of PNG optimizations
    #
    
    def self.optimize_file(path, options = { }, &block)
        ext = File.extname(path)
        
        # Loads options
        opts = OptionsHash::get(options) [
            :strip => true,
            :level => 7
        ]
        
        case ext
            when ".png"
                if Whereis.available? :convert
                    begin
                        image = MiniMagick::Image.open(path)
                        image.write(path)
                    rescue MiniMagick::Error
                        # skips
                    end
                end
                
                if Optipng.available?
                    Optipng.optimize(path, { :level => opts[:level] })
                end
                
                block.call(path)
            when ".jpg"
                if Jpegoptim.available?
                    Jpegoptim.optimize(path, opts[:strip].true? ? { :strip => :all } : { })
                end
                
                if Jpegtran.available?
                    Jpegtran.optimize(path, { :optimize => true, :progressive => true })
                end
                
                block.call(path)
        end
    end

end<commit_msg>some strange problem with freezing strings<commit_after># encoding: utf-8
# (c) 2011 <NAME> (<EMAIL>)

require "jpegtran"
require "optipng"
require "jpegoptim"

require "mini_magick"
require "unix/whereis"
require "options-hash" 

module ImageOptimizer
    
    ##
    # Optimizes given file or folder.
    #
    # If array given, go recursively deep to it. Yields paths to 
    # optimized files.
    #
    # @param [String, Array] item path or paths to files or directories
    # @param [Hash] options options for optimizing
    # @param [Proc] block block for giving back the results
    # @option options [Boolean] :strip indicates it should strip metadata from JPEG files
    # @option options [Integer] :level indicates level of PNG optimizations
    #
    
    def self.optimize(item, options = { }, &block)
        if item.kind_of? Array
            item.each do |i|
                self::optimize(i, options, &block)
            end
        else
            if File.file? item
                self::optimize_file(item, options, &block)
            elsif File.directory? item
                self::optimize_directory(item, options, &block)
            end
        end
    end
    
    ##
    # Optimizes given folder. 
    # Yields paths to optimized files.
    #
    # @param [String] item path to directory
    # @param [Hash] options options for optimizing
    # @param [Proc] block block for giving back the results
    # @option options [Boolean] :strip indicates it should strip metadata from JPEG files
    # @option options [Integer] :level indicates level of PNG optimizations
    #
    
    def self.optimize_directory(item, options = { }, &block)
        Dir.open(item) do |dir|
            dir.each do |i|
                if i[0].chr != ?.
                    self::optimize(item + "/" + i, options, &block)
                end
            end
        end
    end
    
    ##
    # Optimizes given file.
    # Yields paths to optimized files.
    #
    # @param [String] item path to file
    # @param [Hash] options options for optimizing
    # @param [Proc] block block for giving back the results
    # @option options [Boolean] :strip indicates it should strip metadata from JPEG files
    # @option options [Integer] :level indicates level of PNG optimizations
    #
    
    def self.optimize_file(path, options = { }, &block)
        ext = File.extname(path)
        
        # Loads options
        opts = OptionsHash::get(options) [
            :strip => true,
            :level => 7
        ]
        
        case ext
            when ".png"
                if Whereis.available? :convert
                    begin
                        image = MiniMagick::Image.open(path)
                        image.write(path)
                    rescue MiniMagick::Error
                        # skips
                    end
                end
                
                if Optipng.available?
                    Optipng.optimize(path, { :level => opts[:level] })
                end
                
                block.call(path.dup)
            when ".jpg"
                if Jpegoptim.available?
                    Jpegoptim.optimize(path, opts[:strip].true? ? { :strip => :all } : { })
                end
                
                if Jpegtran.available?
                    Jpegtran.optimize(path, { :optimize => true, :progressive => true })
                end
                
                block.call(path.dup)
        end
    end

end
{'fraction_non_alphanumeric': 0.08785827452110039, 'fraction_numerical': 0.0020164194152383695, 'mean_word_length': 1.5699481865284974, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 19, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '79148', 'n_tokens_mistral': 1917, 'n_tokens_neox': 1823, 'n_words': 592}