<commit_before>require 'json'

module DPL
  class Provider
    class S3 < Provider
      requires 'aws-sdk'
      requires 'mime-types'

      def api
        @api ||= AWS::S3.new(endpoint: options[:endpoint] || 's3.amazonaws.com')
      end

      def needs_key?
        false
      end

      def check_app

      end

      def setup_auth
        AWS.config(:access_key_id => option(:access_key_id), :secret_access_key => option(:secret_access_key), :region => options[:region]||'us-east-1')
      end

      def check_auth
        setup_auth
        log "Logging in with Access Key: #{option(:access_key_id)[-4..-1].rjust(20, '*')}"
      end

      def upload_path(filename)
        [options[:upload_dir], filename].compact.join("/")
      end

      def push_app
        glob_args = ["**/*"]
        glob_args << File::FNM_DOTMATCH if options[:dot_match]
        Dir.chdir(options.fetch(:local_dir, Dir.pwd)) do
          Dir.glob(*glob_args) do |filename|
            content_type = MIME::Types.type_for(filename).first.to_s
            opts         = { :content_type => content_type }.merge(encoding_option_for(filename))
            opts[:cache_control] = get_option_value_by_filename(options[:cache_control], filename) if options[:cache_control]
            opts[:acl]           = options[:acl] if options[:acl]
            opts[:expires]       = get_option_value_by_filename(options[:expires], filename) if options[:expires]
            unless File.directory?(filename)
              api.buckets[option(:bucket)].objects.create(upload_path(filename), File.read(filename), opts)
            end
          end
        end

        if suffix = options[:index_document_suffix]
          api.buckets[option(:bucket)].configure_website do |cfg|
            cfg.index_document_suffix = suffix
          end
        end
      end

      def deploy
        super
      rescue AWS::S3::Errors::InvalidAccessKeyId
        raise Error, "Invalid S3 Access Key Id, Stopping Deploy"
      rescue AWS::S3::Errors::SignatureDoesNotMatch
        raise Error, "Aws Secret Key does not match Access Key Id, Stopping Deploy"
      rescue AWS::S3::Errors::AccessDenied
        raise Error, "Oops, It looks like you tried to write to a bucket that isn't yours or doesn't exist yet. Please create the bucket before trying to write to it."
      end

      private
      def encoding_option_for(path)
        if detect_encoding? && encoding_for(path)
          {:content_encoding => encoding_for(path)}
        else
          {}
        end
      end

      def get_option_value_by_filename(option_values, filename)
        return option_values if !option_values.kind_of?(Array)
        preferred_value = nil
        hashes = option_values.select {|value| value.kind_of?(Hash) }
        hashes.each do |hash|
          hash.each do |value, patterns|
            unless patterns.kind_of?(Array)
              patterns = [patterns]
            end
            patterns.each do |pattern|
              if File.fnmatch?(pattern, filename)
                preferred_value = value
                break
              end
            end
          end
        end
        preferred_value = option_values.select {|value| value.kind_of?(String) }.last if preferred_value.nil?
        return preferred_value
      end
    end
  end
end<commit_msg>Always use last match<commit_after>require 'json'

module DPL
  class Provider
    class S3 < Provider
      requires 'aws-sdk'
      requires 'mime-types'

      def api
        @api ||= AWS::S3.new(endpoint: options[:endpoint] || 's3.amazonaws.com')
      end

      def needs_key?
        false
      end

      def check_app

      end

      def setup_auth
        AWS.config(:access_key_id => option(:access_key_id), :secret_access_key => option(:secret_access_key), :region => options[:region]||'us-east-1')
      end

      def check_auth
        setup_auth
        log "Logging in with Access Key: #{option(:access_key_id)[-4..-1].rjust(20, '*')}"
      end

      def upload_path(filename)
        [options[:upload_dir], filename].compact.join("/")
      end

      def push_app
        glob_args = ["**/*"]
        glob_args << File::FNM_DOTMATCH if options[:dot_match]
        Dir.chdir(options.fetch(:local_dir, Dir.pwd)) do
          Dir.glob(*glob_args) do |filename|
            content_type = MIME::Types.type_for(filename).first.to_s
            opts         = { :content_type => content_type }.merge(encoding_option_for(filename))
            opts[:cache_control] = get_option_value_by_filename(options[:cache_control], filename) if options[:cache_control]
            opts[:acl]           = options[:acl] if options[:acl]
            opts[:expires]       = get_option_value_by_filename(options[:expires], filename) if options[:expires]
            unless File.directory?(filename)
              api.buckets[option(:bucket)].objects.create(upload_path(filename), File.read(filename), opts)
            end
          end
        end

        if suffix = options[:index_document_suffix]
          api.buckets[option(:bucket)].configure_website do |cfg|
            cfg.index_document_suffix = suffix
          end
        end
      end

      def deploy
        super
      rescue AWS::S3::Errors::InvalidAccessKeyId
        raise Error, "Invalid S3 Access Key Id, Stopping Deploy"
      rescue AWS::S3::Errors::SignatureDoesNotMatch
        raise Error, "Aws Secret Key does not match Access Key Id, Stopping Deploy"
      rescue AWS::S3::Errors::AccessDenied
        raise Error, "Oops, It looks like you tried to write to a bucket that isn't yours or doesn't exist yet. Please create the bucket before trying to write to it."
      end

      private
      def encoding_option_for(path)
        if detect_encoding? && encoding_for(path)
          {:content_encoding => encoding_for(path)}
        else
          {}
        end
      end

      def get_option_value_by_filename(option_values, filename)
        return option_values if !option_values.kind_of?(Array)
        preferred_value = nil
        hashes = option_values.select {|value| value.kind_of?(Hash) }
        hashes.each do |hash|
          hash.each do |value, patterns|
            unless patterns.kind_of?(Array)
              patterns = [patterns]
            end
            patterns.each do |pattern|
              if File.fnmatch?(pattern, filename)
                preferred_value = value
              end
            end
          end
        end
        preferred_value = option_values.select {|value| value.kind_of?(String) }.last if preferred_value.nil?
        return preferred_value
      end
    end
  end
end
{'fraction_non_alphanumeric': 0.09800965018094089, 'fraction_numerical': 0.0036188178528347406, 'mean_word_length': 2.2498775110240077, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6501986', 'n_tokens_mistral': 1909, 'n_tokens_neox': 1884, 'n_words': 457}