<commit_before>require 'json'
require 'aws-sdk'
require 'mime-types'

Aws.eager_autoload!

module DPL
  class Provider
    class S3 < Provider
      DEFAULT_MAX_THREADS = 5
      MAX_THREADS = 15

      def api
        @api ||= ::Aws::S3::Resource.new(s3_options)
      end

      def max_threads
        return @max_threads if @max_threads
        if (@max_threads = threads_wanted = options.fetch(:max_threads, DEFAULT_MAX_THREADS)) > MAX_THREADS
          log "Desired thread count #{threads_wanted} is too large. Using #{MAX_THREADS}."
          @max_threads = MAX_THREADS
        end
        @max_threads
      end

      def check_app
      end
      
      def needs_key?
        false
      end

      def access_key_id
        options[:access_key_id] || context.env['AWS_ACCESS_KEY_ID'] || raise(Error, "missing access_key_id")
      end

      def secret_access_key
        options[:secret_access_key] || context.env['AWS_SECRET_ACCESS_KEY'] || raise(Error, "missing secret_access_key")
      end

      def s3_options
        defaults = {
          region:      options[:region] || 'us-east-1',
          credentials: ::Aws::Credentials.new(access_key_id, secret_access_key)
        }

        defaults[:endpoint] = options[:endpoint] if options[:endpoint]

        defaults<commit_msg>Prepend https to s3-endpoint if https is missing<commit_after>require 'json'
require 'aws-sdk'
require 'mime-types'

Aws.eager_autoload!

module DPL
  class Provider
    class S3 < Provider
      DEFAULT_MAX_THREADS = 5
      MAX_THREADS = 15

      def api
        @api ||= ::Aws::S3::Resource.new(s3_options)
      end

      def max_threads
        return @max_threads if @max_threads
        if (@max_threads = threads_wanted = options.fetch(:max_threads, DEFAULT_MAX_THREADS)) > MAX_THREADS
          log "Desired thread count #{threads_wanted} is too large. Using #{MAX_THREADS}."
          @max_threads = MAX_THREADS
        end
        @max_threads
      end

      def check_app
      end
      
      def needs_key?
        false
      end

      def access_key_id
        options[:access_key_id] || context.env['AWS_ACCESS_KEY_ID'] || raise(Error, "missing access_key_id")
      end

      def secret_access_key
        options[:secret_access_key] || context.env['AWS_SECRET_ACCESS_KEY'] || raise(Error, "missing secret_access_key")
      end

      def s3_options
        defaults = {
          region:      options[:region] || 'us-east-1',
          credentials: ::Aws::Credentials.new(access_key_id, secret_access_key)
        }

        if options[:endpoint]
          if options[:endpoint].start_with?('https://', 'http://')
            defaults[:endpoint] = options[:endpoint]
          else
            defaults[:endpoint] = "https://#{options[:endpoint]}"
          end
        end

        defaults
{'fraction_non_alphanumeric': 0.0994671403197158, 'fraction_numerical': 0.006039076376554174, 'mean_word_length': 2.4133333333333336, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7520362', 'n_tokens_mistral': 882, 'n_tokens_neox': 854, 'n_words': 188}