<commit_before>require "aws-sdk"
require "open-uri"
require "refile"
require "refile/s3/version"

module Refile
  # A refile backend which stores files in Amazon S3
  #
  # @example
  #   backend = Refile::Backend::S3.new(
  #     region: "sa-east-1",
  #     bucket: "my-bucket",
  #     prefix: "files"
  #   )
  #   file = backend.upload(StringIO.new("hello"))
  #   backend.read(file.id) # => "hello"
  class S3
    extend Refile::BackendMacros

    attr_reader :max_size

    # Sets up an S3 backend with the given credentials.
    #
    # @param [String] region            The AWS region to connect to
    # @param [String] bucket            The name of the bucket where files will be stored
    # @param [String] prefix            A prefix to add to all files. Prefixes on S3 are kind of like folders.
    # @param [Integer, nil] max_size    The maximum size of an uploaded file
    # @param [#hash] hasher             A hasher which is used to generate ids from files
    # @param [Hash] s3_options          Additional options to initialize S3 with
    # @see http://docs.aws.amazon.com/AWSRubySDK/latest/AWS/Core/Configuration.html
    # @see http://docs.aws.amazon.com/AWSRubySDK/latest/AWS/S3.html
    def initialize(region:, bucket:, max_size: nil, prefix: nil, hasher: Refile::RandomHasher.new, **s3_options)
      @s3_options = { region: region }.merge s3_options
      @s3 = Aws::S3::Resource.new @s3_options
      @bucket_name = bucket
      @bucket = @s3.bucket @bucket_name
      @hasher = hasher
      @prefix = prefix
      @max_size = max_size
    end

    # Upload a file into this backend
    #
    # @param [IO] uploadable      An uploadable IO-like object.
    # @return [Refile::File]      The uploaded file
    verify_uploadable def upload(uploadable)
      id = @hasher.hash(uploadable)

      if uploadable.is_a?(Refile::File) and uploadable.backend.is_a?(S3)<commit_msg>add errors and correct access_key_id<commit_after>require "aws-sdk"
require "open-uri"
require "refile"
require "refile/s3/version"

module Refile

  # @api private
  class S3BackendError < StandardError; end

  # @api private
  class S3CredentialsError < S3BackendError
    def message
      "Credentials not found"
    end
  end

  # A refile backend which stores files in Amazon S3
  #
  # @example
  #   backend = Refile::Backend::S3.new(
  #     region: "sa-east-1",
  #     bucket: "my-bucket",
  #     prefix: "files"
  #   )
  #   file = backend.upload(StringIO.new("hello"))
  #   backend.read(file.id) # => "hello"
  class S3
    extend Refile::BackendMacros

    attr_reader :access_key_id, :max_size

    # Sets up an S3 backend
    #
    # @param [String] region            The AWS region to connect to
    # @param [String] bucket            The name of the bucket where files will be stored
    # @param [String] prefix            A prefix to add to all files. Prefixes on S3 are kind of like folders.
    # @param [Integer, nil] max_size    The maximum size of an uploaded file
    # @param [#hash] hasher             A hasher which is used to generate ids from files
    # @param [Hash] s3_options          Additional options to initialize S3 with
    # @see http://docs.aws.amazon.com/AWSRubySDK/latest/AWS/Core/Configuration.html
    # @see http://docs.aws.amazon.com/AWSRubySDK/latest/AWS/S3.html
    def initialize(region:, bucket:, max_size: nil, prefix: nil, hasher: Refile::RandomHasher.new, **s3_options)
      @s3_options = { region: region }.merge s3_options
      @s3 = Aws::S3::Resource.new @s3_options
      credentials = @s3.client.config.credentials
      raise S3CredentialsError unless credentials
      @access_key_id = credentials.access_key_id
      @bucket_name = bucket
      @bucket = @s3.bucket @bucket_name
      @hasher = hasher
      @prefix = prefix
      @max_size = max_size
    end

    # Upload a file into this backend
    #
    # @param [IO] uploadable      An uploadable IO-like object.
    # @return [Refile::File]      The uploaded file
    verify_uploadable def upload(uploadable)
      id = @hasher.hash(uploadable)

      if uploadable.is_a?(Refile::File) and uploadable.backend.is_a?(S3) and uploadable.backend.access_key_id == access_key_id
{'fraction_non_alphanumeric': 0.10192903072160038, 'fraction_numerical': 0.009764229578471064, 'mean_word_length': 2.8043478260869565, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7343684', 'n_tokens_mistral': 1367, 'n_tokens_neox': 1352, 'n_words': 391}