<commit_before># todo: self.instances and self.prefetch not yet implemented. 
# Currently this provider compares the MD5 hash of the S3 and the local file system file, if that comparison returns 
# false then the S3 object is pulled and written to the :path on the filesystem.
#
# Example:
#   s3 { '/path/to/my/filesystem':
#       ensure              => present,
#       source              => '/bucket/path/to/object',
#       access_key_id       => 'mysecret',
#       secret_access_key   => 'anothersecret',
#       region              => 'us-west-1', # Defaults to us-east-1
#   }
#
#   Author: <NAME>, <EMAIL>

require 'rubygems' if Puppet.features.rubygems? 
require 'aws-sdk' if Puppet.features.awssdk?
require 'digest'
require 'tempfile'

Puppet::Type.type(:s3).provide(:s3) do
  confine :feature => :awssdk
  confine :feature => :rubygems

  desc "Securely get shit out of S3. Note this provider requires Version 2 of the aws-sdk. Ensure that v2 is installed."

  def create
    Puppet.info('Connecting to AWS S3')   
    # TODO set it up to work even if you don't provide access and secret keys (automatic credintial lookup on ec2 or ~/.aws/credentials ?
    s3 = Aws::S3::Client.new( 
        :access_key_id      => resource[:access_key_id], 
        :secret_access_key  => resource[:secret_access_key],
        :region             => resource[:region] || 'us-east-1',
    )

    # Get the name of the bucket and path to the object:
    source_ary  = resource[:source].chomp.split('/')
    source_ary.shift # Remove prefixed white space
    
    bucket      = source_ary.shift
    key         = File.join(source_ary)
    
    Puppet.info("Pulling bucket: #{bucket}, key: #{key}")
    # Handle new S3 object
    resp = s3.get_object(
        response_target:    resource[:path],
        bucket:             bucket,
        key:                key,
    )

    # Create a .etag file when we've downloaded an artifact
    IO.write(resource[:path], resp.etag) #TODO any kind of exception handling required?
    
  end

  def destroy

      # rm rf some file on the filesystem that points to resource[:path]
    
  end

  # TODO Add condition where the ETag isn't cached, but the head_object has
  #      an ETag that is an MD5 (aka, does not match /-\d$/
  def exists?


      # Create a new S3 client object
      s3 = Aws::S3::Client.new( 
          :access_key_id      => resource[:access_key_id], <commit_msg>Added missing .etag extension while writing out the cached metadata<commit_after># todo: self.instances and self.prefetch not yet implemented. 
# Currently this provider compares the MD5 hash of the S3 and the local file system file, if that comparison returns 
# false then the S3 object is pulled and written to the :path on the filesystem.
#
# Example:
#   s3 { '/path/to/my/filesystem':
#       ensure              => present,
#       source              => '/bucket/path/to/object',
#       access_key_id       => 'mysecret',
#       secret_access_key   => 'anothersecret',
#       region              => 'us-west-1', # Defaults to us-east-1
#   }
#
#   Author: <NAME>, <EMAIL>

require 'rubygems' if Puppet.features.rubygems? 
require 'aws-sdk' if Puppet.features.awssdk?
require 'digest'
require 'tempfile'

Puppet::Type.type(:s3).provide(:s3) do
  confine :feature => :awssdk
  confine :feature => :rubygems

  desc "Securely get shit out of S3. Note this provider requires Version 2 of the aws-sdk. Ensure that v2 is installed."

  def create
    Puppet.info('Connecting to AWS S3')   
    # TODO set it up to work even if you don't provide access and secret keys (automatic credintial lookup on ec2 or ~/.aws/credentials ?
    s3 = Aws::S3::Client.new( 
        :access_key_id      => resource[:access_key_id], 
        :secret_access_key  => resource[:secret_access_key],
        :region             => resource[:region] || 'us-east-1',
    )

    # Get the name of the bucket and path to the object:
    source_ary  = resource[:source].chomp.split('/')
    source_ary.shift # Remove prefixed white space
    
    bucket      = source_ary.shift
    key         = File.join(source_ary)
    
    Puppet.info("Pulling bucket: #{bucket}, key: #{key}")
    # Handle new S3 object
    resp = s3.get_object(
        response_target:    resource[:path],
        bucket:             bucket,
        key:                key,
    )

    # Create a .etag file when we've downloaded an artifact
    IO.write(resource[:path]+'.etag', resp.etag) #TODO any kind of exception handling required?
    
  end

  def destroy

      # rm rf some file on the filesystem that points to resource[:path]
    
  end

  # TODO Add condition where the ETag isn't cached, but the head_object has
  #      an ETag that is an MD5 (aka, does not match /-\d$/
  def exists?


      # Create a new S3 client object
      s3 = Aws::S3::Client.new( 
          :access_key_id      => resource[:access_key_id], 
{'fraction_non_alphanumeric': 0.10379282218597063, 'fraction_numerical': 0.00897226753670473, 'mean_word_length': 2.5035714285714286, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 29, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 2, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2824524', 'n_tokens_mistral': 1545, 'n_tokens_neox': 1494, 'n_words': 503}