<commit_before>package coffer

import (
	"bytes"
	"io"
	"net/url"

	s3 "github.com/rlmcpherson/s3gof3r"
)

func mustUpload(bucket string, filename string, payload []byte) {

	u, err := url.Parse(bucket)
	if err != nil {
		Fatalf("Failed to parse bucket URL: %v", err)
	}

	k, err := s3.EnvKeys() // get S3 keys from environment
	if err != nil {
		Fatalf("Unable to load AWS credentials from environment")
	}

	// Open bucket to put file into
	s3 := s3.New("s3.amazonaws.com", k)
	b := s3.Bucket(u.Host)

	// Open a PutWriter for upload
	w, err := b.PutWriter(filename, nil, nil)
	if err != nil {
		Fatalf("Put to bucket failed: %v", err)
	}

	if _, err = io.Copy(w, bytes.NewBuffer(payload)); err != nil { // Copy into S3
		Fatalf("Copy content to bucket failed: %v", err)
	}

	if err = w.Close(); err != nil {
		Fatalf("Closing file failed: %v", err)
	}
}<commit_msg>Now using the aws golang sdk.<commit_after>package coffer

import (
	"bytes"

	"github.com/awslabs/aws-sdk-go/aws"
	"github.com/awslabs/aws-sdk-go/aws/awsutil"
	"github.com/awslabs/aws-sdk-go/service/s3"
	"github.com/davecgh/go-spew/spew"
)

func mustUpload(bucket string, filename string, payload []byte) {

	Infof("putting file=%s into bucket=%s length=%d", filename, bucket, len(payload))

	config := aws.DefaultConfig
	creds, err := config.Credentials.Credentials()
	if err != nil {
		Fatalf("Unable to load creds: %v", err)
	}

	svc := s3.New(config)

	params := &s3.PutObjectInput{
		Bucket:        aws.String(bucket),   // Required
		Key:           aws.String(filename), // Required
		Body:          bytes.NewReader(payload),
		ContentLength: aws.Long(int64(len(payload))),
	}

	resp, err := svc.PutObject(params)

	if awserr := aws.Error(err); awserr != nil {
		// A service error occurred.
		Fatalf("AWS Error code: %v message: %s", awserr.Code, awserr.Message)
	} else if err != nil {
		// A non-service error occurred.
		Fatalf("Error make request to AWS: %v", err)
	}

	spew.Dump(resp)
	spew.Dump(config)
	spew.Dump(creds)

	// Pretty-print the response data.
	Infof("response %s", awsutil.StringValue(resp))

}
{'fraction_non_alphanumeric': 0.1568627450980392, 'fraction_numerical': 0.007173601147776184, 'mean_word_length': 3.5577342047930283, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3116777', 'n_tokens_mistral': 802, 'n_tokens_neox': 786, 'n_words': 203}