<commit_before>package s3

import (
	"net/http"

	"bytes"
	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/aws/credentials"
	"github.com/aws/aws-sdk-go/aws/session"
	"github.com/aws/aws-sdk-go/service/s3"
	"github.com/aws/aws-sdk-go/service/s3/s3manager"
	"github.com/eyecuelab/kit/log"
	"github.com/spf13/cobra"
	"github.com/spf13/viper"
	"io"
	"mime/multipart"
)

var uploder *s3manager.Uploader

func UploadFromForm(fileHeader *multipart.FileHeader, key string) (*s3manager.UploadOutput, error) {
	file, err := fileHeader.Open()
	if err != nil {
		return nil, err
	}

	var buffer bytes.Buffer
	io.Copy(&buffer, file)

	return Upload(buffer.Bytes(), key)
}

func Upload(b []byte, key string) (*s3manager.UploadOutput, error) {

	fileBytes := bytes.NewReader(b)
	fileType := http.DetectContentType(b)

	params := &s3manager.UploadInput{
		Bucket:      aws.String("lub-assets-test"),
		Key:         aws.String(key),
		Body:        fileBytes,
		ContentType: aws.String(fileType),
	}

	return uploder.Upload(params)
}

func newS3Client() *s3.S3 {
	awsAccessKey := viper.GetString("aws_access_key")
	awsSecret := viper.GetString("aws_secret")
	creds := credentials.NewStaticCredentials(awsAccessKey, awsSecret, "")
	_, err := creds.Get()
	log.Check(err)

	session, err := session.NewSession()
	log.Check(err)

	cfg := aws.NewConfig().WithRegion("us-west-1").WithCredentials(creds)
	return s3.New(session, cfg)
}<commit_msg>use s3 bucket from env<commit_after>package s3

import (
	"net/http"

	"bytes"
	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/aws/credentials"
	"github.com/aws/aws-sdk-go/aws/session"
	"github.com/aws/aws-sdk-go/service/s3"
	"github.com/aws/aws-sdk-go/service/s3/s3manager"
	"github.com/eyecuelab/kit/log"
	"github.com/spf13/cobra"
	"github.com/spf13/viper"
	"io"
	"mime/multipart"
)

var uploder *s3manager.Uploader

func UploadFromForm(fileHeader *multipart.FileHeader, key string) (*s3manager.UploadOutput, error) {
	file, err := fileHeader.Open()
	if err != nil {
		return nil, err
	}

	var buffer bytes.Buffer
	io.Copy(&buffer, file)

	return Upload(buffer.Bytes(), key)
}

func Upload(b []byte, key string) (*s3manager.UploadOutput, error) {

	fileBytes := bytes.NewReader(b)
	fileType := http.DetectContentType(b)

	params := &s3manager.UploadInput{
		Bucket:      aws.String(viper.GetString("aws_bucket")),
		Key:         aws.String(key),
		Body:        fileBytes,
		ContentType: aws.String(fileType),
	}

	return uploder.Upload(params)
}

func newS3Client() *s3.S3 {
	awsAccessKey := viper.GetString("aws_access_key")
	awsSecret := viper.GetString("aws_secret")
	creds := credentials.NewStaticCredentials(awsAccessKey, awsSecret, "")
	_, err := creds.Get()
	log.Check(err)

	session, err := session.NewSession()
	log.Check(err)

	cfg := aws.NewConfig().WithRegion("us-west-1").WithCredentials(creds)
	return s3.New(session, cfg)
}
{'fraction_non_alphanumeric': 0.16048526863084922, 'fraction_numerical': 0.012131715771230503, 'mean_word_length': 4.445283018867925, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2025113', 'n_tokens_mistral': 1149, 'n_tokens_neox': 1124, 'n_words': 178}