<commit_before>package hkpg

import (
	"bytes"
	"log"
	"net/http"
	"os"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/aws/awsutil"
	"github.com/aws/aws-sdk-go/aws/credentials"
	"github.com/aws/aws-sdk-go/aws/session"
	"github.com/aws/aws-sdk-go/service/s3"
)

// Upload accepts a *os.File, uploads it to the specified S3 buckets and
// returns the ETag of the uploaded file. Upload can be called multiple times
// with the same file name.
func Upload(file *os.File) (string, error) {
	var bucketName = os.Getenv("S3_BUCKET_NAME")
	if bucketName == "" {
		log.Fatalf("S3_BUCKET_NAME must be set")
	}

	creds := credentials.NewEnvCredentials()
	_, err := creds.Get()
	if err != nil {
		log.Fatalf("bad credentials: %s", err)
	}

	var awsRegion = os.Getenv("AWS_REGION")
	if awsRegion == "" {
		awsRegion = "us-west-1"
	}

	cfg := aws.NewConfig().WithRegion(awsRegion).WithCredentials(creds)
	svc := s3.New(session.New(), cfg)

	fileInfo, err := file.Stat()
	if err != nil {
		log.Fatalf("%v", err)
	}

	buffer := new(bytes.Buffer)
	file.Seek(0, 0)
	_, err = buffer.ReadFrom(file)
	if err != nil {
		log.Fatalf("file read failed: %v", err)
	}
	fileBytes := bytes.NewReader(buffer.Bytes())
	fileType := http.DetectContentType(buffer.Bytes())

	params := &s3.PutObjectInput{
		Bucket:        aws.String(bucketName),
		Key:           aws.String(file.Name()),
		Body:          fileBytes,
		ContentLength: aws.Int64(fileInfo.Size()),
		ContentType:   aws.String(fileType),
	}

	resp, err := svc.PutObject(params)
	if err != nil {
		log.Fatalf("bad response: %s", err)
	}

	var etag = awsutil.StringValue(resp)<commit_msg>check error from file seek<commit_after>package hkpg

import (
	"bytes"
	"log"
	"net/http"
	"os"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/aws/awsutil"
	"github.com/aws/aws-sdk-go/aws/credentials"
	"github.com/aws/aws-sdk-go/aws/session"
	"github.com/aws/aws-sdk-go/service/s3"
)

// Upload accepts a *os.File, uploads it to the specified S3 buckets and
// returns the ETag of the uploaded file. Upload can be called multiple times
// with the same file name.
func Upload(file *os.File) (string, error) {
	var bucketName = os.Getenv("S3_BUCKET_NAME")
	if bucketName == "" {
		log.Fatalf("S3_BUCKET_NAME must be set")
	}

	creds := credentials.NewEnvCredentials()
	_, err := creds.Get()
	if err != nil {
		log.Fatalf("bad credentials: %s", err)
	}

	var awsRegion = os.Getenv("AWS_REGION")
	if awsRegion == "" {
		awsRegion = "us-west-1"
	}

	cfg := aws.NewConfig().WithRegion(awsRegion).WithCredentials(creds)
	svc := s3.New(session.New(), cfg)

	fileInfo, err := file.Stat()
	if err != nil {
		log.Fatalf("%v", err)
	}

	buffer := new(bytes.Buffer)
	_, err = file.Seek(0, 0)
	if err != nil {
		log.Fatalf("failed to seek file: %v", err)
	}
	_, err = buffer.ReadFrom(file)
	if err != nil {
		log.Fatalf("file read failed: %v", err)
	}
	fileBytes := bytes.NewReader(buffer.Bytes())
	fileType := http.DetectContentType(buffer.Bytes())

	params := &s3.PutObjectInput{
		Bucket:        aws.String(bucketName),
		Key:           aws.String(file.Name()),
		Body:          fileBytes,
		ContentLength: aws.Int64(fileInfo.Size()),
		ContentType:   aws.String(fileType),
	}

	resp, err := svc.PutObject(params)
	if err != nil {
		log.Fatalf("bad response: %s", err)
	}

	var etag = awsutil.StringValue(resp)
{'fraction_non_alphanumeric': 0.16502242152466368, 'fraction_numerical': 0.006576980568011958, 'mean_word_length': 3.627939142461964, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4835872', 'n_tokens_mistral': 1315, 'n_tokens_neox': 1293, 'n_words': 273}