<commit_before>package storage

import (
	"crypto/sha512"
	"encoding/json"
	"fmt"
	"path"
	"time"

	"github.com/mitchellh/goamz/aws"
	"github.com/mitchellh/goamz/s3"
)

type S3 struct {
	Bucket   *s3.Bucket
	hashFunc func(string) string
}

func NewS3(auth aws.Auth, region aws.Region, bucketName string) (*S3, error) {
	s := &S3{
		Bucket: s3.New(auth, region).Bucket(bucketName),
		hashFunc: func(s string) string {
			h := sha512.Sum384([]byte(s))
			return string(h[:])
		},
	}

	_, err := s.Bucket.GetBucketContents()
	if s3err, ok := err.(*s3.Error); ok && s3err.Code == "NoSuchBucket" {
		err = s.Bucket.PutBucket(s3.BucketOwnerFull)
	}

	return s, err
}

func (s *S3) saveKey(short, url string) (err error) {
	hashedShort := s.hashFunc(short)

	err = s.Bucket.Put(
		path.Join(hashedShort, "long"),
		[]byte(url),
		"text/plain",
		s3.BucketOwnerFull,
	)
	if err != nil {
		return err
	}
<commit_msg>Whoops, base64 encode that Sha384<commit_after>package storage

import (
	"crypto/sha512"
	"encoding/base64"
	"encoding/json"
	"fmt"
	"path"
	"time"

	"github.com/mitchellh/goamz/aws"
	"github.com/mitchellh/goamz/s3"
)

type S3 struct {
	Bucket   *s3.Bucket
	hashFunc func(string) string
}

func NewS3(auth aws.Auth, region aws.Region, bucketName string) (*S3, error) {
	s := &S3{
		Bucket: s3.New(auth, region).Bucket(bucketName),
		hashFunc: func(s string) string {
			h := sha512.Sum384([]byte(s))
			return base64.StdEncoding.EncodeToString(h[:])
		},
	}

	_, err := s.Bucket.GetBucketContents()
	if s3err, ok := err.(*s3.Error); ok && s3err.Code == "NoSuchBucket" {
		err = s.Bucket.PutBucket(s3.BucketOwnerFull)
	}

	return s, err
}

func (s *S3) saveKey(short, url string) (err error) {
	hashedShort := s.hashFunc(short)

	err = s.Bucket.Put(
		path.Join(hashedShort, "long"),
		[]byte(url),
		"text/plain",
		s3.BucketOwnerFull,
	)
	if err != nil {
		return err
	}

{'fraction_non_alphanumeric': 0.16702127659574467, 'fraction_numerical': 0.028191489361702127, 'mean_word_length': 3.924083769633508, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1593106', 'n_tokens_mistral': 823, 'n_tokens_neox': 775, 'n_words': 157}