<commit_before>package ec2

import (
	"fmt"
	"io"
	"launchpad.net/goamz/s3"
	"launchpad.net/juju/go/environs"
	"sync"
)

// storage implements environs.Storage on
// an ec2.bucket.
type storage struct {
	bucketMutex sync.Mutex
	madeBucket  bool
	bucket      *s3.Bucket
}

// makeBucket makes the environent's control bucket, the
// place where bootstrap information and deployed charms
// are stored. To avoid two round trips on every PUT operation,
// we do this only once for each environ.
func (s *storage) makeBucket() error {
	s.bucketMutex.Lock()
	defer s.bucketMutex.Unlock()
	// try to make the bucket - PutBucket will succeed if the
	// bucket already exists.
	err := s.bucket.PutBucket(s3.Private)
	if err == nil {
		s.madeBucket = true
	}
	return err
}

func (s *storage) Put(file string, r io.Reader, length int64) error {
	if err := s.makeBucket(); err != nil {
		return fmt.Errorf("cannot make S3 control bucket: %v", err)
	}
	err := s.bucket.PutReader(file, r, length, "binary/octet-stream", s3.Private)
	if err != nil {
		return fmt.Errorf("cannot write file %q to control bucket: %v", file, err)
	}
	return nil
}

func (s *storage) Get(file string) (r io.ReadCloser, err error) {
	for a := shortAttempt.start(); a.next(); {
		r, err = s.bucket.GetReader(file)
		if s3ErrorStatusCode(err) == 404 {
			continue
		}
		return
	}
	return r, maybeNotFound(err)
}

func (s *storage) URL(name string) (string, error) {
	return s.bucket.URL(name), nil
}

// s3ErrorStatusCode returns the HTTP status of the S3 request error,
// if it is an error from an S3 operation, or 0 if it was not.
func s3ErrorStatusCode(err error) int {
	if err, _ := err.(*s3.Error); err != nil {
		return err.StatusCode
	}
	return 0
}

func (s *storage) Remove(file string) error {
	err := s.bucket.Del(file)
	// If we can't delete the object because the bucket doesn't
	// exist, then we don't care.
	if s3ErrorStatusCode(err) == 404 {
		return nil
	}
	return err
}

func (s *storage) List(prefix string) ([]string, error) {
	// TODO cope with more than 1000 objects in the bucket.
	resp, err := s.bucket.List(prefix, "", "", 0)
	if err != nil {
		return nil, maybeNotFound(err)
	}
	var names []string
	for _, key := range resp.Contents {
		names = append(names, key.Key)
	}
	return names, nil
}

func (e *environ) Storage() environs.Storage {
	return &e.storage
}

func (e *environ) PublicStorage() environs.StorageReader {
	// TODO use public storage bucket
	return environs.EmptyStorage
}
<commit_msg>environs/ec2: use signed URL<commit_after>package ec2

import (
	"fmt"
	"io"
	"launchpad.net/goamz/s3"
	"launchpad.net/juju/go/environs"
	"sync"
	"time"
)

// storage implements environs.Storage on
// an ec2.bucket.
type storage struct {
	bucketMutex sync.Mutex
	madeBucket  bool
	bucket      *s3.Bucket
}

// makeBucket makes the environent's control bucket, the
// place where bootstrap information and deployed charms
// are stored. To avoid two round trips on every PUT operation,
// we do this only once for each environ.
func (s *storage) makeBucket() error {
	s.bucketMutex.Lock()
	defer s.bucketMutex.Unlock()
	// try to make the bucket - PutBucket will succeed if the
	// bucket already exists.
	err := s.bucket.PutBucket(s3.Private)
	if err == nil {
		s.madeBucket = true
	}
	return err
}

func (s *storage) Put(file string, r io.Reader, length int64) error {
	if err := s.makeBucket(); err != nil {
		return fmt.Errorf("cannot make S3 control bucket: %v", err)
	}
	err := s.bucket.PutReader(file, r, length, "binary/octet-stream", s3.Private)
	if err != nil {
		return fmt.Errorf("cannot write file %q to control bucket: %v", file, err)
	}
	return nil
}

func (s *storage) Get(file string) (r io.ReadCloser, err error) {
	for a := shortAttempt.start(); a.next(); {
		r, err = s.bucket.GetReader(file)
		if s3ErrorStatusCode(err) == 404 {
			continue
		}
		return
	}
	return r, maybeNotFound(err)
}

func (s *storage) URL(name string) (string, error) {
	// 10 years should be good enough.
	return s.bucket.SignedURL(name, time.Now().AddDate(1, 0, 0)), nil
}

// s3ErrorStatusCode returns the HTTP status of the S3 request error,
// if it is an error from an S3 operation, or 0 if it was not.
func s3ErrorStatusCode(err error) int {
	if err, _ := err.(*s3.Error); err != nil {
		return err.StatusCode
	}
	return 0
}

func (s *storage) Remove(file string) error {
	err := s.bucket.Del(file)
	// If we can't delete the object because the bucket doesn't
	// exist, then we don't care.
	if s3ErrorStatusCode(err) == 404 {
		return nil
	}
	return err
}

func (s *storage) List(prefix string) ([]string, error) {
	// TODO cope with more than 1000 objects in the bucket.
	resp, err := s.bucket.List(prefix, "", "", 0)
	if err != nil {
		return nil, maybeNotFound(err)
	}
	var names []string
	for _, key := range resp.Contents {
		names = append(names, key.Key)
	}
	return names, nil
}

func (e *environ) Storage() environs.Storage {
	return &e.storage
}

func (e *environ) PublicStorage() environs.StorageReader {
	// TODO use public storage bucket
	return environs.EmptyStorage
}

{'fraction_non_alphanumeric': 0.1273659305993691, 'fraction_numerical': 0.012618296529968454, 'mean_word_length': 3.590950226244344, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5967112', 'n_tokens_mistral': 1816, 'n_tokens_neox': 1757, 'n_words': 597}