<commit_before>package btrfs

import (
	"bytes"
	"fmt"
	"io"
	"log"
	"os/exec"
	"path"
	"time"

	"github.com/aws/aws-sdk-go/service/s3"

	"github.com/pachyderm/pachyderm/src/s3utils"
	"github.com/pachyderm/pachyderm/src/util"
)

// localReplica implements the Replica interface using a btrfs repo.
type localReplica struct {
	repo string
}

func newLocalReplica(repo string) *localReplica {
	return &localReplica{repo}
}

func (r *localReplica) From() (string, error) {
	from := ""
	err := Commits(r.repo, "", Desc, func(name string) error {
		isCommit, err := IsCommit(path.Join(r.repo, name))
		if err != nil {
			return err
		}
		if isCommit {
			from = name
			return ErrComplete
		}
		return nil
	})
	if err != nil && err != ErrComplete {
		return "", err
	}

	return from, nil
}

func (r *localReplica) Push(diff io.Reader) error {
	return recv(r.repo, diff)
}

func (r *localReplica) Pull(from string, cb Pusher) error {
	return Pull(r.repo, from, cb)
}

// s3Replica implements the Replica interface using an s3 replica.
type s3Replica struct {
	uri   string
	count int // number of sent commits
}

func newS3Replica(uri string) *s3Replica {
	return &s3Replica{uri, 0}
}

func (r *s3Replica) Push(diff io.Reader) error {
	key := fmt.Sprintf("%.10d", r.count)
	r.count++

	p, err := s3utils.GetPath(r.uri)
	if err != nil {
		return err
	}

	return s3utils.PutMulti(r.uri, path.Join(p, key), diff, "application/octet-stream", s3utils.BucketOwnerFull)
}

func (r *s3Replica) Pull(from string, target Pusher) error {
	client := s3.New(nil)
	err := s3utils.ForEachFile(r.uri, from, func(path string, modtime time.Time) (retErr error) {
		response, err := client.GetObject(&s3.GetObjectInput{
			Bucket: &r.uri,
			Key:    &path,
		})

		f := response.Body

		if f == nil {<commit_msg>Use s3utils.NewClient<commit_after>package btrfs

import (
	"bytes"
	"fmt"
	"io"
	"log"
	"os/exec"
	"path"
	"time"

	"github.com/aws/aws-sdk-go/service/s3"

	"github.com/pachyderm/pachyderm/src/s3utils"
	"github.com/pachyderm/pachyderm/src/util"
)

// localReplica implements the Replica interface using a btrfs repo.
type localReplica struct {
	repo string
}

func newLocalReplica(repo string) *localReplica {
	return &localReplica{repo}
}

func (r *localReplica) From() (string, error) {
	from := ""
	err := Commits(r.repo, "", Desc, func(name string) error {
		isCommit, err := IsCommit(path.Join(r.repo, name))
		if err != nil {
			return err
		}
		if isCommit {
			from = name
			return ErrComplete
		}
		return nil
	})
	if err != nil && err != ErrComplete {
		return "", err
	}

	return from, nil
}

func (r *localReplica) Push(diff io.Reader) error {
	return recv(r.repo, diff)
}

func (r *localReplica) Pull(from string, cb Pusher) error {
	return Pull(r.repo, from, cb)
}

// s3Replica implements the Replica interface using an s3 replica.
type s3Replica struct {
	uri   string
	count int // number of sent commits
}

func newS3Replica(uri string) *s3Replica {
	return &s3Replica{uri, 0}
}

func (r *s3Replica) Push(diff io.Reader) error {
	key := fmt.Sprintf("%.10d", r.count)
	r.count++

	p, err := s3utils.GetPath(r.uri)
	if err != nil {
		return err
	}

	return s3utils.PutMulti(r.uri, path.Join(p, key), diff, "application/octet-stream", s3utils.BucketOwnerFull)
}

func (r *s3Replica) Pull(from string, target Pusher) error {
	client := s3utils.NewClient()
	err := s3utils.ForEachFile(r.uri, from, func(path string, modtime time.Time) (retErr error) {
		response, err := client.GetObject(&s3.GetObjectInput{
			Bucket: &r.uri,
			Key:    &path,
		})

		f := response.Body

		if f == nil {
{'fraction_non_alphanumeric': 0.14122562674094707, 'fraction_numerical': 0.010863509749303621, 'mean_word_length': 3.768924302788845, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3265998', 'n_tokens_mistral': 1459, 'n_tokens_neox': 1430, 'n_words': 355}