<commit_before>package s3

import (
	"context"
	"fmt"
	"io"
	"os"
	"path"
	"restic"
	"strings"
	"time"

	"restic/backend"
	"restic/errors"

	"github.com/minio/minio-go"
	"github.com/minio/minio-go/pkg/credentials"

	"restic/debug"
)

// Backend stores data on an S3 endpoint.
type Backend struct {
	client *minio.Client
	sem    *backend.Semaphore
	cfg    Config
	backend.Layout
}

// make sure that *Backend implements backend.Backend
var _ restic.Backend = &Backend{}

const defaultLayout = "default"

func open(cfg Config) (*Backend, error) {
	debug.Log("open, config %#v", cfg)

	if cfg.MaxRetries > 0 {
		minio.MaxRetry = int(cfg.MaxRetries)
	}

	var client *minio.Client
	var err error

	if cfg.KeyID == "" || cfg.Secret == "" {
		debug.Log("key/secret not found, trying to get them from IAM")
		creds := credentials.NewIAM("")
		client, err = minio.NewWithCredentials(cfg.Endpoint, creds, !cfg.UseHTTP, "")

		if err != nil {
			return nil, errors.Wrap(err, "minio.NewWithCredentials")
		}
	} else {
		debug.Log("key/secret found")
		client, err = minio.New(cfg.Endpoint, cfg.KeyID, cfg.Secret, !cfg.UseHTTP)

		if err != nil {
			return nil, errors.Wrap(err, "minio.New")
		}
	}

	sem, err := backend.NewSemaphore(cfg.Connections)
	if err != nil {
		return nil, err
	}

	be := &Backend{
		client: client,
		sem:    sem,
		cfg:    cfg,
	}

	client.SetCustomTransport(backend.Transport())

	l, err := backend.ParseLayout(be, cfg.Layout, defaultLayout, cfg.Prefix)
	if err != nil {
		return nil, err
	}

	be.Layout = l

	return be, nil
}

// Open opens the S3 backend at bucket and region. The bucket is created if it
// does not exist yet.
func Open(cfg Config) (restic.Backend, error) {
	return open(cfg)
}

// Create opens the S3 backend at bucket and region and creates the bucket if
// it does not exist yet.
func Create(cfg Config) (restic.Backend, error) {
	be, err := open(cfg)
	found, err := be.client.BucketExists(cfg.Bucket)
	if err != nil {
		debug.Log("BucketExists(%v) returned err %v", cfg.Bucket, err)
		return nil, errors.Wrap(err, "client.BucketExists")<commit_msg>evaluate open error<commit_after>package s3

import (
	"context"
	"fmt"
	"io"
	"os"
	"path"
	"restic"
	"strings"
	"time"

	"restic/backend"
	"restic/errors"

	"github.com/minio/minio-go"
	"github.com/minio/minio-go/pkg/credentials"

	"restic/debug"
)

// Backend stores data on an S3 endpoint.
type Backend struct {
	client *minio.Client
	sem    *backend.Semaphore
	cfg    Config
	backend.Layout
}

// make sure that *Backend implements backend.Backend
var _ restic.Backend = &Backend{}

const defaultLayout = "default"

func open(cfg Config) (*Backend, error) {
	debug.Log("open, config %#v", cfg)

	if cfg.MaxRetries > 0 {
		minio.MaxRetry = int(cfg.MaxRetries)
	}

	var client *minio.Client
	var err error

	if cfg.KeyID == "" || cfg.Secret == "" {
		debug.Log("key/secret not found, trying to get them from IAM")
		creds := credentials.NewIAM("")
		client, err = minio.NewWithCredentials(cfg.Endpoint, creds, !cfg.UseHTTP, "")

		if err != nil {
			return nil, errors.Wrap(err, "minio.NewWithCredentials")
		}
	} else {
		debug.Log("key/secret found")
		client, err = minio.New(cfg.Endpoint, cfg.KeyID, cfg.Secret, !cfg.UseHTTP)

		if err != nil {
			return nil, errors.Wrap(err, "minio.New")
		}
	}

	sem, err := backend.NewSemaphore(cfg.Connections)
	if err != nil {
		return nil, err
	}

	be := &Backend{
		client: client,
		sem:    sem,
		cfg:    cfg,
	}

	client.SetCustomTransport(backend.Transport())

	l, err := backend.ParseLayout(be, cfg.Layout, defaultLayout, cfg.Prefix)
	if err != nil {
		return nil, err
	}

	be.Layout = l

	return be, nil
}

// Open opens the S3 backend at bucket and region. The bucket is created if it
// does not exist yet.
func Open(cfg Config) (restic.Backend, error) {
	return open(cfg)
}

// Create opens the S3 backend at bucket and region and creates the bucket if
// it does not exist yet.
func Create(cfg Config) (restic.Backend, error) {
	be, err := open(cfg)
	if err != nil {
		return nil, errors.Wrap(err, "open")
	}
	found, err := be.client.BucketExists(cfg.Bucket)
	if err != nil {
		debug.Log("BucketExists(%v) returned err %v", cfg.Bucket, err)
		return nil, errors.Wrap(err, "client.BucketExists")
{'fraction_non_alphanumeric': 0.14036329322953528, 'fraction_numerical': 0.0023590469450342063, 'mean_word_length': 3.5788336933045355, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3280483', 'n_tokens_mistral': 1599, 'n_tokens_neox': 1544, 'n_words': 430}