<commit_before>package s3proxy

import (
	"context"
	"errors"
	"fmt"
	"io"
	"log"
	"path"

	"github.com/buchgr/bazel-remote/cache"
	"github.com/buchgr/bazel-remote/cache/disk/casblob"
	"github.com/minio/minio-go/v7"
	"github.com/minio/minio-go/v7/pkg/credentials"
	"github.com/prometheus/client_golang/prometheus"
	"github.com/prometheus/client_golang/prometheus/promauto"
)

type uploadReq struct {
	hash string
	size int64
	kind cache.EntryKind
	rc   io.ReadCloser
}

type s3Cache struct {
	mcore        *minio.Core
	prefix       string
	bucket       string
	uploadQueue  chan<- uploadReq
	accessLogger cache.Logger
	errorLogger  cache.Logger
	v2mode       bool
	objectKey    func(hash string, kind cache.EntryKind) string
}

var (
	cacheHits = promauto.NewCounter(prometheus.CounterOpts{
		Name: "bazel_remote_s3_cache_hits",
		Help: "The total number of s3 backend cache hits",
	})
	cacheMisses = promauto.NewCounter(prometheus.CounterOpts{
		Name: "bazel_remote_s3_cache_misses",
		Help: "The total number of s3 backend cache misses",
	})
)

// Used in place of minio's verbose "NoSuchKey" error.
var errNotFound = errors.New("NOT FOUND")

// New returns a new instance of the S3-API based cache
func New(
	// S3CloudStorageConfig struct fields:
	Endpoint string,
	Bucket string,
	Prefix string,
	AccessKeyID string,
	SecretAccessKey string,
	DisableSSL bool,
	IAMRoleEndpoint string,
	Region string,

	storageMode string, accessLogger cache.Logger,
	errorLogger cache.Logger, numUploaders, maxQueuedUploads int) cache.Proxy {

	fmt.Println("Using S3 backend.")

	var minioCore *minio.Core
	var err error

	if AccessKeyID != "" && SecretAccessKey != "" {
		// Initialize minio client object.
		opts := &minio.Options{
			Creds:  credentials.NewStaticV4(AccessKeyID, SecretAccessKey, ""),
			Secure: !DisableSSL,
			Region: Region,
		}
		minioCore, err = minio.NewCore(Endpoint, opts)
		if err != nil {
			log.Fatalln(err)
		}
	} else {
		// Initialize minio client object with IAM credentials
		opts := &minio.Options{
			// This config value may be empty.
			Creds: credentials.NewIAM(IAMRoleEndpoint),

			Region: Region,
			Secure: !DisableSSL,
		}
		minioCore, err = minio.NewCore(Endpoint, opts)
		if err != nil {
			log.Fatalln(err)
		}
	}

	if storageMode != "zstd" && storageMode != "uncompressed" {
		log.Fatalf("Unsupported storage mode for the s3proxy backend: %q, must be one of \"zstd\" or \"uncompressed\"",
			storageMode)
	}

	c := &s3Cache{
		mcore:        minioCore,
		prefix:       Prefix,<commit_msg>[s3proxy] Simplify minio client construction<commit_after>package s3proxy

import (
	"context"
	"errors"
	"fmt"
	"io"
	"log"
	"path"

	"github.com/buchgr/bazel-remote/cache"
	"github.com/buchgr/bazel-remote/cache/disk/casblob"
	"github.com/minio/minio-go/v7"
	"github.com/minio/minio-go/v7/pkg/credentials"
	"github.com/prometheus/client_golang/prometheus"
	"github.com/prometheus/client_golang/prometheus/promauto"
)

type uploadReq struct {
	hash string
	size int64
	kind cache.EntryKind
	rc   io.ReadCloser
}

type s3Cache struct {
	mcore        *minio.Core
	prefix       string
	bucket       string
	uploadQueue  chan<- uploadReq
	accessLogger cache.Logger
	errorLogger  cache.Logger
	v2mode       bool
	objectKey    func(hash string, kind cache.EntryKind) string
}

var (
	cacheHits = promauto.NewCounter(prometheus.CounterOpts{
		Name: "bazel_remote_s3_cache_hits",
		Help: "The total number of s3 backend cache hits",
	})
	cacheMisses = promauto.NewCounter(prometheus.CounterOpts{
		Name: "bazel_remote_s3_cache_misses",
		Help: "The total number of s3 backend cache misses",
	})
)

// Used in place of minio's verbose "NoSuchKey" error.
var errNotFound = errors.New("NOT FOUND")

// New returns a new instance of the S3-API based cache
func New(
	// S3CloudStorageConfig struct fields:
	Endpoint string,
	Bucket string,
	Prefix string,
	AccessKeyID string,
	SecretAccessKey string,
	DisableSSL bool,
	IAMRoleEndpoint string,
	Region string,

	storageMode string, accessLogger cache.Logger,
	errorLogger cache.Logger, numUploaders, maxQueuedUploads int) cache.Proxy {

	fmt.Println("Using S3 backend.")

	var minioCore *minio.Core
	var err error
	var creds *credentials.Credentials

	if AccessKeyID != "" && SecretAccessKey != "" {
		creds = credentials.NewStaticV4(AccessKeyID, SecretAccessKey, "")
	} else {
		creds = credentials.NewIAM(IAMRoleEndpoint)
	}

	// Initialize minio client with credentials
	opts := &minio.Options{
		Creds: creds,

		Region: Region,
		Secure: !DisableSSL,
	}
	minioCore, err = minio.NewCore(Endpoint, opts)
	if err != nil {
		log.Fatalln(err)
	}

	if storageMode != "zstd" && storageMode != "uncompressed" {
		log.Fatalf("Unsupported storage mode for the s3proxy backend: %q, must be one of \"zstd\" or \"uncompressed\"",
			storageMode)
	}

	c := &s3Cache{
		mcore:        minioCore,
		prefix:       Prefix,
{'fraction_non_alphanumeric': 0.10755336617405582, 'fraction_numerical': 0.007183908045977011, 'mean_word_length': 3.9522357723577235, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6114794', 'n_tokens_mistral': 1809, 'n_tokens_neox': 1705, 'n_words': 422}