<commit_before>package helpers

import (
	"sync"

	"bytes"

	"io/ioutil"

	"github.com/Seklfreak/Robyul2/cache"
	"github.com/minio/minio-go"
)

var (
	minioBucket string
	minioClient *minio.Client
	minioLock   sync.Mutex
)

// uploads a file to the minio object storage
// objectName	: the name of the file to upload
// data			: the data for the new object
// metadata		: additional metadata attached to the object
func UploadFile(objectName string, data []byte, metadata map[string]string) (err error) {
	// setup minioClient if not yet done
	if minioClient == nil {
		err = setupMinioClient()
		if err != nil {
			return err
		}
	}

	cache.GetLogger().WithField("module", "storage").Infof("uploading " + objectName + " to minio storage")

	options := minio.PutObjectOptions{}

	// add content type
	filetype, err := SniffMime(data)
	if err == nil {
		options.ContentType = filetype
	}

	// add metadata
	if metadata != nil && len(metadata) > 0 {
		options.UserMetadata = metadata
	}

	// upload the data
	_, err = minioClient.PutObject(minioBucket, objectName, bytes.NewReader(data), -1, options)
	return err
}

// TODO: redis cache? file cache?
// retrieves a file from the minio object storage
// objectName	: the name of the file to retrieve
func RetrieveFile(objectName string) (data []byte, err error) {
	// setup minioClient if not yet done
	if minioClient == nil {
		err = setupMinioClient()
		if err != nil {
			return data, err
		}
	}

	cache.GetLogger().WithField("module", "storage").Infof("retrieving " + objectName + " from minio storage")

	// retrieve the object
	minioObject, err := minioClient.GetObject(minioBucket, objectName, minio.GetObjectOptions{})
	if err != nil {
		return data, err
	}

	// read the object into a byte slice
	data, err = ioutil.ReadAll(minioObject)
	if err != nil {
		return data, err
	}
	return data, nil
}

// deletes a file from the minio object storage
// objectName	: the file to delete
func DeleteFile(objectName string) (err error) {
	// setup minioClient if not yet done
	if minioClient == nil {
		err = setupMinioClient()
		if err != nil {
			return err
		}
	}

	cache.GetLogger().WithField("module", "storage").Infof("deleting " + objectName + " from minio storage")

	// delete the object
	err = minioClient.RemoveObject(minioBucket, objectName)
	return err
}

// Initialize the minio client object, and creates the bucket if it doesn't exist yet
func setupMinioClient() (err error) {
	minioLock.Lock()
	minioBucket = GetConfig().Path("s3.bucket").Data().(string)
	minioClient, err = minio.New(
		GetConfig().Path("s3.endpoint").Data().(string),<commit_msg>[storage] adds filecache<commit_after>package helpers

import (
	"sync"

	"bytes"

	"io/ioutil"

	"os"

	"path/filepath"

	"github.com/Seklfreak/Robyul2/cache"
	"github.com/kennygrant/sanitize"
	"github.com/minio/minio-go"
)

var (
	minioBucket string
	minioClient *minio.Client
	minioLock   sync.Mutex
)

// TODO: watch cache folder size

// uploads a file to the minio object storage
// objectName	: the name of the file to upload
// data			: the data for the new object
// metadata		: additional metadata attached to the object
func UploadFile(objectName string, data []byte, metadata map[string]string) (err error) {
	// setup minioClient if not yet done
	if minioClient == nil {
		err = setupMinioClient()
		if err != nil {
			return err
		}
	}

	cache.GetLogger().WithField("module", "storage").Infof("uploading " + objectName + " to minio storage")

	options := minio.PutObjectOptions{}

	// add content type
	filetype, err := SniffMime(data)
	if err == nil {
		options.ContentType = filetype
	}

	// add metadata
	if metadata != nil && len(metadata) > 0 {
		options.UserMetadata = metadata
	}

	// upload the data
	_, err = minioClient.PutObject(minioBucket, sanitize.BaseName(objectName), bytes.NewReader(data), -1, options)
	return err
}

// TODO: redis cache? file cache?
// retrieves a file from the minio object storage
// objectName	: the name of the file to retrieve
func RetrieveFile(objectName string) (data []byte, err error) {
	// setup minioClient if not yet done
	if minioClient == nil {
		err = setupMinioClient()
		if err != nil {
			return data, err
		}
	}

	data = getBucketCache(objectName)
	if data != nil {
		cache.GetLogger().WithField("module", "storage").Infof("retrieving " + objectName + " from minio cache")
		return data, nil
	}

	cache.GetLogger().WithField("module", "storage").Infof("retrieving " + objectName + " from minio storage")

	// retrieve the object
	minioObject, err := minioClient.GetObject(minioBucket, sanitize.BaseName(objectName), minio.GetObjectOptions{})
	if err != nil {
		return data, err
	}

	// read the object into a byte slice
	data, err = ioutil.ReadAll(minioObject)
	if err != nil {
		return data, err
	}

	go func() {
		defer Recover()
		cache.GetLogger().WithField("module", "storage").Infof("caching " + objectName + " into minio cache")
		err = setBucketCache(objectName, data)
		RelaxLog(err)
	}()

	return data, nil
}

// deletes a file from the minio object storage
// objectName	: the file to delete
func DeleteFile(objectName string) (err error) {
	// setup minioClient if not yet done
	if minioClient == nil {
		err = setupMinioClient()
		if err != nil {
			return err
		}
	}

	cache.GetLogger().WithField("module", "storage").Infof("deleting " + objectName + " from minio storage")

	go func() {
		defer Recover()
		cache.GetLogger().WithField("module", "storage").Infof("deleting " + objectName + " from minio cache")
		err = deleteBucketCache(objectName)
		RelaxLog(err)
	}()

	// delete the object
	err = minioClient.RemoveObject(minioBucket, sanitize.BaseName(objectName))
	return err
}

func getBucketCache(objectName string) (data []byte) {
	var err error

	if _, err = os.Stat(getObjectPath(objectName)); os.IsNotExist(err) {
		return nil
	}

	data, err = ioutil.ReadFile(getObjectPath(objectName))
	if err != nil {
		return nil
	}

	return data
}

func setBucketCache(objectName string, data []byte) (err error) {
	if _, err = os.Stat(filepath.Dir(getObjectPath(objectName))); os.IsNotExist(err) {
		err = os.MkdirAll(filepath.Dir(getObjectPath(objectName)), os.ModePerm)
		if err != nil {
			return err
		}
	}

	err = ioutil.WriteFile(getObjectPath(objectName), data, 0644)
	return err
}

func deleteBucketCache(objectName string) (err error) {
	if _, err = os.Stat(getObjectPath(objectName)); os.IsNotExist(err) {
		return nil
	}

	err = os.Remove(getObjectPath(objectName))
	return err
}

func getObjectPath(objectName string) (path string) {
	return GetConfig().Path("cache_folder").Data().(string) + "/minio-" + GetConfig().Path("s3.bucket").Data().(string) + "/" + sanitize.BaseName(objectName)
}

// Initialize the minio client object, and creates the bucket if it doesn't exist yet
func setupMinioClient() (err error) {
	minioLock.Lock()
	minioBucket = GetConfig().Path("s3.bucket").Data().(string)
	minioClient, err = minio.New(
		GetConfig().Path("s3.endpoint").Data().(string),
{'fraction_non_alphanumeric': 0.12816840899326937, 'fraction_numerical': 0.0021480738937419445, 'mean_word_length': 4.028077753779698, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '305801', 'n_tokens_mistral': 2426, 'n_tokens_neox': 2319, 'n_words': 656}