<commit_before>package gcs

import (
	"compress/gzip"
	"context"
	"encoding/json"
	"fmt"
	"path"

	"cloud.google.com/go/storage"
	"github.com/estafette/estafette-ci-api/config"
	contracts "github.com/estafette/estafette-ci-contracts"
	"github.com/opentracing/opentracing-go"
)

// CloudStorageClient is the interface for connecting to google cloud storage
type CloudStorageClient interface {
	InsertBuildLog(ctx context.Context, buildLog contracts.BuildLog) (err error)
	InsertReleaseLog(ctx context.Context, releaseLog contracts.ReleaseLog) (err error)
	GetPipelineBuildLogs(ctx context.Context, buildLog contracts.BuildLog) (updatedBuildLog contracts.BuildLog, err error)
	GetPipelineReleaseLogs(ctx context.Context, releaseLog contracts.ReleaseLog) (updatedReleaseLog contracts.ReleaseLog, err error)
}

type cloudStorageClientImpl struct {
	client *storage.Client
	config *config.CloudStorageConfig
}

// NewCloudStorageClient returns new CloudStorageClient
func NewCloudStorageClient(config *config.CloudStorageConfig) (CloudStorageClient, error) {

	if config == nil {
		return &cloudStorageClientImpl{
			config: config,
		}, nil
	}

	ctx := context.Background()

	storageClient, err := storage.NewClient(ctx)
	if err != nil {
		return nil, err
	}

	return &cloudStorageClientImpl{
		client: storageClient,
		config: config,
	}, nil
}

func (impl *cloudStorageClientImpl) InsertBuildLog(ctx context.Context, buildLog contracts.BuildLog) (err error) {

	span, ctx := opentracing.StartSpanFromContext(ctx, "CloudStorageClient::InsertBuildLog")
	defer span.Finish()

	logPath := impl.getBuildLogPath(buildLog)

	return impl.insertLog(ctx, logPath, buildLog.Steps)
}

func (impl *cloudStorageClientImpl) InsertReleaseLog(ctx context.Context, releaseLog contracts.ReleaseLog) (err error) {

	span, ctx := opentracing.StartSpanFromContext(ctx, "CloudStorageClient::InsertReleaseLog")
	defer span.Finish()

	logPath := impl.getReleaseLogPath(releaseLog)

	return impl.insertLog(ctx, logPath, releaseLog.Steps)
}

func (impl *cloudStorageClientImpl) GetPipelineBuildLogs(ctx context.Context, buildLog contracts.BuildLog) (updatedBuildLog contracts.BuildLog, err error) {

	span, ctx := opentracing.StartSpanFromContext(ctx, "CloudStorageClient::GetPipelineBuildLogs")
	defer span.Finish()

	logPath := impl.getBuildLogPath(buildLog)

	steps, err := impl.getLog(ctx, logPath)
	if err != nil {
		return buildLog, err
	}

	updatedBuildLog = buildLog
	updatedBuildLog.Steps = steps

	return updatedBuildLog, nil
}

func (impl *cloudStorageClientImpl) GetPipelineReleaseLogs(ctx context.Context, releaseLog contracts.ReleaseLog) (updatedReleaseLog contracts.ReleaseLog, err error) {

	span, ctx := opentracing.StartSpanFromContext(ctx, "CloudStorageClient::GetPipelineReleaseLogs")
	defer span.Finish()

	logPath := impl.getReleaseLogPath(releaseLog)

	steps, err := impl.getLog(ctx, logPath)
	if err != nil {
		return releaseLog, err
	}

	updatedReleaseLog = releaseLog
	updatedReleaseLog.Steps = steps

	return updatedReleaseLog, nil

}

func (impl *cloudStorageClientImpl) insertLog(ctx context.Context, path string, steps []*contracts.BuildLogStep) (err error) {

	bucket := impl.client.Bucket(impl.config.Bucket)

	// marshal json
	jsonBytes, err := json.Marshal(steps)
	if err != nil {
		return err
	}

	// create writer for cloud storage object
	logObject := bucket.Object(path)
	writer := logObject.NewWriter(ctx)
	if writer == nil {
		return fmt.Errorf("Writer for logobject %v is nil", path)
	}
	defer writer.Close()

	// write compressed bytes
	gz, err := gzip.NewWriterLevel(writer, gzip.BestSpeed)
	if err != nil {
		return err
	}
	_, err = gz.Write(jsonBytes)
	if err != nil {
		_ = writer.Close()
		return err
	}
	err = gz.Close()
	if err != nil {
		_ = writer.Close()
		return err
	}
	err = writer.Close()
	if err != nil {
		return err
	}

	return nil
}<commit_msg>don't overwrite logs in cloud storage, just return without error<commit_after>package gcs

import (
	"compress/gzip"
	"context"
	"encoding/json"
	"fmt"
	"path"

	"cloud.google.com/go/storage"
	"github.com/estafette/estafette-ci-api/config"
	contracts "github.com/estafette/estafette-ci-contracts"
	"github.com/opentracing/opentracing-go"
)

// CloudStorageClient is the interface for connecting to google cloud storage
type CloudStorageClient interface {
	InsertBuildLog(ctx context.Context, buildLog contracts.BuildLog) (err error)
	InsertReleaseLog(ctx context.Context, releaseLog contracts.ReleaseLog) (err error)
	GetPipelineBuildLogs(ctx context.Context, buildLog contracts.BuildLog) (updatedBuildLog contracts.BuildLog, err error)
	GetPipelineReleaseLogs(ctx context.Context, releaseLog contracts.ReleaseLog) (updatedReleaseLog contracts.ReleaseLog, err error)
}

type cloudStorageClientImpl struct {
	client *storage.Client
	config *config.CloudStorageConfig
}

// NewCloudStorageClient returns new CloudStorageClient
func NewCloudStorageClient(config *config.CloudStorageConfig) (CloudStorageClient, error) {

	if config == nil {
		return &cloudStorageClientImpl{
			config: config,
		}, nil
	}

	ctx := context.Background()

	storageClient, err := storage.NewClient(ctx)
	if err != nil {
		return nil, err
	}

	return &cloudStorageClientImpl{
		client: storageClient,
		config: config,
	}, nil
}

func (impl *cloudStorageClientImpl) InsertBuildLog(ctx context.Context, buildLog contracts.BuildLog) (err error) {

	span, ctx := opentracing.StartSpanFromContext(ctx, "CloudStorageClient::InsertBuildLog")
	defer span.Finish()

	logPath := impl.getBuildLogPath(buildLog)

	return impl.insertLog(ctx, logPath, buildLog.Steps)
}

func (impl *cloudStorageClientImpl) InsertReleaseLog(ctx context.Context, releaseLog contracts.ReleaseLog) (err error) {

	span, ctx := opentracing.StartSpanFromContext(ctx, "CloudStorageClient::InsertReleaseLog")
	defer span.Finish()

	logPath := impl.getReleaseLogPath(releaseLog)

	return impl.insertLog(ctx, logPath, releaseLog.Steps)
}

func (impl *cloudStorageClientImpl) GetPipelineBuildLogs(ctx context.Context, buildLog contracts.BuildLog) (updatedBuildLog contracts.BuildLog, err error) {

	span, ctx := opentracing.StartSpanFromContext(ctx, "CloudStorageClient::GetPipelineBuildLogs")
	defer span.Finish()

	logPath := impl.getBuildLogPath(buildLog)

	steps, err := impl.getLog(ctx, logPath)
	if err != nil {
		return buildLog, err
	}

	updatedBuildLog = buildLog
	updatedBuildLog.Steps = steps

	return updatedBuildLog, nil
}

func (impl *cloudStorageClientImpl) GetPipelineReleaseLogs(ctx context.Context, releaseLog contracts.ReleaseLog) (updatedReleaseLog contracts.ReleaseLog, err error) {

	span, ctx := opentracing.StartSpanFromContext(ctx, "CloudStorageClient::GetPipelineReleaseLogs")
	defer span.Finish()

	logPath := impl.getReleaseLogPath(releaseLog)

	steps, err := impl.getLog(ctx, logPath)
	if err != nil {
		return releaseLog, err
	}

	updatedReleaseLog = releaseLog
	updatedReleaseLog.Steps = steps

	return updatedReleaseLog, nil

}

func (impl *cloudStorageClientImpl) insertLog(ctx context.Context, path string, steps []*contracts.BuildLogStep) (err error) {

	bucket := impl.client.Bucket(impl.config.Bucket)

	// marshal json
	jsonBytes, err := json.Marshal(steps)
	if err != nil {
		return err
	}

	// create writer for cloud storage object
	logObject := bucket.Object(path)

	// don't allow overwrites, return when file already exists
	_, err = logObject.Attrs(ctx)
	if err == nil {
		// log file already exists, return
		return nil
	}
	if err != nil && err != storage.ErrObjectNotExist {
		// some other error happened, return it
		return err
	}

	// object doesn't exist, okay to write it
	writer := logObject.NewWriter(ctx)
	if writer == nil {
		return fmt.Errorf("Writer for logobject %v is nil", path)
	}

	// write compressed bytes
	gz, err := gzip.NewWriterLevel(writer, gzip.BestSpeed)
	if err != nil {
		_ = writer.Close()
		return err
	}
	_, err = gz.Write(jsonBytes)
	if err != nil {
		_ = writer.Close()
		return err
	}
	err = gz.Close()
	if err != nil {
		_ = writer.Close()
		return err
	}
	err = writer.Close()
	if err != nil {
		return err
	}

	return nil
}
{'fraction_non_alphanumeric': 0.10582727608722435, 'fraction_numerical': 0.0, 'mean_word_length': 5.1546626231993935, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3711214', 'n_tokens_mistral': 2629, 'n_tokens_neox': 2532, 'n_words': 634}