<commit_before>package backup

import (
	"io/ioutil"
	"path/filepath"
	"time"
)

type DataStore interface {
	ExportTo(tmpdir string) (string, error)
}

type Storage interface {
	Copy(localfile string, remotefile string) error
}

type Config struct {
	// Name of backup this job belongs to
	name string

	tempDir       string
	tempDirPrefix string
}

func DefaultConfig() Config {
	return Config{
		tempDir:       "", // empty string forces to use system's defaults
		tempDirPrefix: "",
	}
}

type BackupJobStatus struct {
	Name     string
	Duration time.Duration
	Error    error
}

func errorStatus(name string, start time.Time, err error) BackupJobStatus {
	end := time.Now()
	return BackupJobStatus{
		Name:     name,
		Duration: end.Sub(start),
		Error:    err,
	}
}

type BackupJob struct {
	conf      Config
	datastore DataStore
	storage   Storage
	statusc   chan BackupJobStatus
}

func NewBackupJob(ds DataStore, s Storage, name string, statusc chan BackupJobStatus) *BackupJob {
	conf := DefaultConfig()
	conf.name = name

	return &BackupJob{
		datastore: ds,
		storage:   s,
		conf:      conf,
		statusc:   statusc,
	}
}

func (b *BackupJob) Run() {
	start := time.Now()

	tmpDir, err := ioutil.TempDir(b.conf.tempDir, b.conf.tempDirPrefix)
	if err != nil {
		b.statusc <- errorStatus(b.conf.name, start, err)
		return
	}

	localfile, err := b.datastore.ExportTo(tmpDir)
	if err != nil {
		b.statusc <- errorStatus(b.conf.name, start, err)
		return
	}

	if err := b.storage.Copy(localfile, filepath.Base(localfile)); err != nil {
		b.statusc <- errorStatus(b.conf.name, start, err)
		return
	}

	end := time.Now()

	b.statusc <- BackupJobStatus{
		Name:     b.conf.name,
		Duration: end.Sub(start),
	}
}<commit_msg>feat(backup): set remote storage prefix<commit_after>package backup

import (
	"io/ioutil"
	"path/filepath"
	"time"
)

type DataStore interface {
	ExportTo(tmpdir string) (string, error)
}

type Storage interface {
	Copy(localfile string, remotefile string) error
}

type Config struct {
	// Name of backup this job belongs to
	name string

	tempDir       string
	tempDirPrefix string
}

func DefaultConfig() Config {
	return Config{
		tempDir:       "", // empty string forces to use system's defaults
		tempDirPrefix: "",
	}
}

type BackupJobStatus struct {
	Name     string
	Duration time.Duration
	Error    error
}

func errorStatus(name string, start time.Time, err error) BackupJobStatus {
	end := time.Now()
	return BackupJobStatus{
		Name:     name,
		Duration: end.Sub(start),
		Error:    err,
	}
}

type BackupJob struct {
	conf      Config
	datastore DataStore
	storage   Storage
	statusc   chan BackupJobStatus
}

func NewBackupJob(ds DataStore, s Storage, name string, statusc chan BackupJobStatus) *BackupJob {
	conf := DefaultConfig()
	conf.name = name

	return &BackupJob{
		datastore: ds,
		storage:   s,
		conf:      conf,
		statusc:   statusc,
	}
}

func (b *BackupJob) Run() {
	start := time.Now()

	tmpDir, err := ioutil.TempDir(b.conf.tempDir, b.conf.tempDirPrefix)
	if err != nil {
		b.statusc <- errorStatus(b.conf.name, start, err)
		return
	}

	localfile, err := b.datastore.ExportTo(tmpDir)
	if err != nil {
		b.statusc <- errorStatus(b.conf.name, start, err)
		return
	}

	remotefile := filepath.Join(
		b.conf.name,
		time.Now().Format(time.RFC3339),
		filepath.Base(localfile))
	if err := b.storage.Copy(localfile, remotefile); err != nil {
		b.statusc <- errorStatus(b.conf.name, start, err)
		return
	}

	end := time.Now()

	b.statusc <- BackupJobStatus{
		Name:     b.conf.name,
		Duration: end.Sub(start),
	}
}
{'fraction_non_alphanumeric': 0.11631165919282511, 'fraction_numerical': 0.0011210762331838565, 'mean_word_length': 3.4724310776942358, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1409771', 'n_tokens_mistral': 1330, 'n_tokens_neox': 1279, 'n_words': 335}