<commit_before>// Copyright © 2017 <NAME> <<EMAIL>>
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
// THE SOFTWARE.
package util

import (
	"bytes"
	"errors"
	"io"
	"io/ioutil"
	"os/exec"
	"regexp"
	"time"

	log "github.com/Sirupsen/logrus"
)

const (
	BackupTimeFormat  = time.RFC3339
	saneBackupMinSize = 2 * 1000000 // ~ 4MB

	// Larger files are most likely no backup label
	maxBackupLabelSize = 2048
)

// Backup stores information about a backup
type Backup struct {
	Name             string
	Extension        string
	Path             string
	Bucket           string
	Size             int64
	Created          time.Time
	LabelFile        string
	BackupLabel      string
	StartWalLocation string
	Backups          *Backups
}

// IsSane returns true if the backup seams sane
func (b *Backup) IsSane() (sane bool) {
	if b.Size < saneBackupMinSize {
		return false
	}

	if b.StorageType() == "" {
		return false
	}

	return true
}

// StorageType returns the type of storage the backup is on
func (b *Backup) StorageType() (storageType string) {
	if b.Path > "" {
		return "file"
	}

	if b.Bucket > "" {
		return "s3"
	}

	// Not defined
	return ""
}

// GetStartWalLocation returns the oldest needed WAL file
// Every older WAL file is not required to use this backup
func (b *Backup) GetStartWalLocation() (startWalLocation string, err error) {
	switch b.StorageType() {
	case "file":
		return b.GetStartWalLocationFromFile()
	case "s3":
		return b.GetStartWalLocationFromS3()
	}
	return "", errors.New("Not supported StorageType: " + b.StorageType())
}

// GetStartWalLocationFromFile returns the oldest needed WAL file
// Every older WAL file is not required to use this backup
func (b *Backup) GetStartWalLocationFromFile() (startWalLocation string, err error) {
	// Regex to identify a backup label file
	// 000000010000001200000062.00000028.backup, make better regex
	regBackupLabelFile := regexp.MustCompile(`.*\.backup`)

	// Regex to identify the right file
	regLabel := regexp.MustCompile(`.*LABEL: ` + b.Name)

	files, _ := ioutil.ReadDir(b.Backups.WalDir)
	// find all backup labels
	for _, f := range files {
		if f.Size() > maxBackupLabelSize {
			// size is to big for backup label
			continue
		}
		if regBackupLabelFile.MatchString(f.Name()) {
			log.Debug(f.Name(), " => seems to be a backup Label, by size and name")

			labelFile := b.Backups.WalDir + "/" + f.Name()
			catCmd := exec.Command("/usr/bin/zstdcat", labelFile)
			catCmdStdout, err := catCmd.StdoutPipe()
			if err != nil {
				// if we can not open the file we continue with next
				log.Warn("catCmd.StdoutPipe(), ", err)
				continue
			}

			err = catCmd.Start()<commit_msg>Fix WAL deletion problem for fs backups<commit_after>// Copyright © 2017 <NAME> <<EMAIL>>
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
// THE SOFTWARE.
package util

import (
	"bytes"
	"errors"
	"io"
	"io/ioutil"
	"os/exec"
	"regexp"
	"time"

	log "github.com/Sirupsen/logrus"
)

const (
	BackupTimeFormat  = time.RFC3339
	saneBackupMinSize = 2 * 1000000 // ~ 4MB

	// Larger files are most likely no backup label
	maxBackupLabelSize = 2048
)

// Backup stores information about a backup
type Backup struct {
	Name             string
	Extension        string
	Path             string
	Bucket           string
	Size             int64
	Created          time.Time
	LabelFile        string
	BackupLabel      string
	StartWalLocation string
	Backups          *Backups
}

// IsSane returns true if the backup seams sane
func (b *Backup) IsSane() (sane bool) {
	if b.Size < saneBackupMinSize {
		return false
	}

	if b.StorageType() == "" {
		return false
	}

	return true
}

// StorageType returns the type of storage the backup is on
func (b *Backup) StorageType() (storageType string) {
	if b.Path > "" {
		return "file"
	}

	if b.Bucket > "" {
		return "s3"
	}

	// Not defined
	return ""
}

// GetStartWalLocation returns the oldest needed WAL file
// Every older WAL file is not required to use this backup
func (b *Backup) GetStartWalLocation() (startWalLocation string, err error) {
	switch b.StorageType() {
	case "file":
		return b.GetStartWalLocationFromFile()
	case "s3":
		return b.GetStartWalLocationFromS3()
	}
	return "", errors.New("Not supported StorageType: " + b.StorageType())
}

// GetStartWalLocationFromFile returns the oldest needed WAL file
// Every older WAL file is not required to use this backup
func (b *Backup) GetStartWalLocationFromFile() (startWalLocation string, err error) {
	// Regex to identify a backup label file
	// 000000010000001200000062.00000028.backup, make better regex
	regBackupLabelFile := regexp.MustCompile(`.*\.backup`)

	// Escape the name so we can use it in a regular expression
	searchName := regexp.QuoteMeta(b.Name)
	// Regex to identify the right file
	regLabel := regexp.MustCompile(`.*LABEL: ` + searchName)
	log.Debug("regLabel: ", regLabel)

	files, _ := ioutil.ReadDir(b.Backups.WalDir)
	// find all backup labels
	for _, f := range files {
		if f.Size() > maxBackupLabelSize {
			// size is to big for backup label
			continue
		}
		if regBackupLabelFile.MatchString(f.Name()) {
			log.Debug(f.Name(), " => seems to be a backup Label, by size and name")

			labelFile := b.Backups.WalDir + "/" + f.Name()
			catCmd := exec.Command("/usr/bin/zstdcat", labelFile)
			catCmdStdout, err := catCmd.StdoutPipe()
			if err != nil {
				// if we can not open the file we continue with next
				log.Warn("catCmd.StdoutPipe(), ", err)
				continue
			}

			err = catCmd.Start()
{'fraction_non_alphanumeric': 0.0966196433324461, 'fraction_numerical': 0.015437849347883949, 'mean_word_length': 3.336410848240046, 'pattern_counts': {'":': 4, '<': 11, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 2, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6150207', 'n_tokens_mistral': 2448, 'n_tokens_neox': 2227, 'n_words': 879}