<commit_before>// cmd/bk/backup.go
// Copyright(c) 2017 <NAME>
// BSD licensed; see LICENSE for details.

package main

import (
	"bytes"
	"encoding/gob"
	"errors"
	"fmt"
	"github.com/mmp/bk/storage"
	"io"
	"io/ioutil"
	"os"
	"path/filepath"
	"strings"
	"sync"
	"time"
)

///////////////////////////////////////////////////////////////////////////
// BackupRoot

// BackupRoot is where it all starts; one of these is serialized and
// stored in the storage backend for each backup.
type BackupRoot struct {
	Dir  DirEntry
	Time time.Time
}

// NewRoot creates a new BackupRoot (as is done when doing a new backup).
func NewRoot(dirpath string) (BackupRoot, error) {
	fi, err := os.Stat(dirpath)
	if err != nil {
		return BackupRoot{}, err
	}
	if !fi.IsDir() {
		return BackupRoot{}, errors.New("not a directory")
	}

	root := BackupRoot{Time: time.Now()}
	root.Dir, err = NewDirEntry(fi)
	root.Dir.Name = "/" // TODO: unneeded?
	return root, err
}

// Reads a BackupRoot from storage, given its hash.
func ReadRoot(hash storage.Hash, backend storage.Backend) (BackupRoot, error) {
	r, err := backend.Read(hash)
	if err != nil {
		return BackupRoot{}, err
	}

	d := gob.NewDecoder(r)
	var root BackupRoot
	if err = d.Decode(&root); err != nil {
		return root, err
	}
	return root, r.Close()
}

func (br BackupRoot) Bytes() []byte {
	var buf bytes.Buffer
	e := gob.NewEncoder(&buf)
	log.CheckError(e.Encode(br))
	return buf.Bytes()
}

///////////////////////////////////////////////////////////////////////////
// DirEntry

// Each file or directory that is in the backup has an associated DirEntry.
type DirEntry struct {
	// Just the file name, not its full path.
	Name string
	// If non-nil, stores the file's contents. Empty files have nil
	// contents and an invalid hash, so it's important to check the Size
	// before trying to make use of either.  Symlink targets are always
	// stored here.
	//
	// TODO: for directories with just a few files, is it worth serializing
	// their children directly into this?
	Contents []byte
	// For a file, if Contents is nil, Hash points to its contents.  For a
	// directory, it gives theserialized []DirEntry for the files in a
	// directory.
	Hash storage.MerkleHash
	// Not used for directories or symlinks.
	Size    int64
	ModTime time.Time
	Mode    os.FileMode
}

func NewDirEntry(fi os.FileInfo) (DirEntry, error) {
	e := DirEntry{
		Name:    fi.Name(),
		Size:    fi.Size(),
		ModTime: fi.ModTime(),
		Mode:    fi.Mode(),
	}
	if !e.IsDir() && !e.IsFile() && !e.IsSymLink() {
		return DirEntry{}, errors.New("unhandled file type")
	}
	return e, nil
}

func (e *DirEntry) IsDir() bool {
	return e.Mode.IsDir()
}

func (e *DirEntry) IsFile() bool {
	return e.Mode&os.ModeType == 0
}

func isFile(mode os.FileMode) bool {
	return mode&os.ModeType == 0
}

func (e *DirEntry) IsSymLink() bool {
	return e.Mode&os.ModeSymlink != 0
}

func writeDirEntries(entries []DirEntry, backend storage.Backend, splitBits uint) storage.MerkleHash {
	var buf bytes.Buffer
	e := gob.NewEncoder(&buf)
	log.CheckError(e.Encode(entries))
	return storage.SplitAndStore(&buf, backend, splitBits)
}

func readDirEntries(hash storage.MerkleHash, backend storage.Backend) []DirEntry {
	r := hash.NewReader(nil, backend)
	d := gob.NewDecoder(r)
	var entries []DirEntry
	log.CheckError(d.Decode(&entries))
	log.CheckError(r.Close())
	return entries
}

func (e *DirEntry) GetContentsReader(sem chan bool, backend storage.Backend) (io.ReadCloser, error) {
	if !e.IsFile() {
		return nil, errors.New("not a file")
	}
	if e.Size == 0 || e.Contents != nil {
		// TODO: I believe that a nil byte slice is ok to pass to
		// NewReader, but should check.
		return ioutil.NopCloser(bytes.NewReader(e.Contents)), nil
	}
	return e.Hash.NewReader(sem, backend), nil
}

///////////////////////////////////////////////////////////////////////////

func BackupDir(dirpath string, backend storage.Backend, splitBits uint) storage.Hash {
	r, err := NewRoot(dirpath)
	if err != nil {
		log.Fatal("%s", err)
	}
	r.Dir.Hash = backupDirContents(dirpath, nil, backend, splitBits)
	return backend.Write(r.Bytes())
}

func BackupDirIncremental(dirpath string, baseHash storage.Hash,
	backend storage.Backend, splitBits uint) storage.Hash {
	r, err := NewRoot(dirpath)
	if err != nil {
		log.Fatal("%s: %s", dirpath, err)
	}

	// Read the entires for the root directory in the backup being used as
	// the base.
	baseRoot, err := ReadRoot(baseHash, backend)
	if err != nil {
		log.Fatal("%s: %s", dirpath, err)
	}
	baseRootEntries := readDirEntries(baseRoot.Dir.Hash, backend)

	r.Dir.Hash = backupDirContents(dirpath, baseRootEntries, backend, splitBits)
	return backend.Write(r.Bytes())
}

// Back up the contents of the given directory (and subdirectories)
// returning a MerkleHash that identifies the serialized []DirEntry for the
// contents.
func backupDirContents(dirpath string, baseEntries []DirEntry,
	backend storage.Backend, splitBits uint) storage.MerkleHash {
	fileinfo, err := ioutil.ReadDir(dirpath)
	log.CheckError(err)

	// For incremental backups, there is an O(n) search that is performed n
	// times in the following, where n is the number of directory entries.
	// Therefore, warn if we come across a directory where this may be
	// problematic.
	if len(baseEntries) > 1000 {
		log.Warning("O(n^2) search with n=%d should probably be revisited",
			len(baseEntries))
	}

	var entries []DirEntry
	for _, f := range fileinfo {
		// Try to find a corresponding file/directory in the base backup, if
		// one was provided.
		var baseEntry *DirEntry
		for _, e := range baseEntries {
			if e.Name == f.Name() && e.Mode == f.Mode() {
				baseEntry = &e
				break
			}
		}

		path := filepath.Join(dirpath, f.Name())
		e, err := NewDirEntry(f)
		if err != nil {
			log.Warning("%s: %s", path, err)
			continue
		}

		switch {
		case e.IsDir():
			if baseEntry != nil {<commit_msg>Give more context in warning message<commit_after>// cmd/bk/backup.go
// Copyright(c) 2017 <NAME>
// BSD licensed; see LICENSE for details.

package main

import (
	"bytes"
	"encoding/gob"
	"errors"
	"fmt"
	"github.com/mmp/bk/storage"
	"io"
	"io/ioutil"
	"os"
	"path/filepath"
	"strings"
	"sync"
	"time"
)

///////////////////////////////////////////////////////////////////////////
// BackupRoot

// BackupRoot is where it all starts; one of these is serialized and
// stored in the storage backend for each backup.
type BackupRoot struct {
	Dir  DirEntry
	Time time.Time
}

// NewRoot creates a new BackupRoot (as is done when doing a new backup).
func NewRoot(dirpath string) (BackupRoot, error) {
	fi, err := os.Stat(dirpath)
	if err != nil {
		return BackupRoot{}, err
	}
	if !fi.IsDir() {
		return BackupRoot{}, errors.New("not a directory")
	}

	root := BackupRoot{Time: time.Now()}
	root.Dir, err = NewDirEntry(fi)
	root.Dir.Name = "/" // TODO: unneeded?
	return root, err
}

// Reads a BackupRoot from storage, given its hash.
func ReadRoot(hash storage.Hash, backend storage.Backend) (BackupRoot, error) {
	r, err := backend.Read(hash)
	if err != nil {
		return BackupRoot{}, err
	}

	d := gob.NewDecoder(r)
	var root BackupRoot
	if err = d.Decode(&root); err != nil {
		return root, err
	}
	return root, r.Close()
}

func (br BackupRoot) Bytes() []byte {
	var buf bytes.Buffer
	e := gob.NewEncoder(&buf)
	log.CheckError(e.Encode(br))
	return buf.Bytes()
}

///////////////////////////////////////////////////////////////////////////
// DirEntry

// Each file or directory that is in the backup has an associated DirEntry.
type DirEntry struct {
	// Just the file name, not its full path.
	Name string
	// If non-nil, stores the file's contents. Empty files have nil
	// contents and an invalid hash, so it's important to check the Size
	// before trying to make use of either.  Symlink targets are always
	// stored here.
	//
	// TODO: for directories with just a few files, is it worth serializing
	// their children directly into this?
	Contents []byte
	// For a file, if Contents is nil, Hash points to its contents.  For a
	// directory, it gives theserialized []DirEntry for the files in a
	// directory.
	Hash storage.MerkleHash
	// Not used for directories or symlinks.
	Size    int64
	ModTime time.Time
	Mode    os.FileMode
}

func NewDirEntry(fi os.FileInfo) (DirEntry, error) {
	e := DirEntry{
		Name:    fi.Name(),
		Size:    fi.Size(),
		ModTime: fi.ModTime(),
		Mode:    fi.Mode(),
	}
	if !e.IsDir() && !e.IsFile() && !e.IsSymLink() {
		return DirEntry{}, errors.New("unhandled file type")
	}
	return e, nil
}

func (e *DirEntry) IsDir() bool {
	return e.Mode.IsDir()
}

func (e *DirEntry) IsFile() bool {
	return e.Mode&os.ModeType == 0
}

func isFile(mode os.FileMode) bool {
	return mode&os.ModeType == 0
}

func (e *DirEntry) IsSymLink() bool {
	return e.Mode&os.ModeSymlink != 0
}

func writeDirEntries(entries []DirEntry, backend storage.Backend, splitBits uint) storage.MerkleHash {
	var buf bytes.Buffer
	e := gob.NewEncoder(&buf)
	log.CheckError(e.Encode(entries))
	return storage.SplitAndStore(&buf, backend, splitBits)
}

func readDirEntries(hash storage.MerkleHash, backend storage.Backend) []DirEntry {
	r := hash.NewReader(nil, backend)
	d := gob.NewDecoder(r)
	var entries []DirEntry
	log.CheckError(d.Decode(&entries))
	log.CheckError(r.Close())
	return entries
}

func (e *DirEntry) GetContentsReader(sem chan bool, backend storage.Backend) (io.ReadCloser, error) {
	if !e.IsFile() {
		return nil, errors.New("not a file")
	}
	if e.Size == 0 || e.Contents != nil {
		// TODO: I believe that a nil byte slice is ok to pass to
		// NewReader, but should check.
		return ioutil.NopCloser(bytes.NewReader(e.Contents)), nil
	}
	return e.Hash.NewReader(sem, backend), nil
}

///////////////////////////////////////////////////////////////////////////

func BackupDir(dirpath string, backend storage.Backend, splitBits uint) storage.Hash {
	r, err := NewRoot(dirpath)
	if err != nil {
		log.Fatal("%s", err)
	}
	r.Dir.Hash = backupDirContents(dirpath, nil, backend, splitBits)
	return backend.Write(r.Bytes())
}

func BackupDirIncremental(dirpath string, baseHash storage.Hash,
	backend storage.Backend, splitBits uint) storage.Hash {
	r, err := NewRoot(dirpath)
	if err != nil {
		log.Fatal("%s: %s", dirpath, err)
	}

	// Read the entires for the root directory in the backup being used as
	// the base.
	baseRoot, err := ReadRoot(baseHash, backend)
	if err != nil {
		log.Fatal("%s: %s", dirpath, err)
	}
	baseRootEntries := readDirEntries(baseRoot.Dir.Hash, backend)

	r.Dir.Hash = backupDirContents(dirpath, baseRootEntries, backend, splitBits)
	return backend.Write(r.Bytes())
}

// Back up the contents of the given directory (and subdirectories)
// returning a MerkleHash that identifies the serialized []DirEntry for the
// contents.
func backupDirContents(dirpath string, baseEntries []DirEntry,
	backend storage.Backend, splitBits uint) storage.MerkleHash {
	fileinfo, err := ioutil.ReadDir(dirpath)
	log.CheckError(err)

	// For incremental backups, there is an O(n) search that is performed n
	// times in the following, where n is the number of directory entries.
	// Therefore, warn if we come across a directory where this may be
	// problematic.
	if len(baseEntries) > 1000 {
		log.Warning("%s: O(n^2) search with n=%d should probably be revisited",
			dirpath, len(baseEntries))
	}

	var entries []DirEntry
	for _, f := range fileinfo {
		// Try to find a corresponding file/directory in the base backup, if
		// one was provided.
		var baseEntry *DirEntry
		for _, e := range baseEntries {
			if e.Name == f.Name() && e.Mode == f.Mode() {
				baseEntry = &e
				break
			}
		}

		path := filepath.Join(dirpath, f.Name())
		e, err := NewDirEntry(f)
		if err != nil {
			log.Warning("%s: %s", path, err)
			continue
		}

		switch {
		case e.IsDir():
			if baseEntry != nil {
{'fraction_non_alphanumeric': 0.1610800744878957, 'fraction_numerical': 0.0025393600812595226, 'mean_word_length': 3.0967406380027738, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5726873', 'n_tokens_mistral': 4038, 'n_tokens_neox': 3890, 'n_words': 1269}