<commit_before>// package fsync keeps two files or directories in sync.
//
//         err := fsync.Sync("~/dst", ".")
//
// After the above code, if err is nil, every file and directory in the current
// directory is copied to ~/dst and has the same permissions. Consequent calls
// will only copy changed or new files. You can use SyncDel to also delete
// extra files in the destination:
//
//         err := fsync.SyncDel("~/dst", ".")
package fsync

import (
	"bytes"
	"errors"
	"io"
	"io/ioutil"
	"os"
	"path"
	"runtime"
)

var (
	ErrFileOverDir = errors.New(
		"fsync: trying to overwrite a non-empty directory with a file")
)

// Sync copies files and directories inside src into dst.
func Sync(dst, src string) error {
	// make sure src exists
	if _, err := os.Stat(src); err != nil {
		return err
	}
	// return error instead of replacing a non-empty directory with a file
	if b, err := checkDir(dst, src); err != nil {
		return err
	} else if b {
		return ErrFileOverDir
	}

	return syncRecover(false, dst, src)
}

// SyncDel makes sure dst is a copy of src. It's only difference with Sync is in
// deleting files in dst that are not found in src.
func SyncDel(dst, src string) error {
	// return error instead of replacing a non-empty directory with a file
	if b, err := checkDir(dst, src); err != nil {
		return err
	} else if b {
		return ErrFileOverDir
	}

	return syncRecover(true, dst, src)
}

// SyncTo syncs srcs files or directories **into** to directory. Calling
//
//         SyncTo("a", "b", "c/d")
//
// is equivalent to calling
//
//         Sync("a/b", "b")
//         Sync("a/d", "c/d")
//
// Actually, this is also implementation of SyncTo.
func SyncTo(to string, srcs ...string) error {
	for _, src := range srcs {
		dst := path.Join(to, path.Base(src))
		if err := Sync(dst, src); err != nil {
			return err
		}
	}
	return nil
}

// SyncDelTo syncs srcs files or directories **into** to directory. It differs
// with SyncDelTo in using SyncDel instead of Sync.
func SyncDelTo(to string, srcs ...string) error {
	for _, src := range srcs {
		dst := path.Join(to, path.Base(src))
		if err := SyncDel(dst, src); err != nil {
			return err
		}
	}
	return nil
}

// syncRecover handles errors and calls sync
func syncRecover(del bool, dst, src string) (err error) {
	defer func() {
		if r := recover(); r != nil {
			if _, ok := r.(runtime.Error); ok {
				panic(r)
			}
			err = r.(error)
		}
	}()

	sync(del, dst, src)
	return nil
}

// sync updates dst to match with src, handling both files and directories.
func sync(del bool, dst, src string) {
	// sync permissions after handling content
	defer syncperms(dst, src)

	// read files info
	d, err := os.Stat(dst)
	if err != nil && !os.IsNotExist(err) {
		panic(err)
	}
	s, err := os.Stat(src)
	if err != nil && os.IsNotExist(err) {
		return // src was deleted before we could copy it
	}
	check(err)

	if !s.IsDir() {
		// src is a file
		// delete dst if its a directory
		if d != nil && d.IsDir() {
			check(os.RemoveAll(dst))
		}
		if !equal(dst, src) {
			// perform copy
			df, err := os.Create(dst)
			check(err)
			defer df.Close()
			sf, err := os.Open(src)
			check(err)
			defer sf.Close()
			_, err = io.Copy(df, sf)
			check(err)
		}
		return
	}

	// src is a directory
	// make dst if necessary
	if d == nil {
		// dst does not exist; create directory
		check(os.MkdirAll(dst, 0755)) // permissions will be synced later
	} else if !d.IsDir() {
		// dst is a file; remove and create directory
		check(os.Remove(dst))
		check(os.MkdirAll(dst, 0755)) // permissions will be synced later
	}

	// go through sf files and sync them
	files, err := ioutil.ReadDir(src)
	check(err)
	// make a map of filenames for quick lookup; used in deletion<commit_msg>Ignore file-doesn't-exist error when traversing<commit_after>// package fsync keeps two files or directories in sync.
//
//         err := fsync.Sync("~/dst", ".")
//
// After the above code, if err is nil, every file and directory in the current
// directory is copied to ~/dst and has the same permissions. Consequent calls
// will only copy changed or new files. You can use SyncDel to also delete
// extra files in the destination:
//
//         err := fsync.SyncDel("~/dst", ".")
package fsync

import (
	"bytes"
	"errors"
	"io"
	"io/ioutil"
	"os"
	"path"
	"runtime"
)

var (
	ErrFileOverDir = errors.New(
		"fsync: trying to overwrite a non-empty directory with a file")
)

// Sync copies files and directories inside src into dst.
func Sync(dst, src string) error {
	// make sure src exists
	if _, err := os.Stat(src); err != nil {
		return err
	}
	// return error instead of replacing a non-empty directory with a file
	if b, err := checkDir(dst, src); err != nil {
		return err
	} else if b {
		return ErrFileOverDir
	}

	return syncRecover(false, dst, src)
}

// SyncDel makes sure dst is a copy of src. It's only difference with Sync is in
// deleting files in dst that are not found in src.
func SyncDel(dst, src string) error {
	// return error instead of replacing a non-empty directory with a file
	if b, err := checkDir(dst, src); err != nil {
		return err
	} else if b {
		return ErrFileOverDir
	}

	return syncRecover(true, dst, src)
}

// SyncTo syncs srcs files or directories **into** to directory. Calling
//
//         SyncTo("a", "b", "c/d")
//
// is equivalent to calling
//
//         Sync("a/b", "b")
//         Sync("a/d", "c/d")
//
// Actually, this is also implementation of SyncTo.
func SyncTo(to string, srcs ...string) error {
	for _, src := range srcs {
		dst := path.Join(to, path.Base(src))
		if err := Sync(dst, src); err != nil {
			return err
		}
	}
	return nil
}

// SyncDelTo syncs srcs files or directories **into** to directory. It differs
// with SyncDelTo in using SyncDel instead of Sync.
func SyncDelTo(to string, srcs ...string) error {
	for _, src := range srcs {
		dst := path.Join(to, path.Base(src))
		if err := SyncDel(dst, src); err != nil {
			return err
		}
	}
	return nil
}

// syncRecover handles errors and calls sync
func syncRecover(del bool, dst, src string) (err error) {
	defer func() {
		if r := recover(); r != nil {
			if _, ok := r.(runtime.Error); ok {
				panic(r)
			}
			err = r.(error)
		}
	}()

	sync(del, dst, src)
	return nil
}

// sync updates dst to match with src, handling both files and directories.
func sync(del bool, dst, src string) {
	// sync permissions after handling content
	defer syncperms(dst, src)

	// read files info
	d, err := os.Stat(dst)
	if err != nil && !os.IsNotExist(err) {
		panic(err)
	}
	s, err := os.Stat(src)
	if err != nil && os.IsNotExist(err) {
		return // src was deleted before we could copy it
	}
	check(err)

	if !s.IsDir() {
		// src is a file
		// delete dst if its a directory
		if d != nil && d.IsDir() {
			check(os.RemoveAll(dst))
		}
		if !equal(dst, src) {
			// perform copy
			df, err := os.Create(dst)
			check(err)
			defer df.Close()
			sf, err := os.Open(src)
			if os.IsNotExist(err) {
				return
			}
			check(err)
			defer sf.Close()
			_, err = io.Copy(df, sf)
			if os.IsNotExist(err) {
				return
			}
			check(err)
		}
		return
	}

	// src is a directory
	// make dst if necessary
	if d == nil {
		// dst does not exist; create directory
		check(os.MkdirAll(dst, 0755)) // permissions will be synced later
	} else if !d.IsDir() {
		// dst is a file; remove and create directory
		check(os.Remove(dst))
		check(os.MkdirAll(dst, 0755)) // permissions will be synced later
	}

	// go through sf files and sync them
	files, err := ioutil.ReadDir(src)
	if os.IsNotExist(err) {
		return
	}
	check(err)
	// make a map of filenames for quick lookup; used in deletion
{'fraction_non_alphanumeric': 0.13343819635600995, 'fraction_numerical': 0.002097260453532573, 'mean_word_length': 2.8169084542271134, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '150287', 'n_tokens_mistral': 2833, 'n_tokens_neox': 2635, 'n_words': 974}