<commit_before>// Copyright 2017 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package dep

import (
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"path/filepath"
	"runtime"
	"syscall"
)

func IsRegular(name string) (bool, error) {
	// TODO: lstat?
	fi, err := os.Stat(name)
	if os.IsNotExist(err) {
		return false, nil
	}
	if err != nil {
		return false, err
	}
	if fi.IsDir() {
		return false, fmt.Errorf("%q is a directory, should be a file", name)
	}
	return true, nil
}

func IsDir(name string) (bool, error) {
	// TODO: lstat?
	fi, err := os.Stat(name)
	if os.IsNotExist(err) {
		return false, nil
	}
	if err != nil {
		return false, err
	}
	if !fi.IsDir() {
		return false, fmt.Errorf("%q is not a directory", name)
	}
	return true, nil
}

func IsNonEmptyDir(name string) (bool, error) {
	isDir, err := IsDir(name)
	if !isDir || err != nil {
		return isDir, err
	}

	files, err := ioutil.ReadDir(name)
	if err != nil {
		return false, err
	}
	return len(files) != 0, nil
}

func writeFile(path string, in json.Marshaler) error {
	f, err := os.Create(path)
	if err != nil {
		return err
	}
	defer f.Close()

	b, err := in.MarshalJSON()
	if err != nil {
		return err
	}

	_, err = f.Write(b)
	return err
}

// renameWithFallback attempts to rename a file or directory, but falls back to
// copying in the event of a cross-link device error. If the fallback copy
// succeeds, src is still removed, emulating normal rename behavior.
func renameWithFallback(src, dest string) error {
	fi, err := os.Lstat(src)
	if err != nil {
		return err
	}

	// Windows cannot use syscall.Rename to rename a directory
	if runtime.GOOS == "windows" && fi.IsDir() {
		if err := CopyDir(src, dest); err != nil {
			return err
		}
		return os.RemoveAll(src)
	}

	err = os.Rename(src, dest)
	if err == nil {
		return nil
	}

	terr, ok := err.(*os.LinkError)
	if !ok {
		return err
	}

	// Rename may fail if src and dest are on different devices; fall back to
	// copy if we detect that case. syscall.EXDEV is the common name for the
	// cross device link error which has varying output text across different
	// operating systems.
	var cerr error
	if terr.Err == syscall.EXDEV {
		if fi.IsDir() {
			cerr = CopyDir(src, dest)
		} else {
			cerr = CopyFile(src, dest)
		}
	} else if runtime.GOOS == "windows" {
		// In windows it can drop down to an operating system call that
		// returns an operating system error with a different number and
		// message. Checking for that as a fall back.
		noerr, ok := terr.Err.(syscall.Errno)
		// 0x11 (ERROR_NOT_SAME_DEVICE) is the windows error.
		// See https://msdn.microsoft.com/en-us/library/cc231199.aspx
		if ok && noerr == 0x11 {
			cerr = CopyFile(src, dest)
		}
	} else {
		return terr
	}

	if cerr != nil {
		return cerr
	}

	return os.RemoveAll(src)
}

// CopyDir takes in a directory and copies its contents to the destination.
// It preserves the file mode on files as well.
func CopyDir(src string, dest string) error {
	fi, err := os.Lstat(src)
	if err != nil {
		return err
	}

	err = os.MkdirAll(dest, fi.Mode())
	if err != nil {
		return err
	}

	dir, err := os.Open(src)
	if err != nil {
		return err
	}
	defer dir.Close()

	objects, err := dir.Readdir(-1)
	if err != nil {
		return err
	}

	for _, obj := range objects {
		if obj.Mode()&os.ModeSymlink != 0 {
			continue
		}<commit_msg>Wrap fs errors to include a better message and callstack<commit_after>// Copyright 2017 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package dep

import (
	"encoding/json"
	"io"
	"io/ioutil"
	"os"
	"path/filepath"
	"runtime"
	"syscall"

	"github.com/pkg/errors"
)

func IsRegular(name string) (bool, error) {
	// TODO: lstat?
	fi, err := os.Stat(name)
	if os.IsNotExist(err) {
		return false, nil
	}
	if err != nil {
		return false, err
	}
	if fi.IsDir() {
		return false, errors.Errorf("%q is a directory, should be a file", name)
	}
	return true, nil
}

func IsDir(name string) (bool, error) {
	// TODO: lstat?
	fi, err := os.Stat(name)
	if os.IsNotExist(err) {
		return false, nil
	}
	if err != nil {
		return false, err
	}
	if !fi.IsDir() {
		return false, errors.Errorf("%q is not a directory", name)
	}
	return true, nil
}

func IsNonEmptyDir(name string) (bool, error) {
	isDir, err := IsDir(name)
	if !isDir || err != nil {
		return isDir, err
	}

	files, err := ioutil.ReadDir(name)
	if err != nil {
		return false, err
	}
	return len(files) != 0, nil
}

func writeFile(path string, in json.Marshaler) error {
	f, err := os.Create(path)
	if err != nil {
		return err
	}
	defer f.Close()

	b, err := in.MarshalJSON()
	if err != nil {
		return err
	}

	_, err = f.Write(b)
	return err
}

// renameWithFallback attempts to rename a file or directory, but falls back to
// copying in the event of a cross-link device error. If the fallback copy
// succeeds, src is still removed, emulating normal rename behavior.
func renameWithFallback(src, dest string) error {
	fi, err := os.Lstat(src)
	if err != nil {
		return errors.Wrapf(err, "cannot stat %s", src)
	}

	// Windows cannot use syscall.Rename to rename a directory
	if runtime.GOOS == "windows" && fi.IsDir() {
		if err := CopyDir(src, dest); err != nil {
			return err
		}
		return errors.Wrapf(os.RemoveAll(src), "cannot delete %s", src)
	}

	err = os.Rename(src, dest)
	if err == nil {
		return nil
	}

	terr, ok := err.(*os.LinkError)
	if !ok {
		return errors.Wrapf(err, "cannot rename %s to %s", src, dest)
	}

	// Rename may fail if src and dest are on different devices; fall back to
	// copy if we detect that case. syscall.EXDEV is the common name for the
	// cross device link error which has varying output text across different
	// operating systems.
	var cerr error
	if terr.Err == syscall.EXDEV {
		if fi.IsDir() {
			cerr = CopyDir(src, dest)
		} else {
			cerr = CopyFile(src, dest)
		}
	} else if runtime.GOOS == "windows" {
		// In windows it can drop down to an operating system call that
		// returns an operating system error with a different number and
		// message. Checking for that as a fall back.
		noerr, ok := terr.Err.(syscall.Errno)
		// 0x11 (ERROR_NOT_SAME_DEVICE) is the windows error.
		// See https://msdn.microsoft.com/en-us/library/cc231199.aspx
		if ok && noerr == 0x11 {
			cerr = CopyFile(src, dest)
		}
	} else {
		return errors.Wrapf(terr, "link error: cannot rename %s to %s", src, dest)
	}

	if cerr != nil {
		return errors.Wrapf(cerr, "second attemp failed: cannot rename %s to %s", src, dest)
	}

	return errors.Wrapf(os.RemoveAll(src), "cannot delete %s", src)
}

// CopyDir takes in a directory and copies its contents to the destination.
// It preserves the file mode on files as well.
func CopyDir(src string, dest string) error {
	fi, err := os.Lstat(src)
	if err != nil {
		return errors.Wrapf(err, "cannot stat %s", src)
	}

	err = os.MkdirAll(dest, fi.Mode())
	if err != nil {
		return errors.Wrapf(err, "cannot mkdir %s", dest)
	}

	dir, err := os.Open(src)
	if err != nil {
		return errors.Wrapf(err, "cannot open %s", src)
	}
	defer dir.Close()

	objects, err := dir.Readdir(-1)
	if err != nil {
		return errors.Wrapf(err, "cannot read directory %s", dir.Name())
	}

	for _, obj := range objects {
		if obj.Mode()&os.ModeSymlink != 0 {
			continue
		}
{'fraction_non_alphanumeric': 0.12785511555615622, 'fraction_numerical': 0.005135829166103527, 'mean_word_length': 3.1156840934371526, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6993952', 'n_tokens_mistral': 2833, 'n_tokens_neox': 2629, 'n_words': 948}