<commit_before>// Copyright (c) 2016 <NAME> <<EMAIL>>

package logstash

import (
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"path/filepath"
	"sort"
)

// copyFile copies the contents of the source filename to the
// destination file path. If the destination file exists it'll be
// overwritten. Symbolic links will be followed and file mode,
// ownership, etc will not be copied.
func copyFile(sourcePath, destPath string) error {
	r, err := os.Open(sourcePath)
	if err != nil {
		return err
	}
	w, err := os.Create(destPath)
	if err != nil {
		return err
	}
	_, err = io.Copy(w, r)
	return err
}

// getConfigFileDir copies one or more configuration files into the
// root of a newly created temporary directory and returns the path to
// that directory. Returns an error if any I/O error occurs but also
// if the basenames of the configuration files aren't unique, i.e. if
// they'd overwrite one another in the directory.
func getConfigFileDir(configs []string) (string, error) {
	dir, err := ioutil.TempDir("", "")
	if err != nil {
		return "", err
	}

	for _, f := range configs {
		dest := filepath.Join(dir, filepath.Base(f))
		_, err = os.Stat(dest)
		if err == nil {
			_ = os.RemoveAll(dir)
			return "", fmt.Errorf(
				"The collected list of configuration files contains "+
					"two files with the name %q which isn't allowed.",
				filepath.Base(f))
		} else if !os.IsNotExist(err) {
			_ = os.RemoveAll(dir)
			return "", err
		}
		err = copyFile(f, dest)
		if err != nil {
			_ = os.RemoveAll(dir)
			return "", err
		}
	}
	fileList, err := getFilesInDir(dir)
	if err == nil {
		log.Debug("Prepared configuration file directory %s with these files: %v", dir, fileList)
	} else {
		// Don't let this failure fail the whole function
		// call, but log it as a warning since it's highly
		// irregular and might be indicative of other
		// problems.
		log.Warning("Unexpected error when locating configuration files: %s", err)
	}
	return dir, nil
}

// getFilesInDir returns a sorted list of the names of the
// (non-directory) files in the given directory.
func getFilesInDir(dir string) ([]string, error) {
	filenames := make([]string, 0)
	files, err := ioutil.ReadDir(dir)
	if err != nil {
		return nil, err
	}
	for _, f := range files {
		if !f.Mode().IsDir() {
			filenames = append(filenames, f.Name())
		}
	}
	sort.Strings(filenames)
	return filenames, nil
}
<commit_msg>More useful error messages<commit_after>// Copyright (c) 2016 <NAME> <<EMAIL>>

package logstash

import (
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"path/filepath"
	"sort"
)

// copyFile copies the contents of the source filename to the
// destination file path. If the destination file exists it'll be
// overwritten. Symbolic links will be followed and file mode,
// ownership, etc will not be copied.
func copyFile(sourcePath, destPath string) error {
	r, err := os.Open(sourcePath)
	if err != nil {
		return err
	}
	w, err := os.Create(destPath)
	if err != nil {
		return err
	}
	_, err = io.Copy(w, r)
	return err
}

// getConfigFileDir copies one or more configuration files into the
// root of a newly created temporary directory and returns the path to
// that directory. Returns an error if any I/O error occurs but also
// if the basenames of the configuration files aren't unique, i.e. if
// they'd overwrite one another in the directory.
func getConfigFileDir(configs []string) (string, error) {
	dir, err := ioutil.TempDir("", "")
	if err != nil {
		return "", err
	}

	for _, f := range configs {
		dest := filepath.Join(dir, filepath.Base(f))
		_, err = os.Stat(dest)
		if err == nil {
			_ = os.RemoveAll(dir)
			return "", fmt.Errorf(
				"The collected list of configuration files contains "+
					"two files with the name %q which isn't allowed.",
				filepath.Base(f))
		} else if !os.IsNotExist(err) {
			_ = os.RemoveAll(dir)
			return "", err
		}
		err = copyFile(f, dest)
		if err != nil {
			_ = os.RemoveAll(dir)
			return "", fmt.Errorf("Config file copy failed: %s", err)
		}
	}
	fileList, err := getFilesInDir(dir)
	if err == nil {
		log.Debug("Prepared configuration file directory %s with these files: %v", dir, fileList)
	} else {
		// Don't let this failure fail the whole function
		// call, but log it as a warning since it's highly
		// irregular and might be indicative of other
		// problems.
		log.Warning("Unexpected error when locating configuration files: %s", err)
	}
	return dir, nil
}

// getFilesInDir returns a sorted list of the names of the
// (non-directory) files in the given directory.
func getFilesInDir(dir string) ([]string, error) {
	filenames := make([]string, 0)
	files, err := ioutil.ReadDir(dir)
	if err != nil {
		return nil, err
	}
	for _, f := range files {
		if !f.Mode().IsDir() {
			filenames = append(filenames, f.Name())
		}
	}
	sort.Strings(filenames)
	return filenames, nil
}

{'fraction_non_alphanumeric': 0.12360016590626297, 'fraction_numerical': 0.002073828287017835, 'mean_word_length': 3.3528880866425994, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7359040', 'n_tokens_mistral': 1707, 'n_tokens_neox': 1588, 'n_words': 584}