<commit_before>package bob

import (
	"errors"
	"io"
	"io/ioutil"
	"log"
	"os"
)

/*
CopyFile copies one file from source to dest.  Copied from
https://github.com/opesun/copyrecur.
*/
func CopyFile(source string, dest string) (err error) {
	sf, err := os.Open(source)
	if err != nil {
		return err
	}
	defer sf.Close()
	df, err := os.Create(dest)
	if err != nil {
		return err
	}
	defer df.Close()
	_, err = io.Copy(df, sf)
	if err == nil {
		si, err := os.Stat(source)
		if err != nil {
			err = os.Chmod(dest, si.Mode())
		}
	}
	return
}

/*
CopyDir recursively copies one dir from source to dest.  Copied from
https://github.com/opesun/copyrecur.
*/
func CopyDir(source string, dest string) (err error) {
	// get properties of source dir
	fi, err := os.Stat(source)
	if err != nil {
		return err
	}
	if !fi.IsDir() {
		return errors.New("source is not a directory")
	}
	// ensure dest dir does not already exist
	_, err = os.Open(dest)
	if !os.IsNotExist(err) {
		return errors.New("destination already exists")
	}
	// create dest dir
	err = os.MkdirAll(dest, fi.Mode())
	if err != nil {
		return err
	}
	entries, err := ioutil.ReadDir(source)
	for _, entry := range entries {
		sfp := source + "/" + entry.Name()
		dfp := dest + "/" + entry.Name()
		if entry.IsDir() {
			err = CopyDir(sfp, dfp)
			if err != nil {
				log.Println(err)
			}
		} else {
			// perform copy
			err = CopyFile(sfp, dfp)
			if err != nil {
				log.Println(err)
			}
		}

	}
	return
}<commit_msg>Refactoring the file/dir copy helpers<commit_after>package bob

import (
	"errors"
	"io"
	"io/ioutil"
	"os"
)

/*
CopyFile copies one file from source to dest.  Copied from
https://gist.github.com/elazarl/5507969 and modified.
*/
func CopyFile(s string, d string) (err error) {
	source, err := os.Open(s)
	if err != nil {
		return
	}

	defer source.Close()

	dest, err := os.Create(d)
	if err != nil {
		return
	}

	_, err = io.Copy(dest, source)
	if err != nil {
		return
	}

	return dest.Close()
}

/*
CopyDir recursively copies one dir from source to dest.  Copied from
https://github.com/opesun/copyrecur.
*/
func CopyDir(source string, dest string) (err error) {
	// get properties of source dir
	sourceInfo, err := os.Stat(source)
	if err != nil {
		return err
	}
	if !sourceInfo.IsDir() {
		return errors.New("source is not a directory")
	}

	// ensure dest dir does not already exist
	if _, err = os.Open(dest); !os.IsNotExist(err) {
		return errors.New("destination already exists")
	}
	// create dest dir
	if err = os.MkdirAll(dest, sourceInfo.Mode()); err != nil {
		return
	}

	files, err := ioutil.ReadDir(source)

	for _, file := range files {
		sourceFilePath := source + "/" + file.Name()
		destFilePath := dest + "/" + file.Name()

		if file.IsDir() {
			if err = CopyDir(sourceFilePath, destFilePath); err != nil {
				return
			}
		} else {
			if err = CopyFile(sourceFilePath, destFilePath); err != nil {
				return
			}
		}

	}
	return
}
{'fraction_non_alphanumeric': 0.14582623509369677, 'fraction_numerical': 0.002385008517887564, 'mean_word_length': 2.994557823129252, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 4, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7340362', 'n_tokens_mistral': 1165, 'n_tokens_neox': 1087, 'n_words': 313}