<commit_before>package model

import (
	"io"
	"log"
	"os"
	"os/exec"
	"path"
	"path/filepath"
	"strings"
)

func IsDirectory(loc string) bool {
	s, err := os.Stat(loc)
	return err == nil && s.IsDir()
}

func IsRegular(loc string) bool {
	s, err := os.Stat(loc)
	return err == nil && !s.IsDir()
}
func Exists(loc string) bool {
	_, err := os.Stat(loc)
	return err == nil
}

// Cp copy a file (src) to new file (dst).
// Dst is the full directory path and file name
// Src can be a relative directory path and file name
func Cp(dst, src string) error {
	cleanSrc := path.Clean(src)
	cleanDst := path.Clean(dst)
	log.Printf("copy [%s] to [%s]", cleanSrc, cleanDst)
	if err := os.MkdirAll(filepath.Dir(cleanDst), os.ModePerm); err != nil {
		return err
	}
	return exec.Command("cp", cleanSrc, cleanDst).Run()
}

// Copy does what is says. Ignores errors on Close though.
func Copy(dst, src string) error {
	in, err := os.Open(src)
	if err != nil {
		return err
	}
	defer in.Close()
	out, err := os.Create(dst)
	if err != nil {
		return err
	}
	defer out.Close()
	_, err = io.Copy(out, in)
	cerr := out.Close()
	if err != nil {
		return err
	}
	return cerr
}

func Targz(sourceDir, destinationFile string) error {
	log.Printf("creating tape archive [%s] from [%s]\n", destinationFile, sourceDir)
	// and exclude the archive created
	cmd, line := asCommand(
		"tar",
		"-czvf",
		destinationFile,
		"--exclude",
		filepath.Base(destinationFile),
		"-C",
		sourceDir,
		".")
	data, err := cmd.CombinedOutput()
	if err != nil {
		log.Println(line)
		log.Println(string(data))
	}
	return err
}

func Untargz(sourceFile, destinationDir string) error {
	log.Printf("extracting tape archive [%s] to [%s]\n", sourceFile, destinationDir)
	cmd, line := asCommand(
		"tar",
		"-xvf",
		sourceFile,
		"-C",
		destinationDir)
	data, err := cmd.CombinedOutput()
	if err != nil {
		log.Println(line)
		log.Println(string(data))
	}
	return err
}

func FileRemove(source string) error {
	log.Printf("removing [%s]\n", source)
	return os.Remove(source)
}
<commit_msg>handle difference in tar behaviour (OS X vs Ubuntu)<commit_after>package model

import (
	"io"
	"log"
	"os"
	"os/exec"
	"path"
	"path/filepath"
	"strings"
)

func IsDirectory(loc string) bool {
	s, err := os.Stat(loc)
	return err == nil && s.IsDir()
}

func IsRegular(loc string) bool {
	s, err := os.Stat(loc)
	return err == nil && !s.IsDir()
}
func Exists(loc string) bool {
	_, err := os.Stat(loc)
	return err == nil
}

// Cp copy a file (src) to new file (dst).
// Dst is the full directory path and file name
// Src can be a relative directory path and file name
func Cp(dst, src string) error {
	cleanSrc := path.Clean(src)
	cleanDst := path.Clean(dst)
	log.Printf("copy [%s] to [%s]", cleanSrc, cleanDst)
	if err := os.MkdirAll(filepath.Dir(cleanDst), os.ModePerm); err != nil {
		return err
	}
	return exec.Command("cp", cleanSrc, cleanDst).Run()
}

// Copy does what is says. Ignores errors on Close though.
func Copy(dst, src string) error {
	in, err := os.Open(src)
	if err != nil {
		return err
	}
	defer in.Close()
	out, err := os.Create(dst)
	if err != nil {
		return err
	}
	defer out.Close()
	_, err = io.Copy(out, in)
	cerr := out.Close()
	if err != nil {
		return err
	}
	return cerr
}

func Targz(sourceDir, destinationFile string) error {
	log.Printf("creating tape archive [%s] from [%s]\n", destinationFile, sourceDir)
	tmp := filepath.Join(os.TempDir(), filepath.Base(destinationFile))
	cmd, line := asCommand(
		"tar",
		"czvf",
		tmp,
		"-C",
		sourceDir,
		".")
	data, err := cmd.CombinedOutput()
	if err != nil {
		log.Println(line)
		log.Println(string(data))
		return err
	}
	// now move to destination
	return os.Rename(tmp, destinationFile)
}

func Untargz(sourceFile, destinationDir string) error {
	log.Printf("extracting tape archive [%s] to [%s]\n", sourceFile, destinationDir)
	cmd, line := asCommand(
		"tar",
		"xvf",
		sourceFile,
		"-C",
		destinationDir)
	data, err := cmd.CombinedOutput()
	if err != nil {
		log.Println(line)
		log.Println(string(data))
	}
	return err
}

func FileRemove(source string) error {
	log.Printf("removing [%s]\n", source)
	return os.Remove(source)
}

{'fraction_non_alphanumeric': 0.1563400576368876, 'fraction_numerical': 0.0, 'mean_word_length': 3.7168742921857305, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2799018', 'n_tokens_mistral': 1631, 'n_tokens_neox': 1575, 'n_words': 453}