<commit_before>package fsutil

import (
	"archive/zip"
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"path/filepath"
	"runtime"

	"github.com/spf13/afero"
)

// ReadSecureFile ensures a file has 0400 or 0600 permissions before reading it.
// Permissions check is skipped on Windows as it uses a different mechanism than UNIX.
func ReadSecureFile(fs afero.Fs, filename string) ([]byte, error) {
	f, err := fs.Open(filename)
	if err != nil {
		return nil, err
	}
	defer f.Close()

	if runtime.GOOS != "windows" {
		fi, err := f.Stat()
		if err != nil {
			return nil, err
		}
		perm := fi.Mode().Perm()
		if perm&0177 != 0 {
			return nil, fmt.Errorf(
				"permissions %#o for '%s' are too open, expected 0400 or 0600", perm, filename)
		}
	}
	return ioutil.ReadAll(f)
}

// Unzip extracts a ZIP archive into a given destination.
func Unzip(fs afero.Fs, src, dest string) error {
	f, err := fs.Open(src)
	if err != nil {
		return err<commit_msg>Add helpers in fsutil<commit_after>package fsutil

import (
	"archive/zip"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"os"
	"path/filepath"
	"runtime"

	"github.com/spf13/afero"
)

// Copy copies a file into a given destination.
func Copy(fs afero.Fs, src, dest string, perm os.FileMode) error {
	srcFile, err := os.Open(src)
	if err != nil {
		return err
	}
	defer srcFile.Close()

	return CopyReader(fs, srcFile, dest, perm)
}

// CopyReader copies an io.Reader into a given destination.
func CopyReader(fs afero.Fs, r io.Reader, dest string, perm os.FileMode) error {
	destFile, err := fs.OpenFile(dest, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, perm)
	if err != nil {
		return err
	}
	defer destFile.Close()

	_, err = io.Copy(destFile, r)
	return err
}

// ReadSecureFile ensures a file has 0400 or 0600 permissions before reading it.
// Permissions check is skipped on Windows as it uses a different mechanism than UNIX.
func ReadSecureFile(fs afero.Fs, filename string) ([]byte, error) {
	f, err := fs.Open(filename)
	if err != nil {
		return nil, err
	}
	defer f.Close()

	if runtime.GOOS != "windows" {
		fi, err := f.Stat()
		if err != nil {
			return nil, err
		}
		perm := fi.Mode().Perm()
		if perm&0177 != 0 {
			return nil, fmt.Errorf(
				"permissions %#o for '%s' are too open, expected 0400 or 0600", perm, filename)
		}
	}
	return ioutil.ReadAll(f)
}

// DetectMediaType detects a file's media type, as defined in
// https://www.iana.org/assignments/media-types/media-types.xhtml.
func DetectMediaType(fs afero.Fs, path string) (string, error) {
	f, err := fs.Open(path)
	if err != nil {
		return "", err
	}
	defer f.Close()

	sniffBuf := make([]byte, 512)
	if _, err := io.ReadFull(f, sniffBuf); err != nil {
		return "", err
	}
	return http.DetectContentType(sniffBuf), nil
}

// Unzip extracts a ZIP archive into a given destination.
func Unzip(fs afero.Fs, src, dest string) error {
	f, err := fs.Open(src)
	if err != nil {
		return err
{'fraction_non_alphanumeric': 0.13546370267453978, 'fraction_numerical': 0.017019798541160126, 'mean_word_length': 3.458204334365325, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 1, 'lorem ipsum': 0, 'www.': 1, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2184372', 'n_tokens_mistral': 1157, 'n_tokens_neox': 1064, 'n_words': 339}